fix broken test.

.github/workflows/TESTING.md

@@ -1,19 +0,0 @@
-# Testing
-
-Most testing is handled automatically by our GitHub Actions workflows.
-
-## Local GitHub Action Testing
-
-In some cases you may wish to submit changes to the Trufflehog GitHub Action. Unfortunately GitHub does not provide a 1st-party testing environment for testing actions outside of GitHub Actions.
-
-Fortunately [nektos/act](https://github.com/nektos/act) enables local testing of GitHub Actions.
-
-### Instructions
-
-1. Please follow [the installation instructions](http://https://github.com/nektos/act#installation) for your OS.
-2. The first run of `act` will ask you to specify an image. `Medium` should suffice.
-3. You'll need to configure a personal-access-token(PAT) with: `repo:status`, `repo_deployment`, and `public_repo` permissions.
-4. Set an environment variable named `GITHUB_TOKEN` with the PAT from the previous step as the value: `$ export GITHUB_TOKEN=<your_PAT>`
-5. Run the following command from the repository root: `act pull_request -j test -W .github/workflows/secrets.yml -s GITHUB_TOKEN --defaultbranch main`
-6. If the job was successful, you should expect to see output from the scanner showing several detected secrets.
-7. If you want to omit the context of a pull request event and just test that the action starts successfully, run: `act -j test -W .github/workflows/secrets.yml -s GITHUB_TOKEN --defaultbranch main`

.github/workflows/release.yml

@@ -1,9 +1,16 @@
 name: Release
 
+# on:
+#   push:
+#     tags:
+#       - 'v*'
+
 on:
   push:
     tags:
       - v*
+    branches:
+      - main
 
 permissions:
   contents: write

.github/workflows/secrets.yml

@@ -7,7 +7,6 @@ on:
       - v*
     branches:
       - main
-    pull_request:
 
 jobs:
   test:

.github/workflows/smoke.yml

@@ -1,24 +0,0 @@
-
-name: Smoke
-
-on:
-  pull_request:
-
-jobs:
-  smoke:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v3
-      with:
-        go-version: '1.18'
-    - name: Checkout code
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        ref: ${{ github.head_ref }}
-    - name: Smoke
-      run: |
-        set -e
-        go run . git https://github.com/dustin-decker/secretsandstuff.git
-        go run . github --repo https://github.com/dustin-decker/secretsandstuff.git

.gitignore

@@ -1,7 +1,4 @@
 .idea
 dist
 .env
-*.test
+*.test
-
-# binary
-trufflehog

.pre-commit-hooks.yaml

@@ -1,6 +0,0 @@
-- id: trufflehog
-  name: TruffleHog
-  description: Detect secrets in your data with TruffleHog.
-  entry: trufflehog git file://. --since-commit HEAD --only-verified --fail
-  language: golang
-  pass_filenames: false

README.md

@@ -100,12 +100,6 @@ trufflehog s3 --bucket=<bucket name> --only-verified
 docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
 ```
 
-### Example 6: Scan individual files or directories
-
-```bash
-trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
-```
-
 # What's new in v3?
 
 TruffleHog v3 is a complete rewrite in Go with many new powerful features.
@@ -173,10 +167,10 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
 - github
 - gitlab
 - S3
-- filesystem (files and directories)
+- filesystem
 - syslog
 - circleci
-- stdin (coming soon)
+- file and stdin (coming soon)
 
 Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
 
@@ -190,7 +184,6 @@ Flags:
       --help                     Show context-sensitive help (also try --help-long and --help-man).
       --debug                    Run in debug mode.
       --trace                    Run in trace mode.
-      --profile                  Enables profiling and sets a pprof and fgprof server on :18066.
   -j, --json                     Output in JSON format.
       --json-legacy              Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
       --concurrency=10           Number of concurrent workers.
@@ -318,7 +311,7 @@ detectors:
 ```
 
 ```
-$ trufflehog filesystem /tmp --config config.yaml --only-verified
+$ trufflehog filesystem --directory /tmp --config config.yaml --only-verified
 🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷
 
 Found verified result 🐷🔑

go.mod

@@ -11,7 +11,7 @@ replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff
 require (
 	cloud.google.com/go/secretmanager v1.10.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
-	github.com/TheZeroSlave/zapsentry v1.14.0
+	github.com/TheZeroSlave/zapsentry v1.12.0
 	github.com/aws/aws-sdk-go v1.44.83
 	github.com/bill-rich/disk-buffer-reader v0.1.7
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
@@ -22,7 +22,7 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.9.1
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.3
-	github.com/getsentry/sentry-go v0.18.0
+	github.com/getsentry/sentry-go v0.17.0
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.5.2
 	github.com/go-ldap/ldap/v3 v3.4.4
@@ -41,22 +41,25 @@ require (
 	github.com/jpillora/overseer v1.1.6
 	github.com/kylelemons/godebug v1.1.0
 	github.com/lib/pq v1.10.7
+	github.com/mattn/go-colorable v0.1.13
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/mholt/archiver/v4 v4.0.0-alpha.7
 	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/pkg/errors v0.9.1
-	github.com/rabbitmq/amqp091-go v1.7.0
+	github.com/rabbitmq/amqp091-go v1.6.0
 	github.com/sergi/go-diff v1.3.1
-	github.com/stretchr/testify v1.8.2
+	github.com/sirupsen/logrus v1.9.0
+	github.com/stretchr/testify v1.8.1
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
-	github.com/xanzy/go-gitlab v0.80.2
+	github.com/xanzy/go-gitlab v0.78.0
-	go.mongodb.org/mongo-driver v1.11.2
+	go.mongodb.org/mongo-driver v1.11.1
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.6.0
+	golang.org/x/crypto v0.5.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
-	golang.org/x/oauth2 v0.5.0
+	golang.org/x/oauth2 v0.4.0
 	golang.org/x/sync v0.1.0
-	golang.org/x/text v0.7.0
+	golang.org/x/text v0.6.0
+	google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
@@ -113,14 +116,12 @@ require (
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/klauspost/pgzip v1.2.5 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
 	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.23.0 // indirect
-	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
 	github.com/pierrec/lz4/v4 v4.1.14 // indirect
 	github.com/pjbgf/sha1cd v0.2.3 // indirect
 	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
@@ -138,13 +139,12 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/net v0.5.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.3.0 // indirect
 	google.golang.org/api v0.103.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect
 	google.golang.org/grpc v1.51.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -37,8 +37,8 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
 github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
-github.com/TheZeroSlave/zapsentry v1.14.0 h1:z6H0lIyKVpdXsP5K8J11b4/+V+QCDLDslZmtXsnduMk=
+github.com/TheZeroSlave/zapsentry v1.12.0 h1:hhfwvD7pQbnCIOvAEGM4NXLiaX4b6W6ZWNtL6awSBGc=
-github.com/TheZeroSlave/zapsentry v1.14.0/go.mod h1:0hUR1Et5tSroEYhWOZcxE2iBqcFkuG7jGGlUmOA7XAs=
+github.com/TheZeroSlave/zapsentry v1.12.0/go.mod h1:00uO/VpPrSJG/XigAfTi0F4WMFIw2DmP/IDVUhPBvNw=
 github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
 github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
@@ -101,8 +101,8 @@ github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNu
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0=
+github.com/getsentry/sentry-go v0.17.0 h1:UustVWnOoDFHBS7IJUB2QK/nB5pap748ZEp0swnQJak=
-github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ=
+github.com/getsentry/sentry-go v0.17.0/go.mod h1:B82dxtBvxG0KaPD8/hfSV+VcHD+Lg/xUS4JuQn1P4cM=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
@@ -281,8 +281,6 @@ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
-github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
-github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
@@ -297,13 +295,15 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rabbitmq/amqp091-go v1.7.0 h1:V5CF5qPem5OGSnEo8BoSbsDGwejg6VUJsKEdneaoTUo=
+github.com/rabbitmq/amqp091-go v1.6.0 h1:fiFHcMcP+5/A+35MXniiagho1lEXX4+jpHGR+q+LeBU=
-github.com/rabbitmq/amqp091-go v1.7.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
+github.com/rabbitmq/amqp091-go v1.6.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/skeema/knownhosts v1.1.0 h1:Wvr9V0MxhjRbl3f9nMnKnFfiWTJmtECJ9Njkea3ysW0=
 github.com/skeema/knownhosts v1.1.0/go.mod h1:sKFq3RD6/TKZkSWn8boUbDC7Qkgcv+8XXijpFO6roag=
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
@@ -322,9 +322,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
@@ -336,8 +335,8 @@ github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhN
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/xanzy/go-gitlab v0.80.2 h1:CH1Q7NDklqZllox4ICVF4PwlhQGfPtE+w08Jsb74ZX0=
+github.com/xanzy/go-gitlab v0.78.0 h1:8jUHfQVAprG04Av5g0PxVd3CNsZ5hCbojIax7Hba1mE=
-github.com/xanzy/go-gitlab v0.80.2/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M=
+github.com/xanzy/go-gitlab v0.78.0/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -353,8 +352,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
+go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
-go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
@@ -378,8 +377,8 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
-golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 h1:lhh9BuEmjIBOdOhMIGSQzdW5LbTtFbUV1k9O/Rlq3SE=
 golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
@@ -412,11 +411,11 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
+golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
-golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
+golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -459,21 +458,21 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
-golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
+golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

hack/generate/generate.go

@@ -9,8 +9,6 @@ import (
 	"text/template"
 
 	"github.com/go-errors/errors"
-	"golang.org/x/text/cases"
-	"golang.org/x/text/language"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
@@ -26,7 +24,7 @@ func main() {
 	log.SetPrefix("😲 [generate] ")
 
 	kingpin.MustParse(app.Parse(os.Args[1:]))
-	nameTitle = cases.Title(language.AmericanEnglish).String(*name)
+	nameTitle = strings.Title(*name)
 	nameLower = strings.ToLower(*name)
 	nameUpper = strings.ToUpper(*name)
 
@@ -67,7 +65,7 @@ type templateJob struct {
 }
 
 func mustWriteTemplates(jobs []templateJob) {
-	log.Printf("Generating %s %s\n", cases.Title(language.AmericanEnglish).String(*kind), nameTitle)
+	log.Printf("Generating %s %s\n", strings.Title(*kind), nameTitle)
 
 	// Make the folder.
 	log.Printf("Creating folder %s\n", folderPath())
@@ -86,7 +84,7 @@ func mustWriteTemplates(jobs []templateJob) {
 
 		for _, rplString := range job.ReplaceString {
 			tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToLower(rplString), "<<.NameLower>>")
-			tmplRaw = strings.ReplaceAll(tmplRaw, cases.Title(language.AmericanEnglish).String(rplString), "<<.NameTitle>>")
+			tmplRaw = strings.ReplaceAll(tmplRaw, strings.Title(rplString), "<<.NameTitle>>")
 			tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToUpper(rplString), "<<.NameUpper>>")
 		}
 

hack/snifftest/main.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/paulbellamy/ratecounter"
+	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/semaphore"
 	"gopkg.in/alecthomas/kingpin.v2"
 
@@ -18,7 +19,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -42,20 +42,6 @@ var (
 )
 
 func main() {
-	// setup logger
-	logger, flush := log.New("trufflehog", log.WithConsoleSink(os.Stderr))
-	// make it the default logger for contexts
-	context.SetDefaultLogger(logger)
-	defer func() { _ = flush() }()
-	logFatal := func(err error, message string, keyAndVals ...any) {
-		logger.Error(err, message, keyAndVals...)
-		if err != nil {
-			os.Exit(1)
-			return
-		}
-		os.Exit(0)
-	}
-
 	ctx, cancel := context.WithTimeout(context.Background(), time.Hour*2)
 	var cancelOnce sync.Once
 	defer cancelOnce.Do(cancel)
@@ -82,19 +68,19 @@ func main() {
 		} else {
 			_, ok := allScanners[input]
 			if !ok {
-				logFatal(fmt.Errorf("invalid input"), "could not find scanner by that name")
+				log.Fatal("could not find scanner by that name")
 			}
 			selectedScanners[input] = allScanners[input]
 		}
 		if len(selectedScanners) == 0 {
-			logFatal(fmt.Errorf("invalid input"), "no detectors selected")
+			log.Fatal("no detectors selected")
 		}
 
 		for _, excluded := range *scanCmdExclude {
 			delete(selectedScanners, excluded)
 		}
 
-		logger.Info("loaded secret detectors", "count", len(selectedScanners)+3)
+		log.Infof("loaded %d secret detectors", len(selectedScanners)+3)
 
 		var wgScanners sync.WaitGroup
 
@@ -106,7 +92,7 @@ func main() {
 				time.Sleep(60 * time.Second)
 				counter.Incr(int64(chunkCounter - prev))
 				prev = chunkCounter
-				logger.Info("chunk scan rate per second", "rate", counter.Rate()/60)
+				log.Infof("chunk scan rate: %d/sec", counter.Rate()/60)
 			}
 		}()
 
@@ -136,7 +122,7 @@ func main() {
 
 								res, err := scanner.FromData(ctx, *scanVerify, decoded.Data)
 								if err != nil {
-									logFatal(err, "error scanning chunk")
+									log.Fatal(err)
 								}
 								if len(res) > 0 {
 									if resCounter[name] == nil {
@@ -145,19 +131,16 @@ func main() {
 									}
 									atomic.AddUint64(resCounter[name], uint64(len(res)))
 									if *scanThreshold != 0 && int(*resCounter[name]) > *scanThreshold {
-										logger.Error(
+										log.WithField("scanner", name).Errorf("exceeded result threshold of %d", *scanThreshold)
-											fmt.Errorf("exceeded result threshold"), "snifftest failed",
-											"scanner", name, "threshold", *scanThreshold,
-										)
 										failed = true
 										os.Exit(1)
 									}
 
 									if *scanPrintRes {
 										for _, r := range res {
-											logger := logger.WithValues("secret", name, "meta", chunk.SourceMetadata, "result", string(r.Raw))
+											logger := log.WithField("secret", name).WithField("meta", chunk.SourceMetadata).WithField("result", string(r.Raw))
 											if *scanPrintChunkRes {
-												logger = logger.WithValues("chunk", string(decoded.Data))
+												logger = logger.WithField("chunk", string(decoded.Data))
 											}
 											logger.Info("result")
 										}
@@ -173,20 +156,18 @@ func main() {
 		}
 
 		for _, repo := range strings.Split(*scanCmdRepo, ",") {
-			if err := sem.Acquire(ctx, 1); err != nil {
+			sem.Acquire(ctx, 1)
-				logFatal(err, "timed out waiting for semaphore")
-			}
 			wgChunkers.Add(1)
 			go func(r string) {
 				defer sem.Release(1)
 				defer wgChunkers.Done()
-				logger.Info("cloning repo", "repo", r)
+				log.Infof("cloning %s", r)
 				path, repo, err := git.CloneRepoUsingUnauthenticated(ctx, r)
 				if err != nil {
-					logFatal(err, "error cloning repo", "repo", r)
+					log.Fatal(err)
 				}
 
-				logger.Info("cloned repo", "repo", r)
+				log.Infof("cloned %s", r)
 
 				s := git.NewGit(sourcespb.SourceType_SOURCE_TYPE_GIT, 0, 0, "snifftest", false, runtime.NumCPU(),
 					func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
@@ -203,12 +184,12 @@ func main() {
 						}
 					})
 
-				logger.Info("scanning repo", "repo", r)
+				log.Infof("scanning %s", r)
 				err = s.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
 				if err != nil {
-					logFatal(err, "error scanning repo")
+					log.Fatal(err)
 				}
-				logger.Info("scanned repo", "repo", r)
+				log.Infof("scanned %s", r)
 				defer os.RemoveAll(path)
 			}(repo)
 		}
@@ -220,9 +201,9 @@ func main() {
 
 		wgScanners.Wait()
 
-		logger.Info("completed snifftest", "chunks", chunkCounter)
+		log.WithField("chunks", chunkCounter).Info("completed")
 		for scanner, resultsCount := range resCounter {
-			logger.Info(scanner, "results", *resultsCount)
+			log.WithField("results", *resultsCount).Info(scanner)
 		}
 
 		if failed {

main.go

@@ -12,21 +12,19 @@ import (
 	"time"
 
 	"github.com/felixge/fgprof"
-	"github.com/go-logr/logr"
 	"github.com/gorilla/mux"
 	"github.com/jpillora/overseer"
+	"github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/config"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/output"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
@@ -38,7 +36,6 @@ var (
 	cmd              string
 	debug            = cli.Flag("debug", "Run in debug mode.").Bool()
 	trace            = cli.Flag("trace", "Run in trace mode.").Bool()
-	profile          = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
 	jsonOut          = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
 	jsonLegacy       = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
 	concurrency      = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
@@ -50,11 +47,9 @@ var (
 	printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
 	noUpdate             = cli.Flag("no-update", "Don't check for updates.").Bool()
 	fail                 = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
-	archiveMaxSize       = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
+	archiveMaxSize       = cli.Flag("archive-max-size", "Maximum size of archive to scan.").Bytes()
 	archiveMaxDepth      = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
 	archiveTimeout       = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
-	includeDetectors     = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
-	excludeDetectors     = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
 
 	gitScan             = cli.Command("git", "Find credentials in git repositories.")
 	gitScanURI          = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
@@ -67,17 +62,15 @@ var (
 	_                   = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
 	_                   = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
 
-	githubScan             = cli.Command("github", "Find credentials in GitHub repositories.")
+	githubScan           = cli.Command("github", "Find credentials in GitHub repositories.")
-	githubScanEndpoint     = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
+	githubScanEndpoint   = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
-	githubScanRepos        = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
+	githubScanRepos      = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings()
-	githubScanOrgs         = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
+	githubScanOrgs       = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings()
-	githubScanToken        = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
+	githubScanToken      = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String()
-	githubIncludeForks     = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
+	githubIncludeForks   = githubScan.Flag("include-forks", "Include forks in scan.").Bool()
-	githubIncludeMembers   = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
+	githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool()
-	githubIncludeRepos     = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
+	githubIncludeRepos   = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
-	githubExcludeRepos     = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
+	githubExcludeRepos   = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
-	githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
-	githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
 	gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
 	// TODO: Add more GitLab options
@@ -87,10 +80,8 @@ var (
 	gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
-	filesystemScan  = cli.Command("filesystem", "Find credentials in a filesystem.")
+	filesystemScan        = cli.Command("filesystem", "Find credentials in a filesystem.")
-	filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
+	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
-	// DEPRECATED: --directory is deprecated in favor of arguments.
-	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings()
 	// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
 	// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
 	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
@@ -125,26 +116,24 @@ func init() {
 	cli.Version("trufflehog " + version.BuildVersion)
 	cmd = kingpin.MustParse(cli.Parse(os.Args[1:]))
 
+	if *jsonOut {
+		logrus.SetFormatter(&logrus.JSONFormatter{})
+	}
 	switch {
 	case *trace:
 		log.SetLevel(5)
+		logrus.SetLevel(logrus.TraceLevel)
+		logrus.Debugf("running version %s", version.BuildVersion)
 	case *debug:
 		log.SetLevel(2)
+		logrus.SetLevel(logrus.DebugLevel)
+		logrus.Debugf("running version %s", version.BuildVersion)
+	default:
+		logrus.SetLevel(logrus.InfoLevel)
 	}
 }
 
 func main() {
-	// setup logger
-	logFormat := log.WithConsoleSink
-	if *jsonOut {
-		logFormat = log.WithJSONSink
-	}
-	logger, sync := log.New("trufflehog", logFormat(os.Stderr))
-	// make it the default logger for contexts
-	context.SetDefaultLogger(logger)
-	defer func() { _ = sync() }()
-	logFatal := logFatalFunc(logger)
-
 	updateCfg := overseer.Config{
 		Program:       run,
 		Debug:         *debug,
@@ -162,16 +151,14 @@ func main() {
 
 	err := overseer.RunErr(updateCfg)
 	if err != nil {
-		logFatal(err, "error occured with trufflehog updater 🐷")
+		logrus.WithError(err).Fatal("error occured with trufflehog updater 🐷")
 	}
 }
 
 func run(state overseer.State) {
-	ctx := context.Background()
+	if *debug {
-	logger := ctx.Logger()
+		logrus.Debugf("trufflehog %s", version.BuildVersion)
-	logFatal := logFatalFunc(logger)
+	}
-
-	logger.V(2).Info(fmt.Sprintf("trufflehog %s", version.BuildVersion))
 
 	if *githubScanToken != "" {
 		// NOTE: this kludge is here to do an authenticated shallow commit
@@ -184,24 +171,32 @@ func run(state overseer.State) {
 		*concurrency = 1
 	}
 
-	if *profile {
+	if *debug {
 		go func() {
 			router := mux.NewRouter()
 			router.PathPrefix("/debug/pprof").Handler(http.DefaultServeMux)
 			router.PathPrefix("/debug/fgprof").Handler(fgprof.Handler())
-			logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
+			logrus.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
 			if err := http.ListenAndServe(":18066", router); err != nil {
-				logger.Error(err, "error serving pprof and fgprof")
+				logrus.Error(err)
 			}
 		}()
 	}
+	logFormat := log.WithConsoleSink
+	if *jsonOut {
+		logFormat = log.WithJSONSink
+	}
+	logger, sync := log.New("trufflehog", logFormat(os.Stderr))
+	context.SetDefaultLogger(logger)
+	defer func() { _ = sync() }()
 
 	conf := &config.Config{}
 	if *configFilename != "" {
 		var err error
 		conf, err = config.Read(*configFilename)
 		if err != nil {
-			logFatal(err, "error parsing the provided configuration file")
+			logger.Error(err, "error parsing the provided configuration file")
+			os.Exit(1)
 		}
 	}
 
@@ -215,62 +210,12 @@ func run(state overseer.State) {
 		handlers.SetArchiveMaxTimeout(*archiveTimeout)
 	}
 
-	// Build include and exclude detector filter sets.
+	ctx := context.TODO()
-	var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]config.DetectorID
-	{
-		includeList, err := config.ParseDetectors(*includeDetectors)
-		if err != nil {
-			// Exit if there was an error to inform the user of the misconfiguration.
-			logFatal(err, "invalid include list detector configuration")
-		}
-		excludeList, err := config.ParseDetectors(*excludeDetectors)
-		if err != nil {
-			// Exit if there was an error to inform the user of the misconfiguration.
-			logFatal(err, "invalid exclude list detector configuration")
-		}
-		includeDetectorTypes = detectorTypeToMap(includeList)
-		excludeDetectorTypes = detectorTypeToMap(excludeList)
-	}
-	includeFilter := func(d detectors.Detector) bool {
-		id, ok := includeDetectorTypes[d.Type()]
-		if id.Version == 0 {
-			return ok
-		}
-		versionD, ok := d.(detectors.Versioner)
-		if !ok {
-			// Error: version provided but not a detectors.Versioner
-			logFatal(
-				fmt.Errorf("version provided but detector does not have a version"),
-				"invalid include list detector configuration",
-				"detector", id,
-			)
-		}
-		return versionD.Version() == id.Version
-	}
-	excludeFilter := func(d detectors.Detector) bool {
-		id, ok := excludeDetectorTypes[d.Type()]
-		if id.Version == 0 {
-			return !ok
-		}
-		versionD, ok := d.(detectors.Versioner)
-		if !ok {
-			// Error: version provided but not a detectors.Versioner
-			logFatal(
-				fmt.Errorf("version provided but detector does not have a version"),
-				"invalid exclude list detector configuration",
-				"detector", id,
-			)
-		}
-		return versionD.Version() != id.Version
-	}
-
 	e := engine.Start(ctx,
 		engine.WithConcurrency(*concurrency),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
 		engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
 		engine.WithDetectors(!*noVerification, conf.Detectors...),
-		engine.WithFilterDetectors(includeFilter),
-		engine.WithFilterDetectors(excludeFilter),
 		engine.WithFilterUnverified(*filterUnverified),
 	)
 
@@ -280,11 +225,11 @@ func run(state overseer.State) {
 	case gitScan.FullCommand():
 		filter, err := common.FilterFromFiles(*gitScanIncludePaths, *gitScanExcludePaths)
 		if err != nil {
-			logFatal(err, "could not create filter")
+			logrus.WithError(err).Fatal("could not create filter")
 		}
 		repoPath, remote, err = git.PrepareRepoSinceCommit(ctx, *gitScanURI, *gitScanSinceCommit)
 		if err != nil || repoPath == "" {
-			logFatal(err, "error preparing git repo for scanning")
+			logrus.WithError(err).Fatal("error preparing git repo for scanning")
 		}
 		if remote {
 			defer os.RemoveAll(repoPath)
@@ -298,15 +243,11 @@ func run(state overseer.State) {
 			Filter:   filter,
 		}
 		if err = e.ScanGit(ctx, cfg); err != nil {
-			logFatal(err, "Failed to scan Git.")
+			logrus.WithError(err).Fatal("Failed to scan Git.")
 		}
 	case githubScan.FullCommand():
-		filter, err := common.FilterFromFiles(*githubScanIncludePaths, *githubScanExcludePaths)
-		if err != nil {
-			logFatal(err, "could not create filter")
-		}
 		if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
-			logFatal(fmt.Errorf("invalid config"), "You must specify at least one organization or repository.")
+			logrus.Fatal("You must specify at least one organization or repository.")
 		}
 
 		cfg := sources.GithubConfig{
@@ -319,15 +260,14 @@ func run(state overseer.State) {
 			IncludeRepos:   *githubIncludeRepos,
 			Repos:          *githubScanRepos,
 			Orgs:           *githubScanOrgs,
-			Filter:         filter,
 		}
 		if err := e.ScanGitHub(ctx, cfg); err != nil {
-			logFatal(err, "Failed to scan Github.")
+			logrus.WithError(err).Fatal("Failed to scan Github.")
 		}
 	case gitlabScan.FullCommand():
 		filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
 		if err != nil {
-			logFatal(err, "could not create filter")
+			logrus.WithError(err).Fatal("could not create filter")
 		}
 
 		cfg := sources.GitlabConfig{
@@ -337,25 +277,20 @@ func run(state overseer.State) {
 			Filter:   filter,
 		}
 		if err := e.ScanGitLab(ctx, cfg); err != nil {
-			logFatal(err, "Failed to scan GitLab.")
+			logrus.WithError(err).Fatal("Failed to scan GitLab.")
 		}
 	case filesystemScan.FullCommand():
 		filter, err := common.FilterFromFiles(*filesystemScanIncludePaths, *filesystemScanExcludePaths)
 		if err != nil {
-			logFatal(err, "could not create filter")
+			logrus.WithError(err).Fatal("could not create filter")
 		}
-		if len(*filesystemDirectories) > 0 {
+
-			ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
-		}
-		paths := make([]string, 0, len(*filesystemPaths)+len(*filesystemDirectories))
-		paths = append(paths, *filesystemPaths...)
-		paths = append(paths, *filesystemDirectories...)
 		cfg := sources.FilesystemConfig{
-			Paths:  paths,
+			Directories: *filesystemDirectories,
-			Filter: filter,
+			Filter:      filter,
 		}
 		if err = e.ScanFileSystem(ctx, cfg); err != nil {
-			logFatal(err, "Failed to scan filesystem")
+			logrus.WithError(err).Fatal("Failed to scan filesystem")
 		}
 	case s3Scan.FullCommand():
 		cfg := sources.S3Config{
@@ -365,7 +300,7 @@ func run(state overseer.State) {
 			CloudCred: *s3ScanCloudEnv,
 		}
 		if err := e.ScanS3(ctx, cfg); err != nil {
-			logFatal(err, "Failed to scan S3.")
+			logrus.WithError(err).Fatal("Failed to scan S3.")
 		}
 	case syslogScan.FullCommand():
 		cfg := sources.SyslogConfig{
@@ -377,11 +312,11 @@ func run(state overseer.State) {
 			Concurrency: *concurrency,
 		}
 		if err := e.ScanSyslog(ctx, cfg); err != nil {
-			logFatal(err, "Failed to scan syslog.")
+			logrus.WithError(err).Fatal("Failed to scan syslog.")
 		}
 	case circleCiScan.FullCommand():
 		if err := e.ScanCircleCI(ctx, *circleCiScanToken); err != nil {
-			logFatal(err, "Failed to scan CircleCI.")
+			logrus.WithError(err).Fatal("Failed to scan CircleCI.")
 		}
 	}
 	// asynchronously wait for scanning to finish and cleanup
@@ -400,30 +335,24 @@ func run(state overseer.State) {
 		}
 		foundResults = true
 
-		var err error
 		switch {
 		case *jsonLegacy:
-			err = output.PrintLegacyJSON(ctx, &r)
+			output.PrintLegacyJSON(ctx, &r)
 		case *jsonOut:
-			err = output.PrintJSON(&r)
+			output.PrintJSON(&r)
 		default:
-			err = output.PrintPlainOutput(&r)
+			output.PrintPlainOutput(&r)
-		}
-		if err != nil {
-			logFatal(err, "error printing results")
 		}
 	}
-	logger.V(2).Info("finished scanning",
+	logrus.Debugf("scanned %d chunks", e.ChunksScanned())
-		"chunks", e.ChunksScanned(),
+	logrus.Debugf("scanned %d bytes", e.BytesScanned())
-		"bytes", e.BytesScanned(),
-	)
 
 	if *printAvgDetectorTime {
 		printAverageDetectorTime(e)
 	}
 
 	if foundResults && *fail {
-		logger.V(2).Info("exiting with code 183 because results were found")
+		logrus.Debug("exiting with code 183 because results were found")
 		os.Exit(183)
 	}
 }
@@ -439,24 +368,3 @@ func printAverageDetectorTime(e *engine.Engine) {
 		fmt.Fprintf(os.Stderr, "%s: %s\n", detectorName, avgDuration)
 	}
 }
-
-// logFatalFunc returns a log.Fatal style function. Calling the returned
-// function will terminate the program without cleanup.
-func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
-	return func(err error, message string, keyAndVals ...any) {
-		logger.Error(err, message, keyAndVals...)
-		if err != nil {
-			os.Exit(1)
-			return
-		}
-		os.Exit(0)
-	}
-}
-
-func detectorTypeToMap(detectors []config.DetectorID) map[detectorspb.DetectorType]config.DetectorID {
-	output := make(map[detectorspb.DetectorType]config.DetectorID, len(detectors))
-	for _, d := range detectors {
-		output[d.ID] = d
-	}
-	return output
-}

pkg/cache/simple.go

@@ -0,0 +1,50 @@
+package cache
+
+import (
+	"sync"
+)
+
+// Cache in-memory cache.
+type Cache struct {
+	mu sync.RWMutex
+	m  map[string]string
+}
+
+// Simple constructs a new Simple cache.
+func Simple() *Cache {
+	return &Cache{
+		m: make(map[string]string),
+	}
+}
+
+// Set a value in the cache.
+func (c *Cache) Set(key, value string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.m[key] = value
+	return nil
+}
+
+// Get a value and a boolean indicating if the value was found.
+func (c *Cache) Get(key string) (string, bool) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	value, ok := c.m[key]
+	return value, ok
+}
+
+// Delete a value from the cache.
+func (c *Cache) Delete(key string) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.m, key)
+	return nil
+}
+
+// Clear the cache.
+func (c *Cache) Clear() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.m = make(map[string]string)
+	return nil
+}

pkg/cache/simple_test.go

@@ -0,0 +1,57 @@
+package cache
+
+import "testing"
+
+func TestCache(t *testing.T) {
+	tests := []struct {
+		name string
+		key  string
+		val  string
+		want string
+		ok   bool
+	}{
+		{"set", "key", "value", "value", true},
+		{"get", "key", "", "value", true},
+		{"get-not-found", "not-found", "", "", false},
+		{"delete", "key", "", "", false},
+		{"clear", "key", "", "", false},
+	}
+
+	c := Simple()
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			switch test.name {
+			case "set":
+				if err := c.Set(test.key, test.val); err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+			case "get":
+				got, ok := c.Get(test.key)
+				if got != test.want || ok != test.ok {
+					t.Errorf("got (%q, %t), want (%q, %t)", got, ok, test.want, test.ok)
+				}
+			case "get-not-found":
+				got, ok := c.Get(test.key)
+				if got != test.want || ok != test.ok {
+					t.Errorf("got (%q, %t), want (%q, %t)", got, ok, test.want, test.ok)
+				}
+			case "delete":
+				if err := c.Delete(test.key); err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				got, ok := c.Get(test.key)
+				if got != test.want || ok != test.ok {
+					t.Errorf("got (%q, %t), want (%q, %t)", got, ok, test.want, test.ok)
+				}
+			case "clear":
+				if err := c.Clear(); err != nil {
+					t.Fatalf("unexpected error: %v", err)
+				}
+				got, ok := c.Get(test.key)
+				if got != test.want || ok != test.ok {
+					t.Errorf("got (%q, %t), want (%q, %t)", got, ok, test.want, test.ok)
+				}
+			}
+		})
+	}
+}

pkg/common/filter.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"regexp"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	log "github.com/sirupsen/logrus"
 )
 
 type Filter struct {
@@ -20,8 +20,7 @@ type FilterRuleSet []regexp.Regexp
 func FilterEmpty() *Filter {
 	filter, err := FilterFromFiles("", "")
 	if err != nil {
-		context.Background().Logger().Error(err, "could not create empty filter")
+		log.WithError(err).Fatalf("could not create empty filter")
-		os.Exit(1)
 	}
 	return filter
 }
@@ -61,16 +60,13 @@ func FilterRulesFromFile(source string) (*FilterRuleSet, error) {
 	emptyLinePattern := regexp.MustCompile(`^\s*$`)
 
 	file, err := os.Open(source)
-	logger := context.Background().Logger().WithValues("file", source)
 	if err != nil {
-		logger.Error(err, "unable to open filter file", "file", source)
+		log.WithError(err).Fatalf("unable to open filter file: %s", source)
-		os.Exit(1)
 	}
 	defer func(file *os.File) {
 		err := file.Close()
 		if err != nil {
-			logger.Error(err, "unable to close filter file")
+			log.WithError(err).Fatalf("unable to close filter file: %s", source)
-			os.Exit(1)
 		}
 	}(file)
 

pkg/common/patterns.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	log "github.com/sirupsen/logrus"
 )
 
 const EmailPattern = `\b(?:[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])\b`
@@ -22,7 +24,7 @@ func BuildRegex(pattern string, specialChar string, length int) string {
 
 func BuildRegexJWT(firstRange, secondRange, thirdRange string) string {
 	if RangeValidation(firstRange) || RangeValidation(secondRange) || RangeValidation(thirdRange) {
-		panic("Min value should not be greater than or equal to max")
+		log.Error("Min value should not be greater than or equal to max")
 	}
 	return fmt.Sprintf(`\b(ey[%s]{%s}.ey[%s-\/_]{%s}.[%s-\/_]{%s})\b`, AlphaNumPattern, firstRange, AlphaNumPattern, secondRange, AlphaNumPattern, thirdRange)
 }

pkg/common/secrets.go

@@ -7,9 +7,9 @@ import (
 	"time"
 
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
-	"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
 	"github.com/joho/godotenv"
 	"github.com/pkg/errors"
+	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
 )
 
 type Secret struct{ kv map[string]string }

pkg/config/detectors.go

@@ -1,183 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"sort"
-	"strconv"
-	"strings"
-
-	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-var (
-	specialGroups = map[string][]DetectorID{
-		"all": allDetectors(),
-	}
-
-	detectorTypeValue = make(map[string]dpb.DetectorType, len(dpb.DetectorType_value))
-	validDetectors    = make(map[dpb.DetectorType]struct{}, len(dpb.DetectorType_value))
-	maxDetectorType   dpb.DetectorType
-)
-
-// Setup package local global variables.
-func init() {
-	for k, v := range dpb.DetectorType_value {
-		dt := dpb.DetectorType(v)
-		detectorTypeValue[strings.ToLower(k)] = dt
-		validDetectors[dt] = struct{}{}
-		if dt > maxDetectorType {
-			maxDetectorType = dt
-		}
-	}
-}
-
-type DetectorID struct {
-	ID      dpb.DetectorType
-	Version int
-}
-
-// ParseDetectors parses user supplied string into a list of detectors types.
-// "all" will return the list of all available detectors. The input is comma
-// separated and may use the case-insensitive detector name defined in the
-// protobuf, or the protobuf enum number. A range may be used as well in the
-// form "start-end". Order is preserved and duplicates are ignored.
-func ParseDetectors(input string) ([]DetectorID, error) {
-	var output []DetectorID
-	seenDetector := map[DetectorID]struct{}{}
-	for _, item := range strings.Split(input, ",") {
-		item = strings.TrimSpace(item)
-		if item == "" {
-			continue
-		}
-		detectors, ok := specialGroups[strings.ToLower(item)]
-		if !ok {
-			var err error
-			detectors, err = asRange(item)
-			if err != nil {
-				return nil, err
-			}
-		}
-		for _, d := range detectors {
-			if _, ok := seenDetector[d]; ok {
-				continue
-			}
-			seenDetector[d] = struct{}{}
-			output = append(output, d)
-		}
-	}
-	return output, nil
-}
-
-func (id DetectorID) String() string {
-	name := dpb.DetectorType_name[int32(id.ID)]
-	if name == "" {
-		name = "<invalid ID>"
-	}
-	if id.Version == 0 {
-		return name
-	}
-	return fmt.Sprintf("%s.v%d", name, id.Version)
-}
-
-// allDetectors reutrns an ordered slice of all detector types.
-func allDetectors() []DetectorID {
-	all := make([]DetectorID, 0, len(dpb.DetectorType_name))
-	for id := range dpb.DetectorType_name {
-		all = append(all, DetectorID{ID: dpb.DetectorType(id)})
-	}
-	sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })
-	return all
-}
-
-// asRange converts a single input into a slice of detector types. If the input
-// is not in range format, a slice of length 1 is returned. Unbounded ranges
-// are allowed.
-func asRange(input string) ([]DetectorID, error) {
-	// Check if it's a single detector type.
-	dt, err := asDetectorID(input)
-	if err == nil {
-		return []DetectorID{dt}, nil
-	}
-
-	// Check if it's a range; if not return the error from above.
-	start, end, found := strings.Cut(input, "-")
-	if !found {
-		return nil, err
-	}
-	start, end = strings.TrimSpace(start), strings.TrimSpace(end)
-
-	// Convert the range start and end to a DetectorType.
-	dtStart, err := asDetectorID(start)
-	if err != nil {
-		return nil, err
-	}
-	dtEnd, err := asDetectorID(end)
-	// If end is empty it's an unbounded range.
-	if err != nil && end != "" {
-		return nil, err
-	}
-	if end == "" {
-		dtEnd.ID = maxDetectorType
-	}
-
-	// Ensure these ranges don't have versions.
-	if dtEnd.Version != 0 || dtStart.Version != 0 {
-		return nil, fmt.Errorf("versions within ranges are not supported: %s", input)
-	}
-
-	step := dpb.DetectorType(1)
-	if dtStart.ID > dtEnd.ID {
-		step = -1
-	}
-	var output []DetectorID
-	for dt := dtStart.ID; dt != dtEnd.ID; dt += step {
-		if _, ok := validDetectors[dt]; !ok {
-			continue
-		}
-		output = append(output, DetectorID{ID: dt})
-	}
-	return append(output, dtEnd), nil
-}
-
-// asDetectorID converts the case-insensitive input into a DetectorID.
-// Name or ID may be used.
-func asDetectorID(input string) (DetectorID, error) {
-	if input == "" {
-		return DetectorID{}, fmt.Errorf("empty detector")
-	}
-	var detectorID DetectorID
-	// Separate the version if there is one.
-	if detector, version, hasVersion := strings.Cut(input, "."); hasVersion {
-		parsedVersion, err := parseVersion(version)
-		if err != nil {
-			return DetectorID{}, fmt.Errorf("invalid version for input: %q error: %w", input, err)
-		}
-		detectorID.Version = parsedVersion
-		// Because there was a version, the detector type input is the part before the '.'
-		input = detector
-	}
-
-	// Check if it's a named detector.
-	if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
-		detectorID.ID = dt
-		return detectorID, nil
-	}
-	// Check if it's a detector ID.
-	if i, err := strconv.ParseInt(input, 10, 32); err == nil {
-		dt := dpb.DetectorType(i)
-		if _, ok := validDetectors[dt]; !ok {
-			return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)
-		}
-		detectorID.ID = dt
-		return detectorID, nil
-	}
-	return DetectorID{}, fmt.Errorf("unrecognized detector type: %s", input)
-}
-
-func parseVersion(v string) (int, error) {
-	if !strings.HasPrefix(strings.ToLower(v), "v") {
-		return 0, fmt.Errorf("version must start with 'v'")
-	}
-	version := strings.TrimLeft(v, "vV")
-	return strconv.Atoi(version)
-}

pkg/config/detectors_test.go

@@ -1,47 +0,0 @@
-package config
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestDetectorParsing(t *testing.T) {
-	tests := map[string]struct {
-		input    string
-		expected []DetectorID
-	}{
-		"all":                       {"AlL", allDetectors()},
-		"trailing range":            {"0-", allDetectors()},
-		"all after 1":               {"1-", allDetectors()[1:]},
-		"named and valid range":     {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
-		"duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},
-		"named range":               {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
-		"range preserved":           {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},
-		"reverse range":             {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},
-		"range preserved with all":  {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
-		"empty list item":           {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},
-		"invalid end range":         {"0-1337", nil},
-		"invalid name":              {"foo", nil},
-		"negative":                  {"-1", nil},
-		"github.v1":                 {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},
-		"gitlab.v100":               {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},
-		"range with versions":       {"github.v2 - gitlab.v1", nil},
-		"invalid version no v":      {"gitlab.2", nil},
-		"invalid version no number": {"gitlab.github", nil},
-		"capital V is fine":         {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},
-		"id number with version":    {"8.v2", []DetectorID{{ID: 8, Version: 2}}},
-	}
-
-	for name, tt := range tests {
-		t.Run(name, func(t *testing.T) {
-			got, gotErr := ParseDetectors(tt.input)
-			if tt.expected == nil {
-				assert.Error(t, gotErr)
-				return
-			}
-			assert.Equal(t, tt.expected, got)
-		})
-	}
-}

pkg/custom_detectors/custom_detectors.go

@@ -12,7 +12,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"golang.org/x/sync/errgroup"
 )
 
 // The maximum number of matches from one chunk. This const is used when
@@ -83,99 +82,68 @@ func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []b
 	// ]
 	matches := permutateMatches(regexMatches)
 
-	g := new(errgroup.Group)
-
 	// Create result object and test for verification.
-	resultsCh := make(chan detectors.Result, maxTotalMatches)
 	for _, match := range matches {
-		match := match
+		if common.IsDone(ctx) {
-		g.Go(func() error {
+			// TODO: Log we're possibly leaving out results.
-			return c.createResults(ctx, match, verify, resultsCh)
+			return results, nil
+		}
+		var raw string
+		for _, values := range match {
+			// values[0] contains the entire regex match.
+			raw += values[0]
+		}
+		result := detectors.Result{
+			DetectorType: detectorspb.DetectorType_CustomRegex,
+			Raw:          []byte(raw),
+		}
+
+		if !verify {
+			results = append(results, result)
+			continue
+		}
+		// Verify via webhook.
+		jsonBody, err := json.Marshal(map[string]map[string][]string{
+			c.GetName(): match,
 		})
-	}
+		if err != nil {
-
+			continue
-	if err := g.Wait(); err != nil {
+		}
-		return nil, err
+		// Try each config until we successfully verify.
-	}
+		for _, verifyConfig := range c.GetVerify() {
-	close(resultsCh)
+			if common.IsDone(ctx) {
-
+				// TODO: Log we're possibly leaving out results.
-	for result := range resultsCh {
+				return results, nil
+			}
+			req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
+			if err != nil {
+				continue
+			}
+			for _, header := range verifyConfig.GetHeaders() {
+				key, value, found := strings.Cut(header, ":")
+				if !found {
+					// Should be unreachable due to validation.
+					continue
+				}
+				req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
+			}
+			res, err := httpClient.Do(req)
+			if err != nil {
+				continue
+			}
+			// TODO: Read response body.
+			res.Body.Close()
+			if res.StatusCode == http.StatusOK {
+				result.Verified = true
+				break
+			}
+		}
 		results = append(results, result)
 	}
 
 	return results, nil
 }
 
-func (c *customRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
-	if common.IsDone(ctx) {
-		// TODO: Log we're possibly leaving out results.
-		return ctx.Err()
-	}
-	var raw string
-	for _, values := range match {
-		// values[0] contains the entire regex match.
-		raw += values[0]
-	}
-	result := detectors.Result{
-		DetectorType: detectorspb.DetectorType_CustomRegex,
-		Raw:          []byte(raw),
-	}
-
-	if !verify {
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case results <- result:
-			return nil
-		}
-	}
-	// Verify via webhook.
-	jsonBody, err := json.Marshal(map[string]map[string][]string{
-		c.GetName(): match,
-	})
-	if err != nil {
-		// This should never happen, but if it does, return nil to not
-		// disrupt other verification.
-		return nil
-	}
-	// Try each config until we successfully verify.
-	for _, verifyConfig := range c.GetVerify() {
-		if common.IsDone(ctx) {
-			// TODO: Log we're possibly leaving out results.
-			return ctx.Err()
-		}
-		req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
-		if err != nil {
-			continue
-		}
-		for _, header := range verifyConfig.GetHeaders() {
-			key, value, found := strings.Cut(header, ":")
-			if !found {
-				// Should be unreachable due to validation.
-				continue
-			}
-			req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
-		}
-		res, err := httpClient.Do(req)
-		if err != nil {
-			continue
-		}
-		// TODO: Read response body.
-		res.Body.Close()
-		if res.StatusCode == http.StatusOK {
-			result.Verified = true
-			break
-		}
-	}
-
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case results <- result:
-		return nil
-	}
-}
-
 func (c *customRegexWebhook) Keywords() []string {
 	return c.GetKeywords()
 }

pkg/detectors/abbysale/abbysale.go

@@ -2,6 +2,7 @@ package abbysale
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", resMatch)
+			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/bitfinex/bitfinex.go

@@ -86,7 +86,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				s1.Verified = isValid
 				// If there is a valid one, we need to stop iterating now and return the valid result
-				if isValid {
+				if isValid == true {
 					break
 				}
 			}

pkg/detectors/brandfetch/brandfetch.go

@@ -2,6 +2,7 @@ package brandfetch
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -55,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("x-api-key", resMatch)
+			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/browserstack/browserstack.go

@@ -21,7 +21,7 @@ var (
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "key", "automate", "local"}) + `\b([0-9a-zA-Z]{20})\b`)
-	userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "user", "automate", "local"}) + `\b([a-zA-Z\d]{3,18}[._-]?[a-zA-Z\d]{6})\b`)
+	userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "user", "automate", "local"}) + `\b(^[a-zA-Z\d]+([._-]?[a-zA-Z\d]+)*[a-zA-Z\d]+$)\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.

pkg/detectors/browserstack/browserstack_test.go

@@ -51,7 +51,6 @@ func TestBrowserStack_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_BrowserStack,
 					Verified:     true,
-					RawV2:        []byte(fmt.Sprintf("%s%s", secret, secretUser)),
 				},
 			},
 			wantErr: false,
@@ -68,7 +67,6 @@ func TestBrowserStack_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_BrowserStack,
 					Verified:     false,
-					RawV2:        []byte(fmt.Sprintf("%s%s", inactiveSecret, secretUser)),
 				},
 			},
 			wantErr: false,

pkg/detectors/clockify/clockify.go

@@ -2,6 +2,7 @@ package clockify
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("content-type", "application/json")
-			req.Header.Add("X-Api-Key", resMatch)
+			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/cloudflarecakey/cloudflarecakey.go

@@ -66,6 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				}
+			} else {
 			}
 		}
 

pkg/detectors/detectors.go

@@ -25,12 +25,6 @@ type Detector interface {
 	Type() detectorspb.DetectorType
 }
 
-// Versioner is an optional interface that a detector can implement to
-// differentiate instances of the same detector type.
-type Versioner interface {
-	Version() int
-}
-
 type Result struct {
 	// DetectorType is the type of Detector.
 	DetectorType detectorspb.DetectorType

pkg/detectors/everhour/everhour.go

@@ -2,6 +2,7 @@ package everhour
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Api-Key", resMatch)
+			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/fastlypersonaltoken/fastlypersonaltoken.go

@@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Fastly-Key", resMatch)
+			req.Header.Add("Fastly-Key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)

pkg/detectors/figmapersonalaccesstoken/figmapersonalaccesstoken.go

@@ -2,6 +2,7 @@ package figmapersonalaccesstoken
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -50,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Figma-Token", resMatch)
+			req.Header.Add("X-Figma-Token", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/fulcrum/fulcrum.go

@@ -2,6 +2,7 @@ package fulcrum
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("X-ApiToken", resMatch)
+			req.Header.Add("X-ApiToken", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/generic/generic.go

@@ -44,7 +44,10 @@ func New() Scanner {
 }
 
 type Scanner struct {
-	excludeMatchers []*regexp.Regexp
+	wordList             []string
+	badList              []string
+	programmingBookWords []string
+	excludeMatchers      []*regexp.Regexp
 }
 
 // Ensure the Scanner satisfies the interface at compile time.

pkg/detectors/github/github.go

@@ -15,11 +15,8 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interfaces at compile time.
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
-var _ detectors.Versioner = (*Scanner)(nil)
-
-func (s Scanner) Version() int { return 2 }
 
 var (
 	// Oauth token

pkg/detectors/github_old/github_old.go

@@ -14,11 +14,8 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interfaces at compile time.
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
-var _ detectors.Versioner = (*Scanner)(nil)
-
-func (s Scanner) Version() int { return 1 }
 
 var (
 	// Oauth token
@@ -41,7 +38,7 @@ type userRes struct {
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"github", "gh", "pat"}
+	return []string{"github","gh","pat"}
 }
 
 // FromData will find and optionally verify GitHub secrets in a given set of bytes.

pkg/detectors/gitlab/gitlab.go

@@ -14,11 +14,8 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interfaces at compile time.
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
-var _ detectors.Versioner = (*Scanner)(nil)
-
-func (s Scanner) Version() int { return 1 }
 
 var (
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)

pkg/detectors/gitlabv2/gitlab.go

@@ -13,11 +13,8 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interfaces at compile time.
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
-var _ detectors.Versioner = (*Scanner)(nil)
-
-func (*Scanner) Version() int { return 2 }
 
 var (
 	keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)

pkg/detectors/impala/impala.go

@@ -2,6 +2,7 @@ package impala
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", resMatch)
+			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/interseller/interseller.go

@@ -56,6 +56,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+					// if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+					// 	continue
+					// }
 				}
 			}
 		}

pkg/detectors/keenio/keenio.go

@@ -2,6 +2,7 @@ package keenio
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -60,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("Authorization", resMatch)
+				req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()

pkg/detectors/moonclerck/moonclerck.go

@@ -0,0 +1,78 @@
+package moonclerck
+
+import (
+	"context"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"moonclerck"}) + `\b([0-9a-z]{32})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"moonclerck"}
+}
+
+// FromData will find and optionally verify Moonclerck secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Moonclerk,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.moonclerk.com/forms", nil)
+			if err != nil {
+				continue
+			}
+			req.Header.Add("Accept", "application/vnd.moonclerk+json;version=1")
+			req.Header.Add("Authorization", "Token token="+resMatch)
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+				}
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Moonclerk
+}

pkg/detectors/moonclerck/moonclerck_test.go

@@ -0,0 +1,116 @@
+//go:build detectors
+// +build detectors
+
+package moonclerck
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestMoonclerck_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("MOONCLERCK")
+	inactiveSecret := testSecrets.MustGetField("MOONCLERCK_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a moonclerck secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Moonclerk,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a moonclerck secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Moonclerk,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Moonclerck.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Moonclerck.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				s.FromData(ctx, false, data)
+			}
+		})
+	}
+}

pkg/detectors/newrelicpersonalapikey/newrelicpersonalapikey.go

@@ -2,6 +2,7 @@ package newrelicpersonalapikey
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Api-Key", resMatch)
+			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/newscatcher/newscatcher.go

@@ -2,6 +2,7 @@ package newscatcher
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", resMatch)
+			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/nftport/nftport.go

@@ -2,6 +2,7 @@ package nftport
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", resMatch)
+			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/npmtoken/npmtoken.go

@@ -14,11 +14,8 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interfaces at compile time.
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
-var _ detectors.Versioner = (*Scanner)(nil)
-
-func (s Scanner) Version() int { return 1 }
 
 var (
 	client = common.SaneHttpClient()
@@ -37,7 +34,7 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+  	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue

pkg/detectors/npmtokenv2/npmtokenv2.go

@@ -13,11 +13,8 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interfaces at compile time.
+// Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
-var _ detectors.Versioner = (*Scanner)(nil)
-
-func (s Scanner) Version() int { return 2 }
 
 var (
 	client = common.SaneHttpClient()

pkg/detectors/omnisend/omnisend.go

@@ -2,6 +2,7 @@ package omnisend
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-API-KEY", resMatch)
+			req.Header.Add("X-API-KEY", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/openuv/openuv.go

@@ -2,6 +2,7 @@ package openuv
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-access-token", resMatch)
+			req.Header.Add("x-access-token", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/pepipost/pepipost.go

@@ -2,6 +2,7 @@ package pepipost
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("api_key", resMatch)
+			req.Header.Add("api_key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/postman/postman.go

@@ -2,6 +2,7 @@ package postman
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", resMatch)
+			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/postmark/postmark.go

@@ -2,6 +2,7 @@ package postmark
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Accept", "application/json")
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Postmark-Server-Token", resMatch)
+			req.Header.Add("X-Postmark-Server-Token", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/privatekey/privatekey.go

@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"time"
 
+	log "github.com/sirupsen/logrus"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -65,6 +66,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 				if data != nil {
 					secret.Verified = true
 				}
+			} else {
+				log.Warn(err)
 			}
 		}
 

pkg/detectors/prospectio/prospectio.go

@@ -2,6 +2,7 @@ package prospectio
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", resMatch)
+			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
 			req.Header.Add("Content-Type", "application/vnd.api+json; charset=utf-8")
 			res, err := client.Do(req)
 			if err == nil {

pkg/detectors/pushbulletapikey/pushbulletapikey.go

@@ -2,6 +2,7 @@ package pushbulletapikey
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Access-Token", resMatch)
+			req.Header.Add("Access-Token", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/qubole/qubole.go

@@ -2,6 +2,7 @@ package qubole
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Content-Type", "application/json")
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("X-AUTH-TOKEN", resMatch)
+			req.Header.Add("X-AUTH-TOKEN", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/rapidapi/rapidapi.go

@@ -2,6 +2,7 @@ package rapidapi
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -50,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-rapidapi-key", resMatch)
+			req.Header.Add("x-rapidapi-key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/rechargepayments/rechargepayments.go

@@ -16,6 +16,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
+	client    = common.SaneHttpClient()
 	verifyURL = "https://api.rechargeapps.com/token_information"
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.

pkg/detectors/scalewaykey/scalewaykey.go

@@ -2,6 +2,7 @@ package scalewaykey
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -50,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Auth-Token", resMatch)
+			req.Header.Add("X-Auth-Token", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/sendinbluev2/sendinbluev2.go

@@ -2,6 +2,7 @@ package sendinbluev2
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -50,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("api-key", resMatch)
+			req.Header.Add("api-key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/shortcut/shortcut.go

@@ -65,11 +65,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				if strings.Contains(body, "app_url") {
 					s1.Verified = true
+				} else {
+					// if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+					// 	continue
+					// }
 				}
+
 			}
+
 		}
+
 		results = append(results, s1)
 	}
+
 	return results, nil
 }
 

pkg/detectors/skrappio/skrappio.go

@@ -2,6 +2,7 @@ package skrappio
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Access-Key", resMatch)
+			req.Header.Add("X-Access-Key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/splunkobservabilitytoken/splunkobservabilitytoken.go

@@ -56,7 +56,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			req.Header.Set("X-Sf-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
-				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {

pkg/detectors/stormglass/stormglass.go

@@ -2,6 +2,7 @@ package stormglass
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", resMatch)
+			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/tatumio/tatumio.go

@@ -2,6 +2,7 @@ package tatumio
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/vnd.tatumio+json; version=3")
-			req.Header.Add("x-api-key", resMatch)
+			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/detectors/uri/uri.go

@@ -26,6 +26,10 @@ var (
 	client = common.SaneHttpClient()
 )
 
+type proxyRes struct {
+	Verified bool `json:"verified"`
+}
+
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {

pkg/detectors/websitepulse/websitepulse.go

@@ -61,7 +61,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 				res, err := client.Do(req)
 				if err == nil {
-					defer res.Body.Close()
 					bodyBytes, err := io.ReadAll(res.Body)
 					if err != nil {
 						continue

pkg/detectors/zonkafeedback/zonkafeedback.go

@@ -2,6 +2,7 @@ package zonkafeedback
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Z-API-TOKEN", resMatch)
+			req.Header.Add("Z-API-TOKEN", fmt.Sprintf("%s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()

pkg/engine/circleci.go

@@ -10,6 +10,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
 )
 
@@ -29,11 +30,19 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 	}
 
 	circleSource := circleci.Source{}
+	cfg := sources.NewSourceConfig(
+		"trufflehog - Circle CI",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI),
+		&conn,
+		sources.WithConcurrency(runtime.NumCPU()),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", circleSource.Type().String(),
 		"source_name", "Circle CI",
 	)
-	err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU())
+	err = circleSource.Init(ctx, cfg)
 	if err != nil {
 		return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
 	}

pkg/engine/circleci_test.go

@@ -0,0 +1,37 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+)
+
+func TestScanCircleCI(t *testing.T) {
+	tests := []struct {
+		name    string
+		token   string
+		wantErr bool
+	}{
+		{
+			name:  "valid token",
+			token: "valid_token",
+		},
+		{
+			name:  "no token",
+			token: "",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			engine := &Engine{}
+			engine.sourcesWg = sync.WaitGroup{}
+
+			err := engine.ScanCircleCI(ctx, test.token)
+			if (err != nil) != test.wantErr {
+				t.Errorf("ScanCircleCI() error = %v, wantErr %v", err, test.wantErr)
+			}
+		})
+	}
+}

pkg/engine/defaults.go

@@ -405,6 +405,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/monday"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mongodb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/monkeylearn"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moonclerck"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moonclerk"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/moosend"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/mrticktock"
@@ -746,7 +747,9 @@ func DefaultDetectors() []detectors.Detector {
 		&mailchimp.Scanner{},
 		&okta.Scanner{},
 		&onelogin.Scanner{},
+		&heroku.Scanner{},
 		&dropbox.Scanner{},
+		&linearapi.Scanner{},
 		&stripe.Scanner{},
 		&square.Scanner{},
 		&squareapp.Scanner{},
@@ -1270,6 +1273,7 @@ func DefaultDetectors() []detectors.Detector {
 		teamworkdesk.Scanner{},
 		tallyfy.Scanner{},
 		apimatic.Scanner{},
+		moonclerck.Scanner{},
 		boostnote.Scanner{},
 		freshbooks.Scanner{},
 		cashboard.Scanner{},

pkg/engine/engine.go

@@ -9,7 +9,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -37,10 +36,6 @@ type Engine struct {
 	// If there are multiple unverified results for the same chunk for the same detector,
 	// only the first one will be kept.
 	filterUnverified bool
-
-	// prefilter is a ahocorasick struct used for doing efficient string
-	// matching given a set of words (keywords from the rules in the config)
-	prefilter ahocorasick.AhoCorasick
 }
 
 type EngineOption func(*Engine)
@@ -78,31 +73,6 @@ func WithFilterUnverified(filter bool) EngineOption {
 	}
 }
 
-// WithFilterDetectors applies a filter to the configured list of detectors. If
-// the filterFunc returns true, the detector will be included for scanning.
-// This option applies to the existing list of detectors configured, so the
-// order this option appears matters. All filtering happens before scanning.
-func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
-	return func(e *Engine) {
-		// If no detectors are configured, do nothing.
-		if e.detectors == nil {
-			return
-		}
-		e.detectors[true] = filterDetectors(filterFunc, e.detectors[true])
-		e.detectors[false] = filterDetectors(filterFunc, e.detectors[false])
-	}
-}
-
-func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
-	var output []detectors.Detector
-	for _, detector := range input {
-		if filterFunc(detector) {
-			output = append(output, detector)
-		}
-	}
-	return output
-}
-
 func Start(ctx context.Context, options ...EngineOption) *Engine {
 	e := &Engine{
 		chunks:          make(chan *sources.Chunk),
@@ -115,6 +85,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 
 	// Set defaults.
+
 	if e.concurrency == 0 {
 		numCPU := runtime.NumCPU()
 		ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
@@ -132,23 +103,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		e.detectors[false] = []detectors.Detector{}
 	}
 
-	// build ahocorasick prefilter for efficient string matching
-	// on keywords
-	keywords := []string{}
-	for _, d := range e.detectors[false] {
-		keywords = append(keywords, d.Keywords()...)
-	}
-	for _, d := range e.detectors[true] {
-		keywords = append(keywords, d.Keywords()...)
-	}
-	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
-		AsciiCaseInsensitive: true,
-		MatchOnlyWholeWords:  false,
-		MatchKind:            ahocorasick.LeftMostLongestMatch,
-		DFA:                  true,
-	})
-	e.prefilter = builder.Build(keywords)
-
 	ctx.Logger().V(2).Info("loaded decoders", "count", len(e.decoders))
 	ctx.Logger().V(2).Info("loaded detectors",
 		"total", len(e.detectors[true])+len(e.detectors[false]),
@@ -229,7 +183,6 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 func (e *Engine) detectorWorker(ctx context.Context) {
 	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
-			matchedKeywords := make(map[string]struct{})
 			atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
 			for _, decoder := range e.decoders {
 				var decoderType detectorspb.DecoderType
@@ -246,29 +199,21 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 				if decoded == nil {
 					continue
 				}
-
 				dataLower := strings.ToLower(string(decoded.Data))
-				matches := e.prefilter.FindAll(dataLower)
-
-				for _, m := range matches {
-					matchedKeywords[dataLower[m.Start():m.End()]] = struct{}{}
-				}
-
 				for verify, detectorsSet := range e.detectors {
 					for _, detector := range detectorsSet {
-						chunkContainsKeyword := false
+						start := time.Now()
+						foundKeyword := false
 						for _, kw := range detector.Keywords() {
-							if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
+							if strings.Contains(dataLower, strings.ToLower(kw)) {
-								chunkContainsKeyword = true
+								foundKeyword = true
+								break
 							}
 						}
-
+						if !foundKeyword {
-						if !chunkContainsKeyword {
 							continue
 						}
 
-						start := time.Now()
-
 						results, err := func() ([]detectors.Result, error) {
 							ctx, cancel := context.WithTimeout(ctx, time.Second*10)
 							defer cancel()

pkg/engine/filesystem.go

@@ -17,7 +17,7 @@ import (
 // ScanFileSystem scans a given file system.
 func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error {
 	connection := &sourcespb.Filesystem{
-		Paths: c.Paths,
+		Directories: c.Directories,
 	}
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
@@ -27,12 +27,19 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 	}
 
 	fileSystemSource := filesystem.Source{}
-
+	cfg := sources.NewSourceConfig(
+		"trufflehog - filesystem",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM),
+		&conn,
+		sources.WithConcurrency(runtime.NumCPU()),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", fileSystemSource.Type().String(),
 		"source_name", "filesystem",
 	)
-	err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU())
+	err = fileSystemSource.Init(ctx, cfg)
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}

pkg/engine/filesystem_test.go

@@ -0,0 +1,44 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestEngine_ScanFileSystem(t *testing.T) {
+	tests := []struct {
+		name    string
+		config  sources.FilesystemConfig
+		wantErr bool
+	}{
+		{
+			name: "valid config",
+			config: sources.FilesystemConfig{
+				Directories: []string{"/tmp"},
+			},
+		},
+		{
+			name: "empty directories",
+			config: sources.FilesystemConfig{
+				Directories: []string{},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			e := &Engine{
+				sourcesWg: sync.WaitGroup{},
+			}
+
+			ctx := context.Background()
+			err := e.ScanFileSystem(ctx, test.config)
+			if (err != nil) != test.wantErr {
+				t.Errorf("ScanFileSystem(%v) error = %v, wantErr %v", test.config, err, test.wantErr)
+			}
+		})
+	}
+}

pkg/engine/github.go

@@ -1,7 +1,6 @@
 package engine
 
 import (
-	gogit "github.com/go-git/go-git/v5"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
@@ -9,7 +8,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
 )
 
@@ -40,24 +38,24 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 		return err
 	}
 
+	cfg := sources.NewSourceConfig(
+		"trufflehog - github",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_GITHUB),
+		&conn,
+		sources.WithConcurrency(c.Concurrency),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", source.Type().String(),
 		"source_name", "github",
 	)
-	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
+	err = source.Init(ctx, cfg)
 	if err != nil {
 		ctx.Logger().Error(err, "failed to initialize github source")
 		return err
 	}
 
-	logOptions := &gogit.LogOptions{}
-	opts := []git.ScanOption{
-		git.ScanOptionFilter(c.Filter),
-		git.ScanOptionLogOptions(logOptions),
-	}
-	scanOptions := git.NewScanOptions(opts...)
-	source.WithScanOptions(scanOptions)
-
 	e.sourcesWg.Add(1)
 	go func() {
 		defer common.RecoverWithExit(ctx)

pkg/engine/github_test.go

@@ -0,0 +1,63 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestEngine_ScanGitHub(t *testing.T) {
+	tests := []struct {
+		name    string
+		config  sources.GithubConfig
+		wantErr bool
+	}{
+		{
+			name: "valid config",
+			config: sources.GithubConfig{
+				Endpoint:     "https://api.github.com",
+				Token:        "valid_token",
+				Orgs:         []string{"org1", "org2"},
+				Repos:        []string{"repo1", "repo2"},
+				IncludeForks: true,
+			},
+		},
+		{
+			name: "empty endpoint",
+			config: sources.GithubConfig{
+				Endpoint: "",
+				Token:    "valid_token",
+				Orgs:     []string{"org1", "org2"},
+			},
+		},
+		{
+			name: "empty token",
+			config: sources.GithubConfig{
+				Endpoint: "https://api.github.com",
+				Orgs:     []string{"org1", "org2"},
+			},
+		},
+		{
+			name: "empty orgs",
+			config: sources.GithubConfig{
+				Endpoint: "https://api.github.com",
+				Token:    "valid_token",
+				Orgs:     []string{},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			e := &Engine{
+				sourcesWg: sync.WaitGroup{},
+			}
+			ctx := context.Background()
+			err := e.ScanGitHub(ctx, test.config)
+			if (err != nil) != test.wantErr {
+				t.Errorf("ScanGitHub(%v) error = %v, wantErr %v", test.config, err, test.wantErr)
+			}
+		})
+	}
+}

pkg/engine/gitlab.go

@@ -53,11 +53,19 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
 	}
 
 	gitlabSource := gitlab.Source{}
+	cfg := sources.NewSourceConfig(
+		"trufflehog - gitlab",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB),
+		&conn,
+		sources.WithConcurrency(runtime.NumCPU()),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", gitlabSource.Type().String(),
 		"source_name", "gitlab",
 	)
-	err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU())
+	err = gitlabSource.Init(ctx, cfg)
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init GitLab source", 0)
 	}

pkg/engine/gitlab_test.go

@@ -0,0 +1,49 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestScanGitLab(t *testing.T) {
+	tests := []struct {
+		name    string
+		c       sources.GitlabConfig
+		wantErr bool
+	}{
+		{
+			name: "Successful scan with valid token and endpoint",
+			c: sources.GitlabConfig{
+				Token:    "valid-token",
+				Endpoint: "https://gitlab.com",
+				Repos:    []string{"repo1", "repo2"},
+			},
+		},
+		{
+			name: "Failed scan with empty token",
+			c: sources.GitlabConfig{
+				Token:    "",
+				Endpoint: "https://gitlab.com",
+				Repos:    []string{"repo1", "repo2"},
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			e := &Engine{
+				sourcesWg: sync.WaitGroup{},
+			}
+
+			err := e.ScanGitLab(ctx, test.c)
+			if (err != nil) != test.wantErr {
+				t.Errorf("ScanGitLab(%v) error = %v, wantErr %v", test.c, err, test.wantErr)
+			}
+		})
+	}
+}

pkg/engine/s3.go

@@ -46,11 +46,19 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
 	}
 
 	s3Source := s3.Source{}
+	cfg := sources.NewSourceConfig(
+		"trufflehog - s3",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_S3),
+		&conn,
+		sources.WithConcurrency(runtime.NumCPU()),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", s3Source.Type().String(),
 		"source_name", "s3",
 	)
-	err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
+	err = s3Source.Init(ctx, cfg)
 	if err != nil {
 		return errors.WrapPrefix(err, "failed to init S3 source", 0)
 	}

pkg/engine/s3_test.go

@@ -0,0 +1,66 @@
+package engine
+
+import (
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestScanS3(t *testing.T) {
+	tests := []struct {
+		name      string
+		c         sources.S3Config
+		expectErr bool
+	}{
+		{
+			name: "Test with cloud credentials",
+			c: sources.S3Config{
+				CloudCred: true,
+				Buckets:   []string{"bucket1", "bucket2"},
+			},
+			expectErr: false,
+		},
+		{
+			name: "Test with basic authentication",
+			c: sources.S3Config{
+				CloudCred: false,
+				Key:       "accessKey",
+				Secret:    "secretKey",
+				Buckets:   []string{"bucket1", "bucket2"},
+			},
+			expectErr: false,
+		},
+		{
+			name: "Test with both cloud credentials and basic authentication",
+			c: sources.S3Config{
+				CloudCred: true,
+				Key:       "accessKey",
+				Secret:    "secretKey",
+				Buckets:   []string{"bucket1", "bucket2"},
+			},
+			expectErr: true,
+		},
+		{
+			name: "Test with unauthenticated",
+			c: sources.S3Config{
+				CloudCred: false,
+				Buckets:   []string{"bucket1", "bucket2"},
+			},
+			expectErr: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			e := Engine{}
+			err := e.ScanS3(ctx, test.c)
+			if test.expectErr && err == nil {
+				t.Errorf("expected an error but got nil")
+			}
+			if !test.expectErr && err != nil {
+				t.Errorf("expected no error but got %v", err)
+			}
+		})
+	}
+}

pkg/engine/syslog.go

@@ -41,12 +41,21 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
 	if err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
+
 	source := syslog.Source{}
+	cfg := sources.NewSourceConfig(
+		"trufflehog - syslog",
+		0,
+		int64(sourcespb.SourceType_SOURCE_TYPE_SYSLOG),
+		&conn,
+		sources.WithConcurrency(c.Concurrency),
+		sources.WithVerify(true),
+	)
 	ctx = context.WithValues(ctx,
 		"source_type", source.Type().String(),
 		"source_name", "syslog",
 	)
-	err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
+	err = source.Init(ctx, cfg)
 	source.InjectConnection(connection)
 	if err != nil {
 		ctx.Logger().Error(err, "failed to initialize syslog source")

pkg/engine/syslog_test.go

@@ -0,0 +1,61 @@
+package engine
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestScanSyslog(t *testing.T) {
+	tests := []struct {
+		name    string
+		c       sources.SyslogConfig
+		wantErr bool
+	}{
+		{
+			name: "Scan syslog successfully",
+			c: sources.SyslogConfig{
+				Protocol:    "tcp",
+				Address:     "127.0.0.1:1514",
+				Format:      "rfc3164",
+				Concurrency: 1,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Scan syslog with invalid cert",
+			c: sources.SyslogConfig{
+				CertPath:    "bad",
+				KeyPath:     "valid",
+				Address:     "127.0.0.1:1514",
+				Format:      "rfc3164",
+				Concurrency: 1,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Scan syslog with invalid keypath",
+			c: sources.SyslogConfig{
+				Protocol:    "tcp",
+				KeyPath:     "invalid",
+				CertPath:    "valid",
+				Format:      "rfc3164",
+				Concurrency: 1,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			e := &Engine{
+				sourcesWg: sync.WaitGroup{},
+			}
+			ctx := context.Background()
+			if err := e.ScanSyslog(ctx, tt.c); (err != nil) != tt.wantErr {
+				t.Errorf("ScanSyslog() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

pkg/gitparse/gitparse.go

@@ -11,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
@@ -117,7 +119,10 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
 
 // RepoPath parses the output of the `git log` command for the `source` path.
 func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
-	args := []string{"-C", source, "log", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
+	args := []string{"-C", source, "log", "-p", "-U5", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
+	if abbreviatedLog {
+		args = append(args, "--diff-filter=AM")
+	}
 	if head != "" {
 		args = append(args, head)
 	} else {
@@ -131,13 +136,12 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, source, cmd)
+	return c.executeCommand(ctx, cmd)
 }
 
 // Unstaged parses the output of the `git diff` command for the `source` path.
 func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, error) {
-	// Provide the --cached flag to diff to get the diff of the staged changes.
+	args := []string{"-C", source, "diff", "-p", "-U5", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
-	args := []string{"-C", source, "diff", "-p", "-U5", "--cached", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
 
 	cmd := exec.Command("git", args...)
 
@@ -146,11 +150,11 @@ func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, erro
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, source, cmd)
+	return c.executeCommand(ctx, cmd)
 }
 
 // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
-func (c *Parser) executeCommand(ctx context.Context, source string, cmd *exec.Cmd) (chan Commit, error) {
+func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
 	commitChan := make(chan Commit, 64)
 
 	stdOut, err := cmd.StdoutPipe()
@@ -170,26 +174,25 @@ func (c *Parser) executeCommand(ctx context.Context, source string, cmd *exec.Cm
 	go func() {
 		scanner := bufio.NewScanner(stdErr)
 		for scanner.Scan() {
-			ctx.Logger().V(2).Info(scanner.Text())
+			log.Debug(scanner.Text())
 		}
 	}()
 
 	go func() {
-		c.fromReader(ctx, source, stdOut, commitChan)
+		c.fromReader(ctx, stdOut, commitChan)
 		if err := cmd.Wait(); err != nil {
-			ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
+			log.WithError(err).Debugf("Error waiting for git command to complete.")
 		}
 	}()
 
 	return commitChan, nil
 }
 
-func (c *Parser) fromReader(ctx context.Context, source string, stdOut io.Reader, commitChan chan Commit) {
+func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
 	outReader := bufio.NewReader(stdOut)
 	var currentCommit *Commit
 	var currentDiff *Diff
 
-	lastCommit := ""
 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
 	for {
@@ -200,143 +203,6 @@ func (c *Parser) fromReader(ctx context.Context, source string, stdOut io.Reader
 		if err != nil && len(line) == 0 {
 			break
 		}
-		// for i := 0; i < len(line); i++ {
-		// 	b := make([]byte, 1)
-		// 	n, err := stdOut.Read(b)
-		// 	if errors.Is(err, io.EOF) || n == 0 {
-		// 		return
-		// 	}
-		// 	line[i] = b[0]
-		// 	if b[0] == byte('\n') {
-		// 		line = line[:0]
-		// 		break
-		// 	}
-		// }
-
-		switch {
-		case isCommitLine(line):
-			// If there is a currentDiff, add it to currentCommit.
-			if currentDiff != nil && currentDiff.Content.Len() > 0 {
-				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
-			}
-			// If there is a currentCommit, send it to the channel.
-			if currentCommit != nil {
-				args := []string{"-C", source, "diff", "--date=format:%a %b %d %H:%M:%S %Y %z", currentCommit.Hash}
-				if lastCommit != "" {
-					args = append(args, lastCommit)
-				}
-				lastCommit = currentCommit.Hash
-
-				cmd := exec.Command("git", args...)
-
-				absPath, err := filepath.Abs(source)
-				if err == nil {
-					cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
-				}
-
-				// stdnOut, err := cmd.StdoutPipe()
-				// if err != nil {
-				// 	return
-				// }
-				err = cmd.Start()
-				if err != nil {
-					return
-				}
-
-				_ = cmd.Wait()
-				//c.funkReader(ctx, stdnOut, commitChan, currentCommit)
-			}
-			// Create a new currentDiff and currentCommit
-			currentDiff = &Diff{}
-			currentCommit = &Commit{
-				Message: strings.Builder{},
-			}
-			// Check that the commit line contains a hash and set it.
-			if len(line) >= 47 {
-				currentCommit.Hash = string(line[7:47])
-			}
-		case isAuthorLine(line):
-			currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
-		case isDateLine(line):
-			date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
-			if err != nil {
-				ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
-			}
-			currentCommit.Date = date
-		case isModeLine(line):
-			// NoOp
-		case isIndexLine(line):
-			// NoOp
-		case isPlusFileLine(line):
-			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
-		case isMinusFileLine(line):
-			// NoOp
-		case isPlusDiffLine(line):
-			currentDiff.Content.Write(line[1:])
-		case isMinusDiffLine(line):
-			// NoOp. We only care about additions.
-		case isMessageLine(line):
-			currentCommit.Message.Write(line[4:])
-		case isContextDiffLine(line):
-			currentDiff.Content.Write([]byte("\n"))
-		case isBinaryLine(line):
-			currentDiff.IsBinary = true
-			currentDiff.PathB = pathFromBinaryLine(line)
-		case isLineNumberDiffLine(line):
-			if currentDiff != nil && currentDiff.Content.Len() > 0 {
-				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
-			}
-			newDiff := &Diff{
-				PathB: currentDiff.PathB,
-			}
-
-			currentDiff = newDiff
-
-			words := bytes.Split(line, []byte(" "))
-			if len(words) >= 3 {
-				startSlice := bytes.Split(words[2], []byte(","))
-				lineStart, err := strconv.Atoi(string(startSlice[0]))
-				if err == nil {
-					currentDiff.LineStart = lineStart
-				}
-			}
-		}
-		if currentDiff.Content.Len() > c.maxDiffSize {
-			ctx.Logger().V(2).Info(fmt.Sprintf(
-				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
-			))
-			break
-		}
-	}
-	cleanupParse(currentCommit, currentDiff, commitChan)
-}
-
-func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, currentCommit *Commit) {
-	outReader := bufio.NewReader(stdOut)
-	var currentDiff *Diff
-
-	defer common.RecoverWithExit(ctx)
-	for {
-		if common.IsDone(ctx) {
-			break
-		}
-		line, err := outReader.ReadBytes([]byte("\n")[0])
-		if err != nil && len(line) == 0 {
-			break
-		}
-		// for i := 0; i < len(line); i++ {
-		// 	b := make([]byte, 1)
-		// 	n, err := stdOut.Read(b)
-		// 	if errors.Is(err, io.EOF) || n == 0 {
-		// 		return
-		// 	}
-		// 	line[i] = b[0]
-		// 	if b[0] == byte('\n') {
-		// 		line = line[:0]
-		// 		break
-		// 	}
-		// }
-
 		switch {
 		case isCommitLine(line):
 			// If there is a currentDiff, add it to currentCommit.
@@ -361,7 +227,7 @@ func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		case isDateLine(line):
 			date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
 			if err != nil {
-				ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
+				log.WithError(err).Debug("Could not parse date from git stream.")
 			}
 			currentCommit.Date = date
 		case isDiffLine(line):
@@ -377,17 +243,14 @@ func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					totalSize += diff.Content.Len()
 				}
 				if totalSize > c.maxCommitSize {
-					oldCommit := currentCommit
 					commitChan <- *currentCommit
 					currentCommit = &Commit{
 						Hash:    currentCommit.Hash,
 						Author:  currentCommit.Author,
 						Date:    currentCommit.Date,
-						Message: strings.Builder{},
+						Message: currentCommit.Message,
 						Diffs:   []Diff{},
 					}
-					// Message needs to be recreated here otherwise writing to it again will result in a panic.
-					currentCommit.Message.WriteString(oldCommit.Message.String())
 				}
 			}
 			currentDiff = &Diff{}
@@ -396,7 +259,7 @@ func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		case isIndexLine(line):
 			// NoOp
 		case isPlusFileLine(line):
-			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
+			currentDiff.PathB = strings.TrimRight(string(line[6:]), "\n")
 		case isMinusFileLine(line):
 			// NoOp
 		case isPlusDiffLine(line):
@@ -430,9 +293,7 @@ func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan ch
 			}
 		}
 		if currentDiff.Content.Len() > c.maxDiffSize {
-			ctx.Logger().V(2).Info(fmt.Sprintf(
+			log.Debugf("Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize)
-				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
-			))
 			break
 		}
 	}
@@ -560,14 +421,13 @@ func isLineNumberDiffLine(line []byte) bool {
 	return false
 }
 
-// Get the b/ file path. Ignoring the edge case of files having `and /b` in the name for simplicity.
+// Get the b/ file path.
 func pathFromBinaryLine(line []byte) string {
-	logger := context.Background().Logger()
+	sbytes := bytes.Split(line, []byte(" and "))
-	sbytes := bytes.Split(line, []byte(" and b/"))
 	if len(sbytes) != 2 {
-		logger.V(2).Info("Expected binary line to be in 'Binary files a/fileA and b/fileB differ' format.", "got", line)
+		log.Debugf("Expected binary line to be in 'Binary files a/filaA and b/fileB differ' format. Got: %s", line)
 		return ""
 	}
 	bRaw := sbytes[1]
-	return string(bRaw[:len(bRaw)-7]) // drop the "b/" and " differ"
+	return string(bRaw[2 : len(bRaw)-7]) // drop the "b/" and " differ"
 }

pkg/handlers/archive.go

@@ -9,8 +9,8 @@ import (
 	"time"
 
 	"github.com/mholt/archiver/v4"
+	log "github.com/sirupsen/logrus"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )
 
 type ctxKey int
@@ -55,7 +55,6 @@ func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]
 	archiveChan := make(chan ([]byte), 512)
 	go func() {
 		ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
-		logger := logContext.AddLogger(ctx).Logger()
 		defer cancel()
 		defer close(archiveChan)
 		err := d.openArchive(ctx, 0, data, archiveChan)
@@ -63,7 +62,7 @@ func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]
 			if errors.Is(err, archiver.ErrNoMatch) {
 				return
 			}
-			logger.V(2).Info("Error unarchiving chunk.")
+			log.WithError(err).Debug("Error unarchiving chunk.")
 		}
 	}()
 	return archiveChan
@@ -130,8 +129,7 @@ func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
 // extractorHandler is applied to each file in an archiver.Extractor file.
 func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error {
 	return func(ctx context.Context, f archiver.File) error {
-		logger := logContext.AddLogger(ctx).Logger()
+		log.WithField("filename", f.Name()).Trace("Handling extracted file.")
-		logger.V(5).Info("Handling extracted file.", "filename", f.Name())
 		depth := 0
 		if ctxDepth, ok := ctx.Value(depthKey).(int); ok {
 			depth = ctxDepth
@@ -161,20 +159,19 @@ func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 	// rardecode. There is a bug somewhere with rar decoder format 29
 	// that can lead to a panic. An issue is open in rardecode repo
 	// https://github.com/nwaples/rardecode/issues/30.
-	logger := logContext.AddLogger(ctx).Logger()
 	defer func() {
 		if r := recover(); r != nil {
+			log.Errorf("Panic occurred when reading archive: %v", r)
 			// Return an error from ReadToMax.
 			if e, ok := r.(error); ok {
 				err = e
 			} else {
 				err = fmt.Errorf("Panic occurred: %v", r)
 			}
-			logger.Error(err, "Panic occurred when reading archive")
 		}
 	}()
 	fileContent := bytes.Buffer{}
-	logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-d.size)
+	log.Tracef("Remaining buffer capacity: %d", maxSize-d.size)
 	for i := 0; i <= maxSize/512; i++ {
 		if common.IsDone(ctx) {
 			return nil, ctx.Err()
@@ -192,7 +189,7 @@ func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 			return fileContent.Bytes(), nil
 		}
 		if d.size >= maxSize && bRead == 512 {
-			logger.V(2).Info("Max archive size reached.")
+			log.Debug("Max archive size reached.")
 			return fileContent.Bytes(), nil
 		}
 	}

pkg/output/json.go

@@ -4,13 +4,15 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 )
 
-func PrintJSON(r *detectors.ResultWithMetadata) error {
+func PrintJSON(r *detectors.ResultWithMetadata) {
 	v := &struct {
 		// SourceMetadata contains source-specific contextual information.
 		SourceMetadata *source_metadatapb.MetaData
@@ -50,8 +52,7 @@ func PrintJSON(r *detectors.ResultWithMetadata) error {
 	}
 	out, err := json.Marshal(v)
 	if err != nil {
-		return fmt.Errorf("could not marshal result: %w", err)
+		logrus.WithError(err).Fatal("could not marshal result")
 	}
 	fmt.Println(string(out))
-	return nil
 }

pkg/output/legacy_json.go

@@ -12,6 +12,7 @@ import (
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/sergi/go-diff/diffmatchpatch"
+	"github.com/sirupsen/logrus"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +20,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
-func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) error {
+func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) {
 	var repo string
 	switch r.SourceType {
 	case sourcespb.SourceType_SOURCE_TYPE_GIT:
@@ -29,31 +30,27 @@ func PrintLegacyJSON(ctx context.Context, r *detectors.ResultWithMetadata) error
 	case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
 		repo = r.SourceMetadata.GetGitlab().Repository
 	default:
-		return fmt.Errorf("unsupported source type for legacy json output: %s", r.SourceType)
+		logrus.Errorf("unsupported source type for legacy json output: %s", r.SourceType)
 	}
 
 	// cloning the repo again here is not great and only works with unauthed repos
 	repoPath, remote, err := git.PrepareRepo(ctx, repo)
 	if err != nil || repoPath == "" {
-		return fmt.Errorf("error preparing git repo for scanning: %w", err)
+		logrus.WithError(err).Fatal("error preparing git repo for scanning")
 	}
 	if remote {
 		defer os.RemoveAll(repoPath)
 	}
 
-	legacy, err := ConvertToLegacyJSON(r, repoPath)
+	legacy := ConvertToLegacyJSON(r, repoPath)
-	if err != nil {
-		return fmt.Errorf("could not convert to legacy JSON: %w", err)
-	}
 	out, err := json.Marshal(legacy)
 	if err != nil {
-		return fmt.Errorf("could not marshal result: %w", err)
+		logrus.WithError(err).Fatal("could not marshal result")
 	}
 	fmt.Println(string(out))
-	return nil
 }
 
-func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*LegacyJSONOutput, error) {
+func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) *LegacyJSONOutput {
 	var source LegacyJSONCompatibleSource
 	switch r.SourceType {
 	case sourcespb.SourceType_SOURCE_TYPE_GIT:
@@ -63,14 +60,14 @@ func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*Leg
 	case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
 		source = r.SourceMetadata.GetGitlab()
 	default:
-		return nil, fmt.Errorf("legacy JSON output can not be used with this source: %s", r.SourceName)
+		log.Fatalf("legacy JSON output can not be used with this source: %s", r.SourceName)
 	}
 
 	// The repo will be needed to gather info needed for the legacy output that isn't included in the new
 	// output format.
 	repo, err := gogit.PlainOpenWithOptions(repoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
 	if err != nil {
-		return nil, fmt.Errorf("could not open repo %q: %w", repoPath, err)
+		logrus.WithError(err).Fatalf("could not open repo: %s", repoPath)
 	}
 
 	fileName := source.GetFile()
@@ -99,7 +96,7 @@ func ConvertToLegacyJSON(r *detectors.ResultWithMetadata, repoPath string) (*Leg
 		Reason:       r.Result.DetectorType.String(),
 		StringsFound: []string{foundString},
 	}
-	return output, nil
+	return output
 }
 
 // BranchHeads creates a map of branch names to their head commit. This can be used to find if a commit is an ancestor
@@ -111,17 +108,16 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
 		return branches, err
 	}
 
-	logger := context.Background().Logger()
 	err = branchIter.ForEach(func(branchRef *plumbing.Reference) error {
 		branchName := branchRef.Name().String()
 		headHash, err := repo.ResolveRevision(plumbing.Revision(branchName))
 		if err != nil {
-			logger.Error(err, "unable to resolve head of branch", "branch", branchRef.Name().String())
+			logrus.WithError(err).Errorf("unable to resolve head of branch: %s", branchRef.Name().String())
 			return nil
 		}
 		headCommit, err := repo.CommitObject(*headHash)
 		if err != nil {
-			logger.Error(err, "unable to get commit", "commit", headCommit.String())
+			logrus.WithError(err).Errorf("unable to get commit: %s", headCommit.String())
 			return nil
 		}
 		branches[branchName] = headCommit
@@ -132,17 +128,15 @@ func BranchHeads(repo *gogit.Repository) (map[string]*object.Commit, error) {
 
 // FindBranch returns the first branch a commit is a part of. Not the most accurate, but it should work similar to pre v3.0.
 func FindBranch(commit *object.Commit, repo *gogit.Repository) string {
-	logger := context.Background().Logger()
 	branches, err := BranchHeads(repo)
 	if err != nil {
-		logger.Error(err, "could not list branches")
+		logrus.WithError(err).Fatal("could not list branches")
-		os.Exit(1)
 	}
 
 	for name, head := range branches {
 		isAncestor, err := commit.IsAncestor(head)
 		if err != nil {
-			logger.Error(err, fmt.Sprintf("could not determine if %s is an ancestor of %s", commit.Hash.String(), head.Hash.String()))
+			logrus.WithError(err).Errorf("could not determine if %s is an ancestor of %s", commit.Hash.String(), head.Hash.String())
 			continue
 		}
 		if isAncestor {
@@ -155,13 +149,12 @@ func FindBranch(commit *object.Commit, repo *gogit.Repository) string {
 // GenerateDiff will take a commit and create a string diff between the commit and its first parent.
 func GenerateDiff(commit *object.Commit, fileName string) string {
 	var diff string
-	logger := context.Background().Logger().WithValues("file", fileName)
 
 	// First grab the first parent of the commit. If there are none, we are at the first commit and should diff against
 	// an empty file.
 	parent, err := commit.Parent(0)
 	if err != object.ErrParentNotFound && err != nil {
-		logger.Error(err, "could not find parent", "commit", commit.Hash.String())
+		logrus.WithError(err).Errorf("could not find parent of %s", commit.Hash.String())
 	}
 
 	// Now get the files from the commit and its parent.
@@ -169,13 +162,13 @@ func GenerateDiff(commit *object.Commit, fileName string) string {
 	if parent != nil {
 		parentFile, err = parent.File(fileName)
 		if err != nil && err != object.ErrFileNotFound {
-			logger.Error(err, "could not get previous version of file")
+			logrus.WithError(err).Errorf("could not get previous version of file: %q", fileName)
 			return diff
 		}
 	}
 	commitFile, err := commit.File(fileName)
 	if err != nil {
-		logger.Error(err, "could not get current version of file")
+		logrus.WithError(err).Errorf("could not get current version of file: %q", fileName)
 		return diff
 	}
 
@@ -186,14 +179,14 @@ func GenerateDiff(commit *object.Commit, fileName string) string {
 	if parentFile != nil {
 		oldContent, err = parentFile.Contents()
 		if err != nil {
-			logger.Error(err, "could not get contents of previous version of file")
+			logrus.WithError(err).Errorf("could not get contents of previous version of file: %q", fileName)
 		}
 	}
 	// commitFile should never be nil at this point, but double-checking so we don't get a nil error.
 	if commitFile != nil {
 		newContent, _ = commitFile.Contents()
 		if err != nil {
-			logger.Error(err, "could not get contents of current version of file")
+			logrus.WithError(err).Errorf("could not get contents of current version of file: %q", fileName)
 		}
 	}
 
@@ -206,7 +199,7 @@ func GenerateDiff(commit *object.Commit, fileName string) string {
 		// The String() method URL escapes the diff, so it needs to be undone.
 		patchDiff, err := url.QueryUnescape(patch.String())
 		if err != nil {
-			logger.Error(err, "unable to unescape diff")
+			logrus.WithError(err).Error("unable to unescape diff")
 		}
 		diff += patchDiff
 	}

pkg/output/plain.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/fatih/color"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 
@@ -20,7 +21,7 @@ var (
 	whitePrinter  = color.New(color.FgWhite)
 )
 
-func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
+func PrintPlainOutput(r *detectors.ResultWithMetadata) {
 	out := outputFormat{
 		DetectorType: r.Result.DetectorType.String(),
 		DecoderType:  r.Result.DecoderType.String(),
@@ -31,7 +32,7 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
 
 	meta, err := structToMap(out.MetaData.Data)
 	if err != nil {
-		return fmt.Errorf("could not marshal result: %w", err)
+		logrus.WithError(err).Fatal("could not marshal result")
 	}
 
 	printer := greenPrinter
@@ -60,7 +61,6 @@ func PrintPlainOutput(r *detectors.ResultWithMetadata) error {
 		printer.Printf("%s: %v\n", cases.Title(language.AmericanEnglish).String(k), aggregateData[k])
 	}
 	fmt.Println("")
-	return nil
 }
 
 func structToMap(obj interface{}) (m map[string]map[string]interface{}, err error) {

pkg/pb/source_metadatapb/source_metadata.pb.validate.go

@@ -1267,117 +1267,6 @@ var _ interface {
 	ErrorName() string
 } = GitlabValidationError{}
 
-// Validate checks the field values on GoogleDrive with the rules defined in
-// the proto definition for this message. If any rules are violated, the first
-// error encountered is returned, or nil if there are no violations.
-func (m *GoogleDrive) Validate() error {
-	return m.validate(false)
-}
-
-// ValidateAll checks the field values on GoogleDrive with the rules defined in
-// the proto definition for this message. If any rules are violated, the
-// result is a list of violation errors wrapped in GoogleDriveMultiError, or
-// nil if none found.
-func (m *GoogleDrive) ValidateAll() error {
-	return m.validate(true)
-}
-
-func (m *GoogleDrive) validate(all bool) error {
-	if m == nil {
-		return nil
-	}
-
-	var errors []error
-
-	// no validation rules for File
-
-	// no validation rules for Link
-
-	// no validation rules for Email
-
-	// no validation rules for Timestamp
-
-	// no validation rules for Shared
-
-	// no validation rules for LastModifiedBy
-
-	if len(errors) > 0 {
-		return GoogleDriveMultiError(errors)
-	}
-
-	return nil
-}
-
-// GoogleDriveMultiError is an error wrapping multiple validation errors
-// returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
-type GoogleDriveMultiError []error
-
-// Error returns a concatenation of all the error messages it wraps.
-func (m GoogleDriveMultiError) Error() string {
-	var msgs []string
-	for _, err := range m {
-		msgs = append(msgs, err.Error())
-	}
-	return strings.Join(msgs, "; ")
-}
-
-// AllErrors returns a list of validation violation errors.
-func (m GoogleDriveMultiError) AllErrors() []error { return m }
-
-// GoogleDriveValidationError is the validation error returned by
-// GoogleDrive.Validate if the designated constraints aren't met.
-type GoogleDriveValidationError struct {
-	field  string
-	reason string
-	cause  error
-	key    bool
-}
-
-// Field function returns field value.
-func (e GoogleDriveValidationError) Field() string { return e.field }
-
-// Reason function returns reason value.
-func (e GoogleDriveValidationError) Reason() string { return e.reason }
-
-// Cause function returns cause value.
-func (e GoogleDriveValidationError) Cause() error { return e.cause }
-
-// Key function returns key value.
-func (e GoogleDriveValidationError) Key() bool { return e.key }
-
-// ErrorName returns error name.
-func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
-
-// Error satisfies the builtin error interface
-func (e GoogleDriveValidationError) Error() string {
-	cause := ""
-	if e.cause != nil {
-		cause = fmt.Sprintf(" | caused by: %v", e.cause)
-	}
-
-	key := ""
-	if e.key {
-		key = "key for "
-	}
-
-	return fmt.Sprintf(
-		"invalid %sGoogleDrive.%s: %s%s",
-		key,
-		e.field,
-		e.reason,
-		cause)
-}
-
-var _ error = GoogleDriveValidationError{}
-
-var _ interface {
-	Field() string
-	Reason() string
-	Key() bool
-	Cause() error
-	ErrorName() string
-} = GoogleDriveValidationError{}
-
 // Validate checks the field values on GCS with the rules defined in the proto
 // definition for this message. If any rules are violated, the first error
 // encountered is returned, or nil if there are no violations.

pkg/pb/sourcespb/sources.pb.validate.go

@@ -705,10 +705,6 @@ func (m *Confluence) validate(all bool) error {
 
 	// no validation rules for InsecureSkipVerifyTls
 
-	// no validation rules for IncludeAttachments
-
-	// no validation rules for SkipHistory
-
 	switch m.Credential.(type) {
 
 	case *Confluence_Unauthenticated:
@@ -1887,112 +1883,6 @@ var _ interface {
 	ErrorName() string
 } = GitHubValidationError{}
 
-// Validate checks the field values on GoogleDrive with the rules defined in
-// the proto definition for this message. If any rules are violated, the first
-// error encountered is returned, or nil if there are no violations.
-func (m *GoogleDrive) Validate() error {
-	return m.validate(false)
-}
-
-// ValidateAll checks the field values on GoogleDrive with the rules defined in
-// the proto definition for this message. If any rules are violated, the
-// result is a list of violation errors wrapped in GoogleDriveMultiError, or
-// nil if none found.
-func (m *GoogleDrive) ValidateAll() error {
-	return m.validate(true)
-}
-
-func (m *GoogleDrive) validate(all bool) error {
-	if m == nil {
-		return nil
-	}
-
-	var errors []error
-
-	switch m.Credential.(type) {
-
-	case *GoogleDrive_RefreshToken:
-		// no validation rules for RefreshToken
-
-	}
-
-	if len(errors) > 0 {
-		return GoogleDriveMultiError(errors)
-	}
-
-	return nil
-}
-
-// GoogleDriveMultiError is an error wrapping multiple validation errors
-// returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
-type GoogleDriveMultiError []error
-
-// Error returns a concatenation of all the error messages it wraps.
-func (m GoogleDriveMultiError) Error() string {
-	var msgs []string
-	for _, err := range m {
-		msgs = append(msgs, err.Error())
-	}
-	return strings.Join(msgs, "; ")
-}
-
-// AllErrors returns a list of validation violation errors.
-func (m GoogleDriveMultiError) AllErrors() []error { return m }
-
-// GoogleDriveValidationError is the validation error returned by
-// GoogleDrive.Validate if the designated constraints aren't met.
-type GoogleDriveValidationError struct {
-	field  string
-	reason string
-	cause  error
-	key    bool
-}
-
-// Field function returns field value.
-func (e GoogleDriveValidationError) Field() string { return e.field }
-
-// Reason function returns reason value.
-func (e GoogleDriveValidationError) Reason() string { return e.reason }
-
-// Cause function returns cause value.
-func (e GoogleDriveValidationError) Cause() error { return e.cause }
-
-// Key function returns key value.
-func (e GoogleDriveValidationError) Key() bool { return e.key }
-
-// ErrorName returns error name.
-func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
-
-// Error satisfies the builtin error interface
-func (e GoogleDriveValidationError) Error() string {
-	cause := ""
-	if e.cause != nil {
-		cause = fmt.Sprintf(" | caused by: %v", e.cause)
-	}
-
-	key := ""
-	if e.key {
-		key = "key for "
-	}
-
-	return fmt.Sprintf(
-		"invalid %sGoogleDrive.%s: %s%s",
-		key,
-		e.field,
-		e.reason,
-		cause)
-}
-
-var _ error = GoogleDriveValidationError{}
-
-var _ interface {
-	Field() string
-	Reason() string
-	Key() bool
-	Cause() error
-	ErrorName() string
-} = GoogleDriveValidationError{}
-
 // Validate checks the field values on JIRA with the rules defined in the proto
 // definition for this message. If any rules are violated, the first error
 // encountered is returned, or nil if there are no violations.

pkg/sources/chunker.go

@@ -5,6 +5,8 @@ import (
 	"bytes"
 	"errors"
 	"io"
+
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -30,6 +32,7 @@ func Chunker(originalChunk *Chunk) chan *Chunk {
 			chunk := *originalChunk
 			n, err := reader.Read(chunkBytes)
 			if err != nil && !errors.Is(err, io.EOF) {
+				logrus.WithError(err).Error("Error chunking reader.")
 				break
 			}
 			peekData, _ := reader.Peek(PeekSize)

pkg/sources/circleci/circleci.go

@@ -51,17 +51,17 @@ func (s *Source) JobID() int64 {
 }
 
 // Init returns an initialized CircleCI source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, cfg sources.SourceConfig) error {
-	s.name = name
+	s.name = cfg.Name
-	s.sourceId = sourceId
+	s.sourceId = cfg.SourceID
-	s.jobId = jobId
+	s.jobId = cfg.JobID
-	s.verify = verify
+	s.verify = cfg.Verify
 	s.jobPool = &errgroup.Group{}
-	s.jobPool.SetLimit(concurrency)
+	s.jobPool.SetLimit(cfg.Concurrency)
 	s.client = common.RetryableHttpClientTimeout(3)
 
 	var conn sourcespb.CircleCI
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
+	if err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{}); err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 

pkg/sources/circleci/circleci_test.go

@@ -71,7 +71,15 @@ func TestSource_Scan(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
+			cfg := sources.NewSourceConfig(
+				"circleci",
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(5),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/sources/errors.go

@@ -1,7 +1,6 @@
 package sources
 
 import (
-	"fmt"
 	"sync"
 )
 
@@ -30,7 +29,3 @@ func (s *ScanErrors) Count() uint64 {
 	defer s.mu.RUnlock()
 	return uint64(len(s.errors))
 }
-
-func (s *ScanErrors) String() string {
-	return fmt.Sprintf("%v", s.errors)
-}

pkg/sources/errors_test.go

@@ -1,7 +1,6 @@
 package sources
 
 import (
-	"fmt"
 	"reflect"
 	"sync"
 	"testing"
@@ -148,12 +147,3 @@ func TestScanErrorsCount(t *testing.T) {
 		})
 	}
 }
-
-func TestScanErrorsString(t *testing.T) {
-	se := NewScanErrors()
-	se.Add(nil)
-	want := "[<nil>]"
-	if got := fmt.Sprintf("%v", se); got != want {
-		t.Errorf("got %q, want %q", got, want)
-	}
-}

pkg/sources/filesystem/filesystem.go

@@ -58,19 +58,20 @@ func (s *Source) JobID() int64 {
 }
 
 // Init returns an initialized Filesystem source.
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
+func (s *Source) Init(aCtx context.Context, cfg sources.SourceConfig) error {
 	s.log = aCtx.Logger()
 
-	s.name = name
+	s.name = cfg.Name
-	s.sourceId = sourceId
+	s.sourceId = cfg.SourceID
-	s.jobId = jobId
+	s.jobId = cfg.JobID
-	s.verify = verify
+	s.verify = cfg.Verify
 
 	var conn sourcespb.Filesystem
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
+	if err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{}); err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
-	s.paths = append(conn.Paths, conn.Directories...)
+
+	s.paths = conn.Directories
 
 	return nil
 }
@@ -82,122 +83,100 @@ func (s *Source) WithFilter(filter *common.Filter) {
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	for i, path := range s.paths {
-		logger := ctx.Logger().WithValues("path", path)
-		if common.IsDone(ctx) {
-			return nil
-		}
 		s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "")
 
 		cleanPath := filepath.Clean(path)
-		fileInfo, err := os.Stat(cleanPath)
+		done := false
-		if err != nil {
+		go func() {
-			logger.Error(err, "unable to get file info")
+			<-ctx.Done()
-			continue
+			done = true
-		}
+		}()
 
-		if fileInfo.IsDir() {
+		err := fs.WalkDir(os.DirFS(cleanPath), ".", func(relativePath string, d fs.DirEntry, err error) error {
-			err = s.scanDir(ctx, cleanPath, chunksChan)
+			if err != nil {
-		} else {
+				return nil
-			err = s.scanFile(ctx, cleanPath, chunksChan)
+			}
-		}
+
+			path := filepath.Join(cleanPath, relativePath)
+			logger := ctx.Logger().WithValues("file", path)
+
+			fileStat, err := os.Stat(path)
+			if err != nil {
+				logger.Error(err, "unable to stat file")
+				return nil
+			}
+			if !fileStat.Mode().IsRegular() {
+				return nil
+			}
+
+			if s.filter != nil && !s.filter.Pass(path) {
+				return nil
+			}
+
+			inputFile, err := os.Open(path)
+			if err != nil {
+				logger.V(2).Info("error opening file", "error", err)
+				return nil
+			}
+			defer inputFile.Close()
+			logger.V(5).Info("scanning file")
+
+			reReader, err := diskbufferreader.New(inputFile)
+			if err != nil {
+				logger.Error(err, "Could not create re-readable reader.")
+			}
+			defer reReader.Close()
+
+			chunkSkel := &sources.Chunk{
+				SourceType: s.Type(),
+				SourceName: s.name,
+				SourceID:   s.SourceID(),
+				SourceMetadata: &source_metadatapb.MetaData{
+					Data: &source_metadatapb.MetaData_Filesystem{
+						Filesystem: &source_metadatapb.Filesystem{
+							File: sanitizer.UTF8(path),
+						},
+					},
+				},
+				Verify: s.verify,
+			}
+			if handlers.HandleFile(ctx, reReader, chunkSkel, chunksChan) {
+				return nil
+			}
+
+			if err := reReader.Reset(); err != nil {
+				return err
+			}
+			reReader.Stop()
+			data, err := io.ReadAll(reReader)
+			if err != nil {
+				return err
+			}
+			chunksChan <- &sources.Chunk{
+				SourceType: s.Type(),
+				SourceName: s.name,
+				SourceID:   s.SourceID(),
+				Data:       data,
+				SourceMetadata: &source_metadatapb.MetaData{
+					Data: &source_metadatapb.MetaData_Filesystem{
+						Filesystem: &source_metadatapb.Filesystem{
+							File: sanitizer.UTF8(path),
+						},
+					},
+				},
+				Verify: s.verify,
+			}
+			return nil
+		})
 
 		if err != nil && err != io.EOF {
-			logger.Info("error scanning filesystem", "error", err)
+			return errors.New(err)
 		}
-	}
+
-	return nil
+		if done {
-}
+			return nil
-
+		}
-func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
+
-	return fs.WalkDir(os.DirFS(path), ".", func(relativePath string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return nil
-		}
-		fullPath := filepath.Join(path, relativePath)
-
-		// Skip over non-regular files. We do this check here to suppress noisy
-		// logs for trying to scan directories and other non-regular files in
-		// our traversal.
-		fileStat, err := os.Stat(fullPath)
-		if err != nil {
-			ctx.Logger().Info("unable to stat file", "path", fullPath, "error", err)
-			return nil
-		}
-		if !fileStat.Mode().IsRegular() {
-			return nil
-		}
-		if s.filter != nil && !s.filter.Pass(fullPath) {
-			return nil
-		}
-
-		if err = s.scanFile(ctx, fullPath, chunksChan); err != nil {
-			ctx.Logger().Info("error scanning file", "path", fullPath, "error", err)
-		}
-		return nil
-	})
-}
-
-func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
-	logger := ctx.Logger().WithValues("path", path)
-	fileStat, err := os.Stat(path)
-	if err != nil {
-		return fmt.Errorf("unable to stat file: %w", err)
-	}
-	if !fileStat.Mode().IsRegular() {
-		return fmt.Errorf("not a regular file")
-	}
-
-	inputFile, err := os.Open(path)
-	if err != nil {
-		return fmt.Errorf("unable to open file: %w", err)
-	}
-	defer inputFile.Close()
-	logger.V(3).Info("scanning file")
-
-	reReader, err := diskbufferreader.New(inputFile)
-	if err != nil {
-		return fmt.Errorf("could not create re-readable reader: %w", err)
-	}
-	defer reReader.Close()
-
-	chunkSkel := &sources.Chunk{
-		SourceType: s.Type(),
-		SourceName: s.name,
-		SourceID:   s.SourceID(),
-		SourceMetadata: &source_metadatapb.MetaData{
-			Data: &source_metadatapb.MetaData_Filesystem{
-				Filesystem: &source_metadatapb.Filesystem{
-					File: sanitizer.UTF8(path),
-				},
-			},
-		},
-		Verify: s.verify,
-	}
-	if handlers.HandleFile(ctx, reReader, chunkSkel, chunksChan) {
-		return nil
-	}
-
-	if err := reReader.Reset(); err != nil {
-		return err
-	}
-	reReader.Stop()
-	data, err := io.ReadAll(reReader)
-	if err != nil {
-		return fmt.Errorf("unable to read file: %w", err)
-	}
-	chunksChan <- &sources.Chunk{
-		SourceType: s.Type(),
-		SourceName: s.name,
-		SourceID:   s.SourceID(),
-		Data:       data,
-		SourceMetadata: &source_metadatapb.MetaData{
-			Data: &source_metadatapb.MetaData_Filesystem{
-				Filesystem: &source_metadatapb.Filesystem{
-					File: sanitizer.UTF8(path),
-				},
-			},
-		},
-		Verify: s.verify,
 	}
 	return nil
 }

pkg/sources/filesystem/filesystem_test.go

@@ -33,7 +33,7 @@ func TestSource_Scan(t *testing.T) {
 			init: init{
 				name: "this repo",
 				connection: &sourcespb.Filesystem{
-					Paths: []string{"."},
+					Directories: []string{"."},
 				},
 				verify: true,
 			},
@@ -56,7 +56,15 @@ func TestSource_Scan(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(5),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/sources/git/git.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
@@ -85,25 +86,25 @@ func (s *Source) JobID() int64 {
 }
 
 // Init returns an initialized GitHub source.
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, cfg sources.SourceConfig) error {
 
-	s.name = name
+	s.name = cfg.Name
-	s.sourceId = sourceId
+	s.sourceId = cfg.SourceID
-	s.jobId = jobId
+	s.jobId = cfg.JobID
-	s.verify = verify
+	s.verify = cfg.Verify
 
 	var conn sourcespb.Git
-	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
+	if err := anypb.UnmarshalTo(cfg.Connection, &conn, proto.UnmarshalOptions{}); err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 
 	s.conn = &conn
 
-	if concurrency == 0 {
+	if cfg.Concurrency == 0 {
-		concurrency = runtime.NumCPU()
+		cfg.Concurrency = runtime.NumCPU()
 	}
 
-	s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, concurrency,
+	s.git = NewGit(s.Type(), s.jobId, s.sourceId, s.name, s.verify, cfg.Concurrency,
 		func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData {
 			return &source_metadatapb.MetaData{
 				Data: &source_metadatapb.MetaData_Git{
@@ -256,7 +257,7 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
 	if err := gitCmdCheck(); err != nil {
 		return "", nil, err
 	}
-	clonePath, err := os.MkdirTemp(os.TempDir(), "trufflehog")
+	clonePath, err := ioutil.TempDir(os.TempDir(), "trufflehog")
 	if err != nil {
 		return "", nil, err
 	}
@@ -472,9 +473,9 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 	}
 }
 
-// ScanStaged chunks staged changes.
+// ScanUnstaged chunks unstaged changes.
-func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
-	// Get the URL metadata for reporting (may be empty).
+	// get the URL metadata for reporting (may be empty)
 	urlMetadata := getSafeRemoteURL(repo, "origin")
 
 	commitChan, err := gitparse.NewParser().Unstaged(ctx, path)
@@ -488,11 +489,11 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 	var depth int64
 	var reachedBase = false
 
-	ctx.Logger().V(1).Info("scanning staged changes", "path", path)
+	ctx.Logger().V(1).Info("scanning unstaged changes", "path", path)
 	for commit := range commitChan {
 		for _, diff := range commit.Diffs {
 			logger := ctx.Logger().WithValues("filename", diff.PathB, "commit", commit.Hash, "file", diff.PathB)
-			logger.V(2).Info("scanning staged changes from git")
+			logger.V(2).Info("scanning unstaged changes from git")
 
 			if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
 				logger.V(1).Info("reached max depth")
@@ -525,7 +526,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 			// Handle binary files by reading the entire file rather than using the diff.
 			if diff.IsBinary {
 				commitHash := plumbing.NewHash(hash)
-				metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, 0)
+				metadata := s.sourceMetadataFunc(fileName, email, "Unstaged", when, urlMetadata, 0)
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
@@ -539,7 +540,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 				continue
 			}
 
-			metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, int64(diff.LineStart))
+			metadata := s.sourceMetadataFunc(fileName, email, "Unstaged", when, urlMetadata, int64(diff.LineStart))
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
@@ -564,7 +565,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
 	if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
 		return err
 	}
-	if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
+	if err := s.ScanUnstaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
 		ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
 	}
 

pkg/sources/git/git_test.go

@@ -130,14 +130,22 @@ func TestSource_Scan(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, tt.init.concurrency)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(tt.init.concurrency),
+			)
+			err = s.Init(ctx, cfg)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			chunksCh := make(chan *sources.Chunk, 1)
 			go func() {
-				assert.NoError(t, s.Chunks(ctx, chunksCh))
+				s.Chunks(ctx, chunksCh)
 			}()
 			gotChunk := <-chunksCh
 			gotChunk.Data = nil
@@ -263,7 +271,15 @@ func TestSource_Chunks_Integration(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(4),
+			)
+			err = s.Init(ctx, cfg)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -406,7 +422,15 @@ func TestSource_Chunks_Edge_Cases(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 4)
+			cfg := sources.NewSourceConfig(
+				tt.init.name,
+				0,
+				0,
+				conn,
+				sources.WithVerify(tt.init.verify),
+				sources.WithConcurrency(4),
+			)
+			err = s.Init(ctx, cfg)
 			if err != nil {
 				t.Errorf("Source.Init() error = %v", err)
 				return