S3 scanner improvements (#938)

* Add custom_detectors proto

* Generate proto code

* Create custom_detectors package

Also create protoyaml package to test YAML unmarshalling the
configuration.

* Simplify custom_detectors proto by removing connection

* Generate proto code

* Update custom_detectors parsing tests

.github/workflows/test.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v3
     - id: 'auth'
-      uses: 'google-github-actions/auth@v0'
+      uses: 'google-github-actions/auth@v1'
       with:
         workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
         service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
@@ -42,7 +42,7 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v3
     - id: 'auth'
-      uses: 'google-github-actions/auth@v0'
+      uses: 'google-github-actions/auth@v1'
       with:
         workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
         service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'

README.md

@@ -162,7 +162,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
 - syslog
 - file and stdin (coming soon)
 
-Each subcommand can have options that you can see with the `-h` flag provided to the sub command:
+Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
 
 ```
 $ trufflehog git --help
@@ -276,6 +276,11 @@ repos:
       stages: ["commit", "push"]
 ```
 
+## Use as a library
+
+Currently, trufflehog is in heavy development and no guarantees can be made on
+the stability of the public APIs at this time.
+
 ## Contributors
 
 This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)].

go.mod

@@ -9,7 +9,7 @@ replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleak
 replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2
 
 require (
-	cloud.google.com/go/secretmanager v1.8.0
+	cloud.google.com/go/secretmanager v1.9.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
 	github.com/TheZeroSlave/zapsentry v1.12.0
 	github.com/aws/aws-sdk-go v1.44.83
@@ -22,9 +22,10 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.6.13
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.3
-	github.com/getsentry/sentry-go v0.14.0
+	github.com/getsentry/sentry-go v0.15.0
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.4.2
+	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/go-logr/logr v1.2.3
 	github.com/go-logr/zapr v1.2.3
 	github.com/go-redis/redis v6.15.9+incompatible
@@ -33,7 +34,6 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v42 v42.0.0
 	github.com/gorilla/mux v1.8.0
-	github.com/h2non/filetype v1.1.3
 	github.com/hashicorp/go-retryablehttp v0.7.1
 	github.com/jlaffaye/ftp v0.1.0
 	github.com/joho/godotenv v1.4.0
@@ -45,6 +45,7 @@ require (
 	github.com/mholt/archiver/v4 v4.0.0-alpha.7
 	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/pkg/errors v0.9.1
+	github.com/rabbitmq/amqp091-go v1.5.0
 	github.com/sergi/go-diff v1.2.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/stretchr/testify v1.8.1
@@ -54,17 +55,19 @@ require (
 	go.uber.org/zap v1.23.0
 	golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
-	golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1
-	golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0
-	google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e
+	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783
+	golang.org/x/sync v0.1.0
+	google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
-	cloud.google.com/go/compute v1.10.0 // indirect
-	cloud.google.com/go/iam v0.5.0 // indirect
+	cloud.google.com/go/compute v1.12.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
+	cloud.google.com/go/iam v0.6.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.24 // indirect
 	github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
@@ -72,6 +75,7 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
 	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
@@ -83,6 +87,7 @@ require (
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -134,9 +139,10 @@ require (
 	golang.org/x/text v0.4.0 // indirect
 	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
 	golang.org/x/tools v0.1.12 // indirect
-	google.golang.org/api v0.99.0 // indirect
+	google.golang.org/api v0.102.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/grpc v1.50.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -17,25 +17,28 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb
 cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
 cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
 cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY=
-cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8=
+cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4=
-cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU=
+cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0=
+cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
+cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
+cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/iam v0.5.0 h1:fz9X5zyTWBmamZsqvqZqD7khbifcZF/q+Z1J8pfhIUg=
-cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc=
+cloud.google.com/go/iam v0.6.0 h1:nsqQC88kT5Iwlm4MeNGTpfMWddp6NB/UOLFTH6m1QfQ=
+cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc=
+cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
 cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/secretmanager v1.8.0 h1:4wYWL2t10q+xUtFFS0QuWlqwQguMrwC6FDpjtMM6cUI=
-cloud.google.com/go/secretmanager v1.8.0/go.mod h1:hnVgi/bN5MYHd3Gt0SPuTPPp5ENina1/LxM+2W9U9J4=
+cloud.google.com/go/secretmanager v1.9.0 h1:xE6uXljAC1kCR8iadt9+/blg1fvSbmenlsDN4fT9gqw=
+cloud.google.com/go/secretmanager v1.9.0/go.mod h1:b71qH2l1yHmWQHt9LC80akm86mX8AL6X1MA01dW8ht4=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
@@ -64,6 +67,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
 github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
+github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
@@ -137,10 +142,12 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70=
-github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I=
+github.com/getsentry/sentry-go v0.15.0 h1:CP9bmA7pralrVUedYZsmIHWpq/pBtXTSew7xvVpfLaA=
+github.com/getsentry/sentry-go v0.15.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I=
 github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
+github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
@@ -155,6 +162,8 @@ github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
+github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -261,8 +270,6 @@ github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
-github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@@ -375,6 +382,8 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rabbitmq/amqp091-go v1.5.0 h1:VouyHPBu1CrKyJVfteGknGOGCzmOz0zcv/tONLkb7rg=
+github.com/rabbitmq/amqp091-go v1.5.0/go.mod h1:JsV0ofX5f1nwOGafb8L5rBItt9GyhfQfcJj+oyz0dGg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -400,6 +409,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
@@ -430,6 +440,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
@@ -446,7 +457,8 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
+go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
@@ -505,6 +517,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -542,6 +555,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -558,8 +572,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1 h1:3VPzK7eqH25j7GYw5w6g/GzNRc0/fYtrxz27z1gD4W0=
-golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
+golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk=
+golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -572,8 +586,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpiB4nXvw1XYzFPGgzLUOY=
-golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -616,9 +630,11 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -699,6 +715,7 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -725,8 +742,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513
 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
 google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
 google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.99.0 h1:tsBtOIklCE2OFxhmcYSVqGwSAN/Y897srxmcvAQnwK8=
-google.golang.org/api v0.99.0/go.mod h1:1YOf74vkVndF7pG6hIHuINsM7eWwpVTAfNMNiL91A08=
+google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I=
+google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -771,8 +788,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e h1:halCgTFuLWDRD61piiNSxPsARANGD3Xl16hPrLgLiIg=
-google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U=
+google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c h1:QgY/XxIAIeccR+Ca/rDdKubLIU9rcJ3xfy1DC/Wd2Oo=
+google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c/go.mod h1:CGI5F/G+E5bKwmfYo09AXuVN4dD894kIKUFmVbP2/Fo=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -841,3 +858,5 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

main.go

@@ -66,9 +66,11 @@ var (
 
 	gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.")
 	// TODO: Add more GitLab options
-	gitlabScanEndpoint = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
-	gitlabScanRepos    = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
-	gitlabScanToken    = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
+	gitlabScanEndpoint     = gitlabScan.Flag("endpoint", "GitLab endpoint.").Default("https://gitlab.com").String()
+	gitlabScanRepos        = gitlabScan.Flag("repo", "GitLab repo url. You can repeat this flag. Leave empty to scan all repos accessible with provided credential. Example: https://gitlab.com/org/repo.git").Strings()
+	gitlabScanToken        = gitlabScan.Flag("token", "GitLab token. Can be provided with environment variable GITLAB_TOKEN.").Envar("GITLAB_TOKEN").Required().String()
+	gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
 
 	filesystemScan        = cli.Command("filesystem", "Find credentials in a filesystem.")
 	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
@@ -223,10 +225,16 @@ func run(state overseer.State) {
 			logrus.WithError(err).Fatal("Failed to scan Github.")
 		}
 	case gitlabScan.FullCommand():
+		filter, err := common.FilterFromFiles(*gitlabScanIncludePaths, *gitlabScanExcludePaths)
+		if err != nil {
+			logrus.WithError(err).Fatal("could not create filter")
+		}
+
 		gitlab := func(c *sources.Config) {
 			c.Endpoint = *gitlabScanEndpoint
 			c.Token = *gitlabScanToken
 			c.Repos = *gitlabScanRepos
+			c.Filter = filter
 		}
 
 		if err = e.ScanGitLab(ctx, sources.NewConfig(gitlab)); err != nil {

pkg/common/recover.go

@@ -18,7 +18,10 @@ func Recover(ctx context.Context) {
 		if eventID := sentry.CurrentHub().Recover(err); eventID != nil {
 			ctx.Logger().Info("panic captured", "event_id", *eventID)
 		}
-		fmt.Fprint(os.Stderr, panicStack)
+		ctx.Logger().Error(fmt.Errorf("panic"), "recovered from panic",
+			"stack-trace", panicStack,
+			"recover", err,
+		)
 		if !sentry.Flush(time.Second * 5) {
 			ctx.Logger().Info("sentry flush failed")
 		}
@@ -32,7 +35,10 @@ func RecoverWithExit(ctx context.Context) {
 		if eventID := sentry.CurrentHub().Recover(err); eventID != nil {
 			ctx.Logger().Info("panic captured", "event_id", *eventID)
 		}
-		fmt.Fprint(os.Stderr, panicStack)
+		ctx.Logger().Error(fmt.Errorf("panic"), "recovered from panic before exiting",
+			"stack-trace", panicStack,
+			"recover", err,
+		)
 		if !sentry.Flush(time.Second * 5) {
 			ctx.Logger().Info("sentry flush failed")
 		}

pkg/common/vars.go

@@ -2,22 +2,14 @@ package common
 
 import (
 	"path/filepath"
-
-	"github.com/h2non/filetype"
 )
 
 var (
 	KB, MB, GB, TB, PB = 1e3, 1e6, 1e9, 1e12, 1e15
-	IGNORED_EXTENSIONS = []string{"pdf", "mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "tif", "tiff", "jpg", "jpeg", "png", "gif", "zip", "webp"}
+	IGNORED_EXTENSIONS = []string{"mp4", "avi", "mpeg", "mpg", "mov", "wmv", "m4p", "swf", "mp2", "flv", "vob", "webm", "hdv", "3gp", "ogg", "mp3", "wav", "flac", "webp"}
 )
 
-func SkipFile(filename string, data []byte) bool {
-	if filepath.Ext(filename) == "" {
-		//no sepcified extension, check mimetype
-		if filetype.IsArchive(data[:256]) {
-			return true
-		}
-	}
+func SkipFile(filename string) bool {
 	for _, ext := range IGNORED_EXTENSIONS {
 		if filepath.Ext(filename) == ext {
 			return true

pkg/custom_detectors/custom_detectors_test.go

@@ -0,0 +1,68 @@
+package custom_detectors
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/protoyaml"
+)
+
+const testCustomRegexYaml = `name: Internal bi tool
+keywords:
+- secret_v1_
+- pat_v2_
+regex:
+  id_pat_example: ([a-zA-Z0-9]{32})
+  secret_pat_example: ([a-zA-Z0-9]{32})
+verify:
+- endpoint: http://localhost:8000/{id_pat_example}
+  unsafe: true
+  headers:
+  - 'Authorization: Bearer {secret_pat_example.0}'
+  successRanges:
+  - 200-250
+  - '288'`
+
+// Helper function to test equality to the data in testCustomRegexYaml.
+func assertExpected(t *testing.T, got *custom_detectorspb.CustomRegex) {
+	assert.Equal(t, "Internal bi tool", got.Name)
+	assert.Equal(t, []string{"secret_v1_", "pat_v2_"}, got.Keywords)
+	assert.Equal(t, map[string]string{
+		"id_pat_example":     "([a-zA-Z0-9]{32})",
+		"secret_pat_example": "([a-zA-Z0-9]{32})",
+	}, got.Regex)
+	assert.Equal(t, 1, len(got.Verify))
+	assert.Equal(t, "http://localhost:8000/{id_pat_example}", got.Verify[0].Endpoint)
+	assert.Equal(t, true, got.Verify[0].Unsafe)
+	assert.Equal(t, []string{"Authorization: Bearer {secret_pat_example.0}"}, got.Verify[0].Headers)
+	assert.Equal(t, []string{"200-250", "288"}, got.Verify[0].SuccessRanges)
+}
+
+func TestCustomRegexParsing(t *testing.T) {
+	var message custom_detectorspb.CustomRegex
+
+	assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testCustomRegexYaml), &message))
+	assertExpected(t, &message)
+}
+
+func TestCustomDetectorsParsing(t *testing.T) {
+	var testYamlConfig string
+	// Build a config file using testCustomRegexYaml.
+	{
+		var lines []string
+		for i, line := range strings.Split(testCustomRegexYaml, "\n") {
+			if i == 0 {
+				lines = append(lines, line)
+				continue
+			}
+			lines = append(lines, "  "+line)
+		}
+		testYamlConfig = "detectors:\n- " + strings.Join(lines, "\n")
+	}
+
+	var messages custom_detectorspb.CustomDetectors
+	assert.NoError(t, protoyaml.UnmarshalStrict([]byte(testYamlConfig), &messages))
+	assertExpected(t, messages.Detectors[0])
+}

pkg/detectors/aws/aws.go

@@ -17,10 +17,35 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+type scanner struct {
+	skipIDs map[string]struct{}
+}
 
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
+func New(opts ...func(*scanner)) *scanner {
+	scanner := &scanner{
+		skipIDs: map[string]struct{}{},
+	}
+	for _, opt := range opts {
+
+		opt(scanner)
+	}
+
+	return scanner
+}
+
+func WithSkipIDs(skipIDs []string) func(*scanner) {
+	return func(s *scanner) {
+		ids := map[string]struct{}{}
+		for _, id := range skipIDs {
+			ids[id] = struct{}{}
+		}
+
+		s.skipIDs = ids
+	}
+}
+
+// Ensure the scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*scanner)(nil)
 
 var (
 	client = common.SaneHttpClient()
@@ -37,7 +62,7 @@ var (
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
+func (s scanner) Keywords() []string {
 	return []string{
 		"AKIA",
 		"ABIA",
@@ -60,7 +85,7 @@ func GetHMAC(key []byte, data []byte) []byte {
 }
 
 // FromData will find and optionally verify AWS secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
 	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
@@ -72,6 +97,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 		resIDMatch := strings.TrimSpace(idMatch[1])
 
+		if s.skipIDs != nil {
+			if _, ok := s.skipIDs[resIDMatch]; ok {
+				continue
+			}
+		}
+
 		for _, secretMatch := range secretMatches {
 			if len(secretMatch) != 2 {
 				continue

pkg/detectors/aws/aws_test.go

@@ -39,14 +39,14 @@ func TestAWS_FromChunk(t *testing.T) {
 	}
 	tests := []struct {
 		name    string
-		s       Scanner
+		s       scanner
 		args    args
 		want    []detectors.Result
 		wantErr bool
 	}{
 		{
 			name: "found, verified",
-			s:    Scanner{},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -68,7 +68,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified",
-			s:    Scanner{},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
@@ -86,7 +86,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "not found",
-			s:    Scanner{},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte("You cannot find the secret within"),
@@ -97,7 +97,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found two, one included for every ID found",
-			s:    Scanner{},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),
@@ -124,7 +124,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "not found, because unverified secret was a hash",
-			s:    Scanner{},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation.
@@ -135,7 +135,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",
-			s:    Scanner{},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),
@@ -162,7 +162,7 @@ func TestAWS_FromChunk(t *testing.T) {
 		},
 		{
 			name: "found, unverified, with leading +",
-			s:    Scanner{},
+			s:    scanner{},
 			args: args{
 				ctx:    context.Background(),
 				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
@@ -177,10 +177,24 @@ func TestAWS_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "skipped",
+			s: scanner{
+				skipIDs: map[string]struct{}{
+					"AKIAWARWQKZNHMZBLY4I": {},
+				},
+			},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
+			s := tt.s
 			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("AWS.FromData() error = %v, wantErr %v", err, tt.wantErr)
@@ -201,7 +215,7 @@ func TestAWS_FromChunk(t *testing.T) {
 
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
-	s := Scanner{}
+	s := scanner{}
 	for name, data := range detectors.MustGetBenchmarkData() {
 		benchmark.Run(name, func(b *testing.B) {
 			for n := 0; n < b.N; n++ {

pkg/detectors/ftp/ftp.go

@@ -50,12 +50,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		if _, ok := parsedURL.User.Password(); !ok {
 			continue
 		}
+		if parsedURL.User.Username() == "anonymous" {
+			continue
+		}
 
-		redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))
+		rawURL, _ := url.Parse(urlMatch)
+		rawURL.Path = ""
+		redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))
 
 		s := detectors.Result{
 			DetectorType: detectorspb.DetectorType_FTP,
-			Raw:          []byte(urlMatch),
+			Raw:          []byte(rawURL.String()),
 			Redacted:     redact,
 		}
 
@@ -70,7 +75,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 		}
 
-		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+		if detectors.IsKnownFalsePositive(string(s.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {
 			continue
 		}
 

pkg/detectors/ftp/ftp_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestURI_FromChunk(t *testing.T) {
+func TestFTP_FromChunk(t *testing.T) {
 	type args struct {
 		ctx    context.Context
 		data   []byte
@@ -71,6 +71,16 @@ func TestURI_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "blocked FP",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("ftp://abc:123@ftp.freebsd.org/pub/FreeBSD/doc/tr/articles/explaining-bsd/explaining-bsd_tr.pdf"),
+				verify: true,
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/github_old/github_old.go

@@ -20,7 +20,7 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	// Oauth token
 	// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
-	keyPat = regexp.MustCompile(`(?i)(?:github)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)
+	keyPat = regexp.MustCompile(`(?i)(?:github|gh|pat)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)
 
 	// TODO: Oauth2 client_id and client_secret
 	// https://developer.github.com/v3/#oauth2-keysecret
@@ -38,7 +38,7 @@ type userRes struct {
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"github"}
+	return []string{"github","gh","pat"}
 }
 
 // FromData will find and optionally verify GitHub secrets in a given set of bytes.

pkg/detectors/ldap/ldap.go

@@ -0,0 +1,144 @@
+package ldap
+
+import (
+	"context"
+	"crypto/tls"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/go-ldap/ldap/v3"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	uriPat = regexp.MustCompile(`\b(?i)ldaps?://[\S]+\b`)
+	// ldap://127.0.0.1:389
+	// ldap://127.0.0.1
+	// ldap://mydomain.test
+	// ldaps://[fe80:4049:92ff:fe44:4bd1]:5060
+	// ldap://[fe80::4bd1]:5060
+	// ldap://ds.example.com:389/dc=example,dc=com?givenName,sn,cn?sub?(uid=john.doe)
+	usernamePat = regexp.MustCompile(detectors.PrefixRegex([]string{"user", "bind"}) + `["']([a-zA-Z=,]{4,150})["']`)
+	passwordPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pass"}) + `["']([\S]{4,48})["']`)
+
+	// https://learn.microsoft.com/en-us/windows/win32/api/iads/nf-iads-iadsopendsobject-opendsobject?redirectedfrom=MSDN
+	// I.E. Set ou = dso.OpenDSObject("LDAP://DC.business.com/OU=IT,DC=Business,DC=com", "Business\administrator", "Pa$$word01", 1)
+	iadPat = regexp.MustCompile(`OpenDSObject\(\"(?i)(ldaps?://[\S]+)\", ?\"([\S]+)\", ?\"([\S]+)\",[ \d]+\)`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"ldaps://", "ldap://"}
+}
+
+// FromData will find and optionally verify Ldap secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	// Check for matches in the URI + username + password format
+	uriMatches := uriPat.FindAllString(dataStr, -1)
+	for _, uri := range uriMatches {
+		ldapURL, err := url.Parse(uri)
+		if err != nil {
+			continue
+		}
+
+		usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
+		for _, username := range usernameMatches {
+			passwordMatches := passwordPat.FindAllStringSubmatch(dataStr, -1)
+			for _, password := range passwordMatches {
+				s1 := detectors.Result{
+					DetectorType: detectorspb.DetectorType_LDAP,
+					Raw:          []byte(strings.Join([]string{ldapURL.String(), username[1], password[1]}, "\t")),
+				}
+
+				if verify {
+					s1.Verified = verifyLDAP(ctx, username[1], password[1], ldapURL)
+				}
+
+				results = append(results, s1)
+			}
+		}
+	}
+
+	// Check for matches for the IAD library format
+	iadMatches := iadPat.FindAllStringSubmatch(dataStr, -1)
+	for _, iad := range iadMatches {
+		uri := iad[1]
+		username := iad[2]
+		password := iad[3]
+
+		ldapURL, err := url.Parse(uri)
+		if err != nil {
+			continue
+		}
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_LDAP,
+			Raw:          []byte(strings.Join([]string{ldapURL.String(), username, password}, "\t")),
+		}
+
+		if verify {
+			s1.Verified = verifyLDAP(ctx, username, password, ldapURL)
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func verifyLDAP(ctx context.Context, username, password string, ldapURL *url.URL) bool {
+	// Tests with non-TLS, TLS, and STARTTLS
+
+	ldap.DefaultTimeout = 5 * time.Second
+
+	uri := ldapURL.String()
+
+	switch ldapURL.Scheme {
+	case "ldap":
+		// Non-TLS dial
+		l, err := ldap.DialURL(uri)
+		if err == nil {
+			defer l.Close()
+			// Non-TLS verify
+			err = l.Bind(username, password)
+			if err == nil {
+				return true
+			}
+
+			// STARTTLS
+			err = l.StartTLS(&tls.Config{InsecureSkipVerify: true})
+			if err == nil {
+				// STARTTLS verify
+				err = l.Bind(username, password)
+				if err == nil {
+					return true
+				}
+			}
+		}
+	case "ldaps":
+		// TLS dial
+		l, err := ldap.DialTLS("tcp", uri, &tls.Config{InsecureSkipVerify: true})
+		if err == nil {
+			defer l.Close()
+			// TLS verify
+			err = l.Bind(username, password)
+			if err == nil {
+				return true
+			}
+		}
+	}
+
+	return false
+}

pkg/detectors/ldap/ldap_integration_test.go

@@ -0,0 +1,214 @@
+//go:build detectors && integration
+// +build detectors,integration
+
+package ldap
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestMain(m *testing.M) {
+	code, err := runMain(m)
+	if err != nil {
+		panic(err)
+	}
+	os.Exit(code)
+}
+
+func runMain(m *testing.M) (int, error) {
+	if err := startOpenLDAP(); err != nil {
+		return 0, err
+	}
+	defer stopOpenLDAP()
+	return m.Run(), nil
+}
+
+func dockerLogLine(hash string, needle string) chan struct{} {
+	ch := make(chan struct{}, 1)
+	go func() {
+		for {
+			out, err := exec.Command("docker", "logs", hash).CombinedOutput()
+			if err != nil {
+				panic(err)
+			}
+			if strings.Contains(string(out), needle) {
+				ch <- struct{}{}
+				return
+			}
+			time.Sleep(1 * time.Second)
+		}
+	}()
+	return ch
+}
+
+func TestLdap_Integration_FromChunk(t *testing.T) {
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found with URI and separate user+password usage, verified",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				data: []byte(`
+		ldap://localhost:1389
+		binddn="cn=admin,dc=example,dc=org"
+		pass="P@55w0rd"`),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_LDAP,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found with URI and separate user+password usage, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx: context.Background(),
+				data: []byte(`
+		ldap://localhost:1389
+		binddn="cn=someuser,dc=example,dc=org"
+		pass="P@55w0rd"`),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_LDAP,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found with IAD lib usage, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(`Set ou = dso.OpenDSObject("LDAP://localhost:1389", "cn=admin,dc=example,dc=org", "P@55w0rd", 1)`),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_LDAP,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found with IAD lib usage, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(`Set ou = dso.OpenDSObject("LDAP://localhost:1389", "cn=admin,dc=example,dc=org", "P@55w0rd", 1)`),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_LDAP,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found with IAD lib usage, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(`Set ou = dso.OpenDSObject("LDAP://localhost:1389", "cn=admin,dc=example,dc=org", "invalid", 1)`),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_LDAP,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Ldap.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Ldap.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+var containerID string
+
+func startOpenLDAP() error {
+	cmd := exec.Command(
+		"docker", "run", "--rm", "-p", "1389:1389",
+		"-e", "LDAP_ROOT=dc=example,dc=org",
+		"-e", "LDAP_ADMIN_USERNAME=admin",
+		"-e", "LDAP_ADMIN_PASSWORD=P@55w0rd",
+		"-d", "bitnami/openldap:latest",
+	)
+	out, err := cmd.Output()
+	if err != nil {
+		return err
+	}
+	containerID = string(bytes.TrimSpace(out))
+	select {
+	case <-dockerLogLine(containerID, "slapd starting"):
+		return nil
+	case <-time.After(30 * time.Second):
+		stopOpenLDAP()
+		return errors.New("timeout waiting for postgres database to be ready")
+	}
+}
+
+func stopOpenLDAP() {
+	exec.Command("docker", "kill", containerID).Run()
+}

pkg/detectors/ldap/ldap_test.go

@@ -0,0 +1,91 @@
+//go:build detectors
+// +build detectors
+
+package ldap
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestLdap_FromChunk(t *testing.T) {
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found with IAD lib usage, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(`Set ou = dso.OpenDSObject("LDAP://DC.business.com/OU=IT,DC=Business,DC=com", "Business\administrator", "Pa$$word01", 1)`),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_LDAP,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Ldap.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Ldap.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/privatekey/fingerprint.go

@@ -35,7 +35,7 @@ func FingerprintPEMKey(in []byte) (string, error) {
 		pubKey = &privateKey.PublicKey
 	case *ecdsa.PrivateKey:
 		pubKey = &privateKey.PublicKey
-	case ed25519.PrivateKey:
+	case *ed25519.PrivateKey:
 		pubKey = privateKey.Public()
 	// No fingerprinting support for DSA
 	// case *dsa.PrivateKey:

pkg/detectors/rabbitmq/rabbitmq.go

@@ -0,0 +1,83 @@
+package rabbitmq
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+
+	amqp "github.com/rabbitmq/amqp091-go"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	keyPat = regexp.MustCompile(`\b(?:amqp:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"amqp"}
+}
+
+// FromData will find and optionally verify RabbitMQ secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		urlMatch := match[0]
+		password := match[1]
+
+		// Skip findings where the password only has "*" characters, this is a redacted password
+		if strings.Trim(password, "*") == "" {
+			continue
+		}
+
+		parsedURL, err := url.Parse(urlMatch)
+		if err != nil {
+			continue
+		}
+		if _, ok := parsedURL.User.Password(); !ok {
+			continue
+		}
+
+		redact := strings.TrimSpace(strings.Replace(parsedURL.String(), password, "********", -1))
+
+		s := detectors.Result{
+			DetectorType: detectorspb.DetectorType_RabbitMQ,
+			Raw:          []byte(urlMatch),
+			Redacted:     redact,
+		}
+
+		if verify {
+			_, err := amqp.Dial(urlMatch)
+			if err == nil {
+				s.Verified = true
+			}
+		}
+
+		if !s.Verified {
+			// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
+			if strings.HasPrefix(password, "$") {
+				continue
+			}
+		}
+
+		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+			continue
+		}
+
+		results = append(results, s)
+	}
+
+	return results, nil
+}

pkg/detectors/rabbitmq/rabbitmq_test.go

@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package rabbitmq
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestRabbitMQ_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("RABBITMQ")
+	inactiveSecret := testSecrets.MustGetField("RABBITMQ_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a rabbitmq secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_RabbitMQ,
+					Verified:     true,
+					Redacted:     "amqp://rabbit123:*******************@127.0.0.1",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a rabbitmq secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_RabbitMQ,
+					Verified:     false,
+					Redacted:     "amqp://rabbituser:*******************@127.0.0.1",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("RabbitMQ.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("RabbitMQ.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/redis/redis.go

@@ -50,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			continue
 		}
 
-		redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))
+		redact := strings.TrimSpace(strings.Replace(urlMatch, password, "********", -1))
 
 		s := detectors.Result{
 			DetectorType: detectorspb.DetectorType_Redis,

pkg/detectors/slackwebhook/slackwebhook.go

@@ -21,7 +21,7 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`(https:\/\/hooks\.slack\.com\/services\/[A-Za-z0-9+\/]{44,46})`)
+	keyPat = regexp.MustCompile(`(https://hooks\.slack\.com/services/[A-Za-z0-9+/]{44,46})`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				body := string(bodyBytes)
-				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "missing_text")) {
+				if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "text")) {
 					s1.Verified = true
 				}
 			}

pkg/detectors/uri/uri.go

@@ -71,8 +71,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 		rawURL, _ := url.Parse(urlMatch)
 		rawURL.Path = ""
-
-		redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, strings.Repeat("*", len(password)), -1))
+		redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))
 
 		s := detectors.Result{
 			DetectorType: detectorspb.DetectorType_URI,

pkg/engine/defaults.go

@@ -350,6 +350,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/languagelayer"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lastfm"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/launchdarkly"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ldap"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/leadfeeder"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lemlist"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lendflow"
@@ -737,7 +738,7 @@ func DefaultDetectors() []detectors.Detector {
 		&heroku.Scanner{},
 		&linearapi.Scanner{},
 		&alibaba.Scanner{},
-		&aws.Scanner{},
+		aws.New(),
 		&azure.Scanner{},
 		&slack.Scanner{}, // has 4 secret types
 		&gitlabv2.Scanner{},
@@ -1489,6 +1490,7 @@ func DefaultDetectors() []detectors.Detector {
 		sqlserver.Scanner{},
 		redis.Scanner{},
 		ftp.Scanner{},
+		ldap.Scanner{},
 		shopify.Scanner{},
 	}
 }

pkg/engine/git.go

@@ -25,7 +25,7 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
 
 	repo, err := gogit.PlainOpenWithOptions(c.RepoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
 	if err != nil {
-		return fmt.Errorf("could open repo: %s: %w", c.RepoPath, err)
+		return fmt.Errorf("could not open repo: %s: %w", c.RepoPath, err)
 	}
 
 	if c.MaxDepth != 0 {

pkg/engine/gitlab.go

@@ -5,6 +5,7 @@ import (
 	"runtime"
 
 	"github.com/go-errors/errors"
+	gogit "github.com/go-git/go-git/v5"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -13,11 +14,19 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
 )
 
 // ScanGitLab scans GitLab with the provided configuration.
 func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
+	logOptions := &gogit.LogOptions{}
+	opts := []git.ScanOption{
+		git.ScanOptionFilter(c.Filter),
+		git.ScanOptionLogOptions(logOptions),
+	}
+	scanOptions := git.NewScanOptions(opts...)
+
 	connection := &sourcespb.GitLab{}
 
 	switch {
@@ -49,6 +58,7 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init GitLab source", 0)
 	}
+	gitlabSource.WithScanOptions(scanOptions)
 
 	e.sourcesWg.Add(1)
 	go func() {

pkg/handlers/archive.go

@@ -136,14 +136,20 @@ func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Conte
 }
 
 // ReadToMax reads up to the max size.
-func (d *Archive) ReadToMax(reader io.Reader) ([]byte, error) {
+func (d *Archive) ReadToMax(reader io.Reader) (data []byte, err error) {
 	// Archiver v4 is in alpha and using an experimental version of
 	// rardecode. There is a bug somewhere with rar decoder format 29
 	// that can lead to a panic. An issue is open in rardecode repo
 	// https://github.com/nwaples/rardecode/issues/30.
 	defer func() {
-		if err := recover(); err != nil {
-			log.Errorf("Panic occurred when reading archive: %v", err)
+		if r := recover(); r != nil {
+			log.Errorf("Panic occurred when reading archive: %v", r)
+			// Return an error from ReadToMax.
+			if e, ok := r.(error); ok {
+				err = e
+			} else {
+				err = fmt.Errorf("Panic occurred: %v", r)
+			}
 		}
 	}()
 	fileContent := bytes.Buffer{}

pkg/pb/custom_detectorspb/custom_detectors.pb.go

@@ -0,0 +1,348 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.26.0
+// 	protoc        v3.20.0
+// source: custom_detectors.proto
+
+package custom_detectorspb
+
+import (
+	_ "github.com/envoyproxy/protoc-gen-validate/validate"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type CustomDetectors struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Detectors []*CustomRegex `protobuf:"bytes,1,rep,name=detectors,proto3" json:"detectors,omitempty"`
+}
+
+func (x *CustomDetectors) Reset() {
+	*x = CustomDetectors{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_custom_detectors_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *CustomDetectors) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CustomDetectors) ProtoMessage() {}
+
+func (x *CustomDetectors) ProtoReflect() protoreflect.Message {
+	mi := &file_custom_detectors_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CustomDetectors.ProtoReflect.Descriptor instead.
+func (*CustomDetectors) Descriptor() ([]byte, []int) {
+	return file_custom_detectors_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *CustomDetectors) GetDetectors() []*CustomRegex {
+	if x != nil {
+		return x.Detectors
+	}
+	return nil
+}
+
+type CustomRegex struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Name     string            `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Keywords []string          `protobuf:"bytes,2,rep,name=keywords,proto3" json:"keywords,omitempty"`
+	Regex    map[string]string `protobuf:"bytes,3,rep,name=regex,proto3" json:"regex,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	Verify   []*VerifierConfig `protobuf:"bytes,4,rep,name=verify,proto3" json:"verify,omitempty"`
+}
+
+func (x *CustomRegex) Reset() {
+	*x = CustomRegex{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_custom_detectors_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *CustomRegex) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*CustomRegex) ProtoMessage() {}
+
+func (x *CustomRegex) ProtoReflect() protoreflect.Message {
+	mi := &file_custom_detectors_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use CustomRegex.ProtoReflect.Descriptor instead.
+func (*CustomRegex) Descriptor() ([]byte, []int) {
+	return file_custom_detectors_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *CustomRegex) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *CustomRegex) GetKeywords() []string {
+	if x != nil {
+		return x.Keywords
+	}
+	return nil
+}
+
+func (x *CustomRegex) GetRegex() map[string]string {
+	if x != nil {
+		return x.Regex
+	}
+	return nil
+}
+
+func (x *CustomRegex) GetVerify() []*VerifierConfig {
+	if x != nil {
+		return x.Verify
+	}
+	return nil
+}
+
+type VerifierConfig struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Endpoint      string   `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"`
+	Unsafe        bool     `protobuf:"varint,2,opt,name=unsafe,proto3" json:"unsafe,omitempty"`
+	Headers       []string `protobuf:"bytes,3,rep,name=headers,proto3" json:"headers,omitempty"`
+	SuccessRanges []string `protobuf:"bytes,4,rep,name=successRanges,proto3" json:"successRanges,omitempty"`
+}
+
+func (x *VerifierConfig) Reset() {
+	*x = VerifierConfig{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_custom_detectors_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *VerifierConfig) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*VerifierConfig) ProtoMessage() {}
+
+func (x *VerifierConfig) ProtoReflect() protoreflect.Message {
+	mi := &file_custom_detectors_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use VerifierConfig.ProtoReflect.Descriptor instead.
+func (*VerifierConfig) Descriptor() ([]byte, []int) {
+	return file_custom_detectors_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *VerifierConfig) GetEndpoint() string {
+	if x != nil {
+		return x.Endpoint
+	}
+	return ""
+}
+
+func (x *VerifierConfig) GetUnsafe() bool {
+	if x != nil {
+		return x.Unsafe
+	}
+	return false
+}
+
+func (x *VerifierConfig) GetHeaders() []string {
+	if x != nil {
+		return x.Headers
+	}
+	return nil
+}
+
+func (x *VerifierConfig) GetSuccessRanges() []string {
+	if x != nil {
+		return x.SuccessRanges
+	}
+	return nil
+}
+
+var File_custom_detectors_proto protoreflect.FileDescriptor
+
+var file_custom_detectors_proto_rawDesc = []byte{
+	0x0a, 0x16, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f,
+	0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x10, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d,
+	0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x1a, 0x17, 0x76, 0x61, 0x6c, 0x69,
+	0x64, 0x61, 0x74, 0x65, 0x2f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x22, 0x4e, 0x0a, 0x0f, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x44, 0x65, 0x74,
+	0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x12, 0x3b, 0x0a, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
+	0x6f, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x63, 0x75, 0x73, 0x74,
+	0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73,
+	0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65, 0x78, 0x52, 0x09, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
+	0x6f, 0x72, 0x73, 0x22, 0xf1, 0x01, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65,
+	0x67, 0x65, 0x78, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+	0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f,
+	0x72, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x77, 0x6f,
+	0x72, 0x64, 0x73, 0x12, 0x3e, 0x0a, 0x05, 0x72, 0x65, 0x67, 0x65, 0x78, 0x18, 0x03, 0x20, 0x03,
+	0x28, 0x0b, 0x32, 0x28, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65,
+	0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67, 0x65,
+	0x78, 0x2e, 0x52, 0x65, 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x72, 0x65,
+	0x67, 0x65, 0x78, 0x12, 0x38, 0x0a, 0x06, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x18, 0x04, 0x20,
+	0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74,
+	0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x2e, 0x56, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, 0x72, 0x43,
+	0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x76, 0x65, 0x72, 0x69, 0x66, 0x79, 0x1a, 0x38, 0x0a,
+	0x0a, 0x52, 0x65, 0x67, 0x65, 0x78, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b,
+	0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a,
+	0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61,
+	0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8e, 0x01, 0x0a, 0x0e, 0x56, 0x65, 0x72, 0x69,
+	0x66, 0x69, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e,
+	0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42,
+	0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74,
+	0x12, 0x16, 0x0a, 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08,
+	0x52, 0x06, 0x75, 0x6e, 0x73, 0x61, 0x66, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64,
+	0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65,
+	0x72, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x52, 0x61, 0x6e,
+	0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x73, 0x75, 0x63, 0x63, 0x65,
+	0x73, 0x73, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x42, 0x44, 0x5a, 0x42, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
+	0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
+	0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x63, 0x75, 0x73, 0x74,
+	0x6f, 0x6d, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_custom_detectors_proto_rawDescOnce sync.Once
+	file_custom_detectors_proto_rawDescData = file_custom_detectors_proto_rawDesc
+)
+
+func file_custom_detectors_proto_rawDescGZIP() []byte {
+	file_custom_detectors_proto_rawDescOnce.Do(func() {
+		file_custom_detectors_proto_rawDescData = protoimpl.X.CompressGZIP(file_custom_detectors_proto_rawDescData)
+	})
+	return file_custom_detectors_proto_rawDescData
+}
+
+var file_custom_detectors_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_custom_detectors_proto_goTypes = []interface{}{
+	(*CustomDetectors)(nil), // 0: custom_detectors.CustomDetectors
+	(*CustomRegex)(nil),     // 1: custom_detectors.CustomRegex
+	(*VerifierConfig)(nil),  // 2: custom_detectors.VerifierConfig
+	nil,                     // 3: custom_detectors.CustomRegex.RegexEntry
+}
+var file_custom_detectors_proto_depIdxs = []int32{
+	1, // 0: custom_detectors.CustomDetectors.detectors:type_name -> custom_detectors.CustomRegex
+	3, // 1: custom_detectors.CustomRegex.regex:type_name -> custom_detectors.CustomRegex.RegexEntry
+	2, // 2: custom_detectors.CustomRegex.verify:type_name -> custom_detectors.VerifierConfig
+	3, // [3:3] is the sub-list for method output_type
+	3, // [3:3] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_custom_detectors_proto_init() }
+func file_custom_detectors_proto_init() {
+	if File_custom_detectors_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_custom_detectors_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CustomDetectors); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_custom_detectors_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*CustomRegex); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_custom_detectors_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*VerifierConfig); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_custom_detectors_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_custom_detectors_proto_goTypes,
+		DependencyIndexes: file_custom_detectors_proto_depIdxs,
+		MessageInfos:      file_custom_detectors_proto_msgTypes,
+	}.Build()
+	File_custom_detectors_proto = out.File
+	file_custom_detectors_proto_rawDesc = nil
+	file_custom_detectors_proto_goTypes = nil
+	file_custom_detectors_proto_depIdxs = nil
+}

pkg/pb/custom_detectorspb/custom_detectors.pb.validate.go

@@ -0,0 +1,421 @@
+// Code generated by protoc-gen-validate. DO NOT EDIT.
+// source: custom_detectors.proto
+
+package custom_detectorspb
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net"
+	"net/mail"
+	"net/url"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"google.golang.org/protobuf/types/known/anypb"
+)
+
+// ensure the imports are used
+var (
+	_ = bytes.MinRead
+	_ = errors.New("")
+	_ = fmt.Print
+	_ = utf8.UTFMax
+	_ = (*regexp.Regexp)(nil)
+	_ = (*strings.Reader)(nil)
+	_ = net.IPv4len
+	_ = time.Duration(0)
+	_ = (*url.URL)(nil)
+	_ = (*mail.Address)(nil)
+	_ = anypb.Any{}
+	_ = sort.Sort
+)
+
+// Validate checks the field values on CustomDetectors with the rules defined
+// in the proto definition for this message. If any rules are violated, the
+// first error encountered is returned, or nil if there are no violations.
+func (m *CustomDetectors) Validate() error {
+	return m.validate(false)
+}
+
+// ValidateAll checks the field values on CustomDetectors with the rules
+// defined in the proto definition for this message. If any rules are
+// violated, the result is a list of violation errors wrapped in
+// CustomDetectorsMultiError, or nil if none found.
+func (m *CustomDetectors) ValidateAll() error {
+	return m.validate(true)
+}
+
+func (m *CustomDetectors) validate(all bool) error {
+	if m == nil {
+		return nil
+	}
+
+	var errors []error
+
+	for idx, item := range m.GetDetectors() {
+		_, _ = idx, item
+
+		if all {
+			switch v := interface{}(item).(type) {
+			case interface{ ValidateAll() error }:
+				if err := v.ValidateAll(); err != nil {
+					errors = append(errors, CustomDetectorsValidationError{
+						field:  fmt.Sprintf("Detectors[%v]", idx),
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			case interface{ Validate() error }:
+				if err := v.Validate(); err != nil {
+					errors = append(errors, CustomDetectorsValidationError{
+						field:  fmt.Sprintf("Detectors[%v]", idx),
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			}
+		} else if v, ok := interface{}(item).(interface{ Validate() error }); ok {
+			if err := v.Validate(); err != nil {
+				return CustomDetectorsValidationError{
+					field:  fmt.Sprintf("Detectors[%v]", idx),
+					reason: "embedded message failed validation",
+					cause:  err,
+				}
+			}
+		}
+
+	}
+
+	if len(errors) > 0 {
+		return CustomDetectorsMultiError(errors)
+	}
+
+	return nil
+}
+
+// CustomDetectorsMultiError is an error wrapping multiple validation errors
+// returned by CustomDetectors.ValidateAll() if the designated constraints
+// aren't met.
+type CustomDetectorsMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m CustomDetectorsMultiError) Error() string {
+	var msgs []string
+	for _, err := range m {
+		msgs = append(msgs, err.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m CustomDetectorsMultiError) AllErrors() []error { return m }
+
+// CustomDetectorsValidationError is the validation error returned by
+// CustomDetectors.Validate if the designated constraints aren't met.
+type CustomDetectorsValidationError struct {
+	field  string
+	reason string
+	cause  error
+	key    bool
+}
+
+// Field function returns field value.
+func (e CustomDetectorsValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e CustomDetectorsValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e CustomDetectorsValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e CustomDetectorsValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e CustomDetectorsValidationError) ErrorName() string { return "CustomDetectorsValidationError" }
+
+// Error satisfies the builtin error interface
+func (e CustomDetectorsValidationError) Error() string {
+	cause := ""
+	if e.cause != nil {
+		cause = fmt.Sprintf(" | caused by: %v", e.cause)
+	}
+
+	key := ""
+	if e.key {
+		key = "key for "
+	}
+
+	return fmt.Sprintf(
+		"invalid %sCustomDetectors.%s: %s%s",
+		key,
+		e.field,
+		e.reason,
+		cause)
+}
+
+var _ error = CustomDetectorsValidationError{}
+
+var _ interface {
+	Field() string
+	Reason() string
+	Key() bool
+	Cause() error
+	ErrorName() string
+} = CustomDetectorsValidationError{}
+
+// Validate checks the field values on CustomRegex with the rules defined in
+// the proto definition for this message. If any rules are violated, the first
+// error encountered is returned, or nil if there are no violations.
+func (m *CustomRegex) Validate() error {
+	return m.validate(false)
+}
+
+// ValidateAll checks the field values on CustomRegex with the rules defined in
+// the proto definition for this message. If any rules are violated, the
+// result is a list of violation errors wrapped in CustomRegexMultiError, or
+// nil if none found.
+func (m *CustomRegex) ValidateAll() error {
+	return m.validate(true)
+}
+
+func (m *CustomRegex) validate(all bool) error {
+	if m == nil {
+		return nil
+	}
+
+	var errors []error
+
+	// no validation rules for Name
+
+	// no validation rules for Regex
+
+	for idx, item := range m.GetVerify() {
+		_, _ = idx, item
+
+		if all {
+			switch v := interface{}(item).(type) {
+			case interface{ ValidateAll() error }:
+				if err := v.ValidateAll(); err != nil {
+					errors = append(errors, CustomRegexValidationError{
+						field:  fmt.Sprintf("Verify[%v]", idx),
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			case interface{ Validate() error }:
+				if err := v.Validate(); err != nil {
+					errors = append(errors, CustomRegexValidationError{
+						field:  fmt.Sprintf("Verify[%v]", idx),
+						reason: "embedded message failed validation",
+						cause:  err,
+					})
+				}
+			}
+		} else if v, ok := interface{}(item).(interface{ Validate() error }); ok {
+			if err := v.Validate(); err != nil {
+				return CustomRegexValidationError{
+					field:  fmt.Sprintf("Verify[%v]", idx),
+					reason: "embedded message failed validation",
+					cause:  err,
+				}
+			}
+		}
+
+	}
+
+	if len(errors) > 0 {
+		return CustomRegexMultiError(errors)
+	}
+
+	return nil
+}
+
+// CustomRegexMultiError is an error wrapping multiple validation errors
+// returned by CustomRegex.ValidateAll() if the designated constraints aren't met.
+type CustomRegexMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m CustomRegexMultiError) Error() string {
+	var msgs []string
+	for _, err := range m {
+		msgs = append(msgs, err.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m CustomRegexMultiError) AllErrors() []error { return m }
+
+// CustomRegexValidationError is the validation error returned by
+// CustomRegex.Validate if the designated constraints aren't met.
+type CustomRegexValidationError struct {
+	field  string
+	reason string
+	cause  error
+	key    bool
+}
+
+// Field function returns field value.
+func (e CustomRegexValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e CustomRegexValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e CustomRegexValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e CustomRegexValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e CustomRegexValidationError) ErrorName() string { return "CustomRegexValidationError" }
+
+// Error satisfies the builtin error interface
+func (e CustomRegexValidationError) Error() string {
+	cause := ""
+	if e.cause != nil {
+		cause = fmt.Sprintf(" | caused by: %v", e.cause)
+	}
+
+	key := ""
+	if e.key {
+		key = "key for "
+	}
+
+	return fmt.Sprintf(
+		"invalid %sCustomRegex.%s: %s%s",
+		key,
+		e.field,
+		e.reason,
+		cause)
+}
+
+var _ error = CustomRegexValidationError{}
+
+var _ interface {
+	Field() string
+	Reason() string
+	Key() bool
+	Cause() error
+	ErrorName() string
+} = CustomRegexValidationError{}
+
+// Validate checks the field values on VerifierConfig with the rules defined in
+// the proto definition for this message. If any rules are violated, the first
+// error encountered is returned, or nil if there are no violations.
+func (m *VerifierConfig) Validate() error {
+	return m.validate(false)
+}
+
+// ValidateAll checks the field values on VerifierConfig with the rules defined
+// in the proto definition for this message. If any rules are violated, the
+// result is a list of violation errors wrapped in VerifierConfigMultiError,
+// or nil if none found.
+func (m *VerifierConfig) ValidateAll() error {
+	return m.validate(true)
+}
+
+func (m *VerifierConfig) validate(all bool) error {
+	if m == nil {
+		return nil
+	}
+
+	var errors []error
+
+	if _, err := url.Parse(m.GetEndpoint()); err != nil {
+		err = VerifierConfigValidationError{
+			field:  "Endpoint",
+			reason: "value must be a valid URI",
+			cause:  err,
+		}
+		if !all {
+			return err
+		}
+		errors = append(errors, err)
+	}
+
+	// no validation rules for Unsafe
+
+	if len(errors) > 0 {
+		return VerifierConfigMultiError(errors)
+	}
+
+	return nil
+}
+
+// VerifierConfigMultiError is an error wrapping multiple validation errors
+// returned by VerifierConfig.ValidateAll() if the designated constraints
+// aren't met.
+type VerifierConfigMultiError []error
+
+// Error returns a concatenation of all the error messages it wraps.
+func (m VerifierConfigMultiError) Error() string {
+	var msgs []string
+	for _, err := range m {
+		msgs = append(msgs, err.Error())
+	}
+	return strings.Join(msgs, "; ")
+}
+
+// AllErrors returns a list of validation violation errors.
+func (m VerifierConfigMultiError) AllErrors() []error { return m }
+
+// VerifierConfigValidationError is the validation error returned by
+// VerifierConfig.Validate if the designated constraints aren't met.
+type VerifierConfigValidationError struct {
+	field  string
+	reason string
+	cause  error
+	key    bool
+}
+
+// Field function returns field value.
+func (e VerifierConfigValidationError) Field() string { return e.field }
+
+// Reason function returns reason value.
+func (e VerifierConfigValidationError) Reason() string { return e.reason }
+
+// Cause function returns cause value.
+func (e VerifierConfigValidationError) Cause() error { return e.cause }
+
+// Key function returns key value.
+func (e VerifierConfigValidationError) Key() bool { return e.key }
+
+// ErrorName returns error name.
+func (e VerifierConfigValidationError) ErrorName() string { return "VerifierConfigValidationError" }
+
+// Error satisfies the builtin error interface
+func (e VerifierConfigValidationError) Error() string {
+	cause := ""
+	if e.cause != nil {
+		cause = fmt.Sprintf(" | caused by: %v", e.cause)
+	}
+
+	key := ""
+	if e.key {
+		key = "key for "
+	}
+
+	return fmt.Sprintf(
+		"invalid %sVerifierConfig.%s: %s%s",
+		key,
+		e.field,
+		e.reason,
+		cause)
+}
+
+var _ error = VerifierConfigValidationError{}
+
+var _ interface {
+	Field() string
+	Reason() string
+	Key() bool
+	Cause() error
+	ErrorName() string
+} = VerifierConfigValidationError{}

pkg/pb/detectorspb/detectors.pb.go

@@ -969,7 +969,9 @@ const (
 	DetectorType_SQLServer                     DetectorType = 898
 	DetectorType_FTP                           DetectorType = 899
 	DetectorType_Redis                         DetectorType = 900
+	DetectorType_LDAP                          DetectorType = 901
 	DetectorType_Shopify                       DetectorType = 902
+	DetectorType_RabbitMQ                      DetectorType = 903
 )
 
 // Enum value maps for DetectorType.
@@ -1872,7 +1874,9 @@ var (
 		898: "SQLServer",
 		899: "FTP",
 		900: "Redis",
+		901: "LDAP",
 		902: "Shopify",
+		903: "RabbitMQ",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2772,7 +2776,9 @@ var (
 		"SQLServer":                     898,
 		"FTP":                           899,
 		"Redis":                         900,
+		"LDAP":                          901,
 		"Shopify":                       902,
+		"RabbitMQ":                      903,
 	}
 )
 
@@ -3137,7 +3143,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
 	0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
 	0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
-	0x02, 0x2a, 0xe5, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
+	0x02, 0x2a, 0xff, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
 	0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12,
 	0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53,
 	0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a,
@@ -4038,12 +4044,14 @@ var file_detectors_proto_rawDesc = []byte{
 	0x12, 0x13, 0x0a, 0x0e, 0x44, 0x69, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x4f, 0x63, 0x65, 0x61, 0x6e,
 	0x56, 0x32, 0x10, 0x81, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x51, 0x4c, 0x53, 0x65, 0x72, 0x76,
 	0x65, 0x72, 0x10, 0x82, 0x07, 0x12, 0x08, 0x0a, 0x03, 0x46, 0x54, 0x50, 0x10, 0x83, 0x07, 0x12,
-	0x0a, 0x0a, 0x05, 0x52, 0x65, 0x64, 0x69, 0x73, 0x10, 0x84, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x53,
-	0x68, 0x6f, 0x70, 0x69, 0x66, 0x79, 0x10, 0x86, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74,
-	0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
-	0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
-	0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
-	0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x0a, 0x0a, 0x05, 0x52, 0x65, 0x64, 0x69, 0x73, 0x10, 0x84, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x4c,
+	0x44, 0x41, 0x50, 0x10, 0x85, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66,
+	0x79, 0x10, 0x86, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x52, 0x61, 0x62, 0x62, 0x69, 0x74, 0x4d, 0x51,
+	0x10, 0x87, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
+	0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
+	0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
+	0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
+	0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/protoyaml/protoyaml.go

@@ -0,0 +1,39 @@
+package protoyaml
+
+import (
+	"google.golang.org/protobuf/encoding/protojson"
+	"google.golang.org/protobuf/proto"
+	"sigs.k8s.io/yaml"
+)
+
+var nonStrict = protojson.UnmarshalOptions{DiscardUnknown: true}
+var strict = protojson.UnmarshalOptions{DiscardUnknown: false}
+
+// Marshal writes the given proto.Message in YAML format.
+func Marshal(m proto.Message) ([]byte, error) {
+	json, err := protojson.Marshal(m)
+	if err != nil {
+		return nil, err
+	}
+	return yaml.JSONToYAML(json)
+}
+
+// Unmarshal reads the given []byte into the given proto.Message, discarding
+// any unknown fields in the input.
+func Unmarshal(b []byte, m proto.Message) error {
+	json, err := yaml.YAMLToJSON(b)
+	if err != nil {
+		return err
+	}
+	return nonStrict.Unmarshal(json, m)
+}
+
+// UnmarshalStrict reads the given []byte into the given proto.Message. If there
+// are any unknown fields in the input, an error is returned.
+func UnmarshalStrict(b []byte, m proto.Message) error {
+	json, err := yaml.YAMLToJSON(b)
+	if err != nil {
+		return err
+	}
+	return strict.Unmarshal(json, m)
+}

pkg/sources/git/git.go

@@ -540,6 +540,9 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 }
 
 func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+	if scanOptions == nil {
+		scanOptions = NewScanOptions()
+	}
 	if err := normalizeConfig(scanOptions, repo); err != nil {
 		return err
 	}

pkg/sources/gitlab/gitlab.go

@@ -41,6 +41,7 @@ type Source struct {
 	repos           []string
 	ignoreRepos     []string
 	git             *git.Git
+	scanOptions     *git.ScanOptions
 	aCtx            context.Context
 	resumeInfoSlice []string
 	resumeInfoMutex sync.Mutex
@@ -325,7 +326,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				return
 			}
 			log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repoURL)
-			err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
+			err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan)
 			if err != nil {
 				errsMut.Lock()
 				errs = append(errs, err)
@@ -432,3 +433,7 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 	// Add the offset to both the index and the repos to give the proper place and proper repo count.
 	s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
 }
+
+func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
+	s.scanOptions = scanOptions
+}

pkg/sources/s3/s3.go

@@ -14,7 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
 	"github.com/go-errors/errors"
-	log "github.com/sirupsen/logrus"
+	"github.com/go-logr/logr"
 	"golang.org/x/sync/semaphore"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
@@ -35,7 +35,7 @@ type Source struct {
 	verify      bool
 	concurrency int
 	aCtx        context.Context
-	log         *log.Entry
+	log         logr.Logger
 	sources.Progress
 	errorCount *sync.Map
 	conn       *sourcespb.S3
@@ -59,7 +59,7 @@ func (s *Source) JobID() int64 {
 
 // Init returns an initialized AWS source
 func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
-	s.log = log.WithField("source", s.Type()).WithField("name", name)
+	s.log = context.WithValues(aCtx, "source", s.Type(), "name", name).Logger()
 
 	s.aCtx = aCtx
 	s.name = name
@@ -120,7 +120,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		if len(s.conn.Buckets) == 0 {
 			res, err := client.ListBuckets(&s3.ListBucketsInput{})
 			if err != nil {
-				s.log.Errorf("could not list s3 buckets: %s", err)
+				s.log.Error(err, "could not list s3 buckets")
 				return errors.WrapPrefix(err, "could not list s3 buckets", 0)
 			}
 			buckets := res.Buckets
@@ -143,23 +143,22 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 
 		s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
 
-		s.log.Debugf("Scanning bucket: %s", bucket)
+		s.log.Info("Scanning bucket", "bucket", bucket)
 		region, err := s3manager.GetBucketRegionWithClient(context.Background(), client, bucket)
 		if err != nil {
-			s.log.WithError(err).Errorf("could not get s3 region for bucket: %s", bucket)
+			s.log.Error(err, "could not get s3 region for bucket", "bucket", bucket)
 			continue
 		}
 		var regionalClient *s3.S3
 		if region != "us-east-1" {
 			regionalClient, err = s.newClient(region)
 			if err != nil {
-				s.log.WithError(err).Error("could not make regional s3 client")
+				s.log.Error(err, "could not make regional s3 client")
 			}
 		} else {
 			regionalClient = client
 		}
-		// Forced prefix for testing
-		// pf := "public"
+
 		errorCount := sync.Map{}
 
 		err = regionalClient.ListObjectsV2PagesWithContext(
@@ -170,7 +169,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			})
 
 		if err != nil {
-			s.log.WithError(err).Errorf("could not list objects in s3 bucket: %s", bucket)
+			s.log.Error(err, "could not list objects in s3 bucket", "bucket", bucket)
 			return errors.WrapPrefix(err, fmt.Sprintf("could not list objects in s3 bucket: %s", bucket), 0)
 		}
 	}
@@ -188,9 +187,37 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 			return
 		}
 
+		if obj == nil {
+			continue
+		}
+
+		// skip GLACIER and GLACIER_IR objects
+		if obj.StorageClass == nil || strings.Contains(*obj.StorageClass, "GLACIER") {
+			s.log.V(5).Info("Skipping object in storage class", "storage_class", *obj.StorageClass, "object", *obj.Key)
+			continue
+		}
+
+		// ignore large files
+		if *obj.Size > int64(250*common.MB) {
+			s.log.V(3).Info("Skipping %d byte file (over 250MB limit)", "object", *obj.Key)
+			return
+		}
+
+		// file empty file
+		if *obj.Size == 0 {
+			s.log.V(5).Info("Skipping 0 byte file", "object", *obj.Key)
+			return
+		}
+
+		// skip incompatible extensions
+		if common.SkipFile(*obj.Key) {
+			s.log.V(5).Info("Skipping file with incompatible extension", "object", *obj.Key)
+			return
+		}
+
 		err := sem.Acquire(ctx, 1)
 		if err != nil {
-			log.WithError(err).Error("could not acquire semaphore")
+			s.log.Error(err, "could not acquire semaphore")
 			continue
 		}
 		wg.Add(1)
@@ -198,12 +225,10 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 			defer common.RecoverWithExit(ctx)
 			defer sem.Release(1)
 			defer wg.Done()
-			// defer log.Debugf("DONE - %s", *obj.Key)
 
 			if (*obj.Key)[len(*obj.Key)-1:] == "/" {
 				return
 			}
-			// log.Debugf("Object: %s", *obj.Key)
 
 			path := strings.Split(*obj.Key, "/")
 			prefix := strings.Join(path[:len(path)-1], "/")
@@ -213,17 +238,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				nErr = 0
 			}
 			if nErr.(int) > 3 {
-				log.Debugf("Skipped: %s", *obj.Key)
-				return
-			}
-
-			// ignore large files
-			if *obj.Size > int64(10*common.MB) {
-				return
-			}
-
-			// file is 0 bytes - likely no permissions - skipping
-			if *obj.Size == 0 {
+				s.log.V(2).Info("Skipped due to excessive errors", "object", *obj.Key)
 				return
 			}
 
@@ -237,7 +252,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 			})
 			if err != nil {
 				if !strings.Contains(err.Error(), "AccessDenied") {
-					s.log.WithError(err).Errorf("could not get S3 object: %s", *obj.Key)
+					s.log.Error(err, "could not get S3 object", "object", *obj.Key)
 				}
 
 				nErr, ok := errorCount.Load(prefix)
@@ -245,23 +260,22 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 					nErr = 0
 				}
 				if nErr.(int) > 3 {
-					log.Debugf("Skipped: %s", *obj.Key)
+					s.log.V(3).Info("Skipped due to excessive errors", "object", *obj.Key)
 					return
 				}
 				nErr = nErr.(int) + 1
 				errorCount.Store(prefix, nErr)
 				// too many consective errors on this page
 				if nErr.(int) > 3 {
-					s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)
+					s.log.V(2).Info("Too many consecutive errors, excluding prefix", "prefix", prefix)
 				}
-				log.Debugf("Error Counts: %s:%d", prefix, nErr)
 				return
 			}
 
 			defer res.Body.Close()
 			reader, err := diskbufferreader.New(res.Body)
 			if err != nil {
-				log.WithError(err).Error("Could not create reader.")
+				s.log.Error(err, "Could not create reader.")
 				return
 			}
 
@@ -292,14 +306,14 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 			}
 
 			if err := reader.Reset(); err != nil {
-				log.WithError(err).Error("Error resetting reader to start.")
+				s.log.Error(err, "Error resetting reader to start.")
 			}
 			reader.Stop()
 
 			chunk := *chunkSkel
 			chunkData, err := io.ReadAll(reader)
 			if err != nil {
-				log.WithError(err).Error("Could not read file data.")
+				s.log.Error(err, "Could not read file data.")
 				return
 			}
 			chunk.Data = chunkData

proto/custom_detectors.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package custom_detectors;
+
+option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb";
+
+import "validate/validate.proto";
+
+message CustomDetectors {
+  repeated CustomRegex detectors = 1;
+}
+
+message CustomRegex {
+  string name = 1;
+  repeated string keywords = 2;
+  map<string, string> regex = 3;
+  repeated VerifierConfig verify = 4;
+}
+
+message VerifierConfig {
+  string endpoint = 1 [(validate.rules).string.uri_ref = true];
+  bool unsafe = 2;
+  repeated string headers = 3;
+  repeated string successRanges = 4;
+}

proto/detectors.proto

@@ -908,7 +908,9 @@ enum DetectorType {
   SQLServer = 898;
   FTP = 899;
   Redis = 900;
+  LDAP = 901;
   Shopify = 902;
+  RabbitMQ = 903;
 }
 
 message Result {

scripts/gen_proto.sh

@@ -30,3 +30,10 @@ protoc -I proto/ \
     --go_out=plugins=grpc:./pkg/pb/source_metadatapb --go_opt=paths=source_relative \
     --validate_out="lang=go,paths=source_relative:./pkg/pb/source_metadatapb" \
     proto/source_metadata.proto
+protoc -I proto/ \
+    -I ${GOPATH}/src \
+    -I /usr/local/include \
+    -I ${GOPATH}/src/github.com/envoyproxy/protoc-gen-validate \
+    --go_out=plugins=grpc:./pkg/pb/custom_detectorspb --go_opt=paths=source_relative \
+    --validate_out="lang=go,paths=source_relative:./pkg/pb/custom_detectorspb" \
+    proto/custom_detectors.proto