saving progress

* Remove password info from log.

* update.

* one more.

.github/dependabot.yml

@@ -8,9 +8,8 @@ updates:
   - package-ecosystem: "gomod" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "monthly"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      # Check for updates to GitHub Actions every weekday
+      interval: "monthly"
-      interval: "weekly"

.github/workflows/performance.yml

@@ -13,7 +13,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.18'
+        go-version: '1.20'
 
     - name: Checkout code
       uses: actions/checkout@v3

.github/workflows/smoke.yml

@@ -11,7 +11,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.18'
+        go-version: '1.20'
     - name: Checkout code
       uses: actions/checkout@v3
     - name: Smoke

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/felixge/fgprof v0.9.3
 	github.com/getsentry/sentry-go v0.22.0
 	github.com/go-errors/errors v1.4.2
-	github.com/go-git/go-git/v5 v5.7.0
+	github.com/go-git/go-git/v5 v5.8.0
 	github.com/go-ldap/ldap/v3 v3.4.5
 	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zapr v1.2.4
@@ -33,7 +33,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-containerregistry v0.14.0
 	github.com/google/go-github/v42 v42.0.0
-	github.com/googleapis/gax-go/v2 v2.11.0
+	github.com/googleapis/gax-go/v2 v2.12.0
 	github.com/hashicorp/go-retryablehttp v0.7.4
 	github.com/jlaffaye/ftp v0.2.0
 	github.com/joho/godotenv v1.5.1
@@ -56,10 +56,10 @@ require (
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.11.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
-	golang.org/x/oauth2 v0.9.0
+	golang.org/x/oauth2 v0.10.0
 	golang.org/x/sync v0.3.0
 	golang.org/x/text v0.11.0
-	google.golang.org/api v0.130.0
+	google.golang.org/api v0.131.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
@@ -68,7 +68,7 @@ require (
 
 require (
 	cloud.google.com/go v0.110.2 // indirect
-	cloud.google.com/go/compute v1.19.3 // indirect
+	cloud.google.com/go/compute v1.20.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -170,10 +170,10 @@ require (
 	golang.org/x/tools v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
-	google.golang.org/grpc v1.56.1 // indirect
+	google.golang.org/grpc v1.56.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -11,8 +11,8 @@ cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
 cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
+cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
-cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
+cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
@@ -178,8 +178,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
 github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
 github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
-github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
+github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
-github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
+github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
@@ -281,8 +281,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki
 github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
+github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
-github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
+github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@@ -574,8 +574,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
-golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
+golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
-golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
+golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -698,8 +698,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ=
+google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
-google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY=
+google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -722,12 +722,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
+google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
-google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
+google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
-google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
+google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
-google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
+google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -740,8 +740,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
+google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
-google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

hack/docs/Adding_Detectors_Internal.md

@@ -62,15 +62,31 @@ If you think that something should be included outside of these guidelines, plea
    1. Add the test secret to GCP Secrets. See [managing test secrets](#managing-test-secrets)
    2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
    3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
+      * Make sure you understand [verification indeterminacy](#verification-indeterminacy).
    4. Update the tests with these test cases at minimum:
       1. Found and verified (using a credential loaded from GCP Secrets)
-      2. Found and unverified
+      2. Found and unverified (determinately, i.e. the secret is invalid)
-      3. Not found
+      3. Found and unverified (indeterminately due to timeout)
-      4. Any false positive cases that you come across
+      4. Found and unverified (indeterminately due to an unexpected API response)
+      5. Not found
+      6. Any false positive cases that you come across
    5. Create a merge request for review. CI tests must be passing.
 
 ## Addendum
 
+### Verification indeterminacy
+
+There are two types of reasons that secret verification can fail:
+* The candidate secret is not actually a valid secret.
+* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
+
+In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
+
+For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
+* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
+* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
+* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
+
 ### Managing Test Secrets
 
 Do not embed test credentials in the test code. Instead, use GCP Secrets Manager.

hack/docs/Adding_Detectors_external.md

@@ -62,15 +62,31 @@ If you think that something should be included outside of these guidelines, plea
    1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
    2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
    3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
+      * Make sure you understand [verification indeterminacy](#verification-indeterminacy).
    4. Update the tests with these test cases at minimum:
       1. Found and verified (using a credential loaded from GCP Secrets)
-      2. Found and unverified
+      2. Found and unverified (determinately, i.e. the secret is invalid)
-      3. Not found
+      3. Found and unverified (indeterminately due to timeout)
-      4. Any false positive cases that you come across
+      4. Found and unverified (indeterminately due to an unexpected API response)
+      5. Not found
+      6. Any false positive cases that you come across
    5. Create a pull request for review.
 
 ## Addendum
 
+### Verification indeterminacy
+
+There are two types of reasons that secret verification can fail:
+* The candidate secret is not actually a valid secret.
+* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
+
+In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
+
+For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
+* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
+* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
+* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
+
 ### Using a test secret file
 
 1. Create a file called `.env` with this env file format:

hack/generate/generate.go

@@ -34,14 +34,14 @@ func main() {
 	case "detector":
 		mustWriteTemplates([]templateJob{
 			{
-				TemplatePath:  "pkg/detectors/heroku/heroku.go",
+				TemplatePath:  "pkg/detectors/alchemy/alchemy.go",
 				WritePath:     filepath.Join(folderPath(), nameLower+".go"),
-				ReplaceString: []string{"heroku"},
+				ReplaceString: []string{"alchemy"},
 			},
 			{
-				TemplatePath:  "pkg/detectors/heroku/heroku_test.go",
+				TemplatePath:  "pkg/detectors/alchemy/alchemy_test.go",
 				WritePath:     filepath.Join(folderPath(), nameLower+"_test.go"),
-				ReplaceString: []string{"heroku"},
+				ReplaceString: []string{"alchemy"},
 			},
 		})
 		// case "source":

main.go

@@ -207,7 +207,7 @@ func run(state overseer.State) {
 	if *profile {
 		go func() {
 			router := http.NewServeMux()
-			router.Handle("/debug/pprof", http.DefaultServeMux)
+			router.Handle("/debug/pprof/", http.DefaultServeMux)
 			router.Handle("/debug/fgprof", fgprof.Handler())
 			logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
 			if err := http.ListenAndServe(":18066", router); err != nil {

pkg/common/http.go

@@ -3,6 +3,7 @@ package common
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"io"
 	"net"
 	"net/http"
 	"strings"
@@ -75,6 +76,14 @@ func PinnedCertPool() *x509.CertPool {
 	return trustedCerts
 }
 
+type FakeTransport struct {
+	CreateResponse func(req *http.Request) (*http.Response, error)
+}
+
+func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	return t.CreateResponse(req)
+}
+
 type CustomTransport struct {
 	T http.RoundTripper
 }
@@ -91,6 +100,21 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
 	return &CustomTransport{T}
 }
 
+func ConstantStatusHttpClient(statusCode int) *http.Client {
+	return &http.Client{
+		Timeout: DefaultResponseTimeout,
+		Transport: FakeTransport{
+			CreateResponse: func(req *http.Request) (*http.Response, error) {
+				return &http.Response{
+					Request:    req,
+					Body:       io.NopCloser(strings.NewReader("")),
+					StatusCode: statusCode,
+				}, nil
+			},
+		},
+	}
+}
+
 func PinnedRetryableHttpClient() *http.Client {
 	httpClient := retryablehttp.NewClient()
 	httpClient.Logger = nil
@@ -152,9 +176,9 @@ func SaneHttpClient() *http.Client {
 }
 
 // SaneHttpClientTimeOut adds a custom timeout for some scanners
-func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
+func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
 	httpClient := &http.Client{}
-	httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
+	httpClient.Timeout = timeout
 	httpClient.Transport = NewCustomTransport(nil)
 	return httpClient
 }

pkg/context/context.go

@@ -18,22 +18,11 @@ var (
 type Context interface {
 	context.Context
 	Logger() logr.Logger
-	Parent() context.Context
-	SetParent(ctx context.Context) Context
 }
 
-// Parent returns the parent context.
+// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
-func (l logCtx) Parent() context.Context {
+// the same types.
-	return l.Context
+type CancelFunc = context.CancelFunc
-}
-
-// SetParent sets the parent context on the context.
-func (l logCtx) SetParent(ctx context.Context) Context {
-	l.Context = ctx
-	return l
-}
-
-type CancelFunc context.CancelFunc
 
 // logCtx implements Context.
 type logCtx struct {

pkg/detectors/alchemy/alchemy.go

@@ -2,6 +2,7 @@ package alchemy
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -11,14 +12,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()
-
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alchemy"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
 )
@@ -47,6 +49,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.alchemy.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
 			if err != nil {
 				continue
@@ -61,7 +67,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
+
+					if res.StatusCode != 401 {
+						s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+					}
 				}
+			} else {
+				s1.VerificationError = err
 			}
 		}
 

pkg/detectors/alchemy/alchemy_test.go

@@ -32,11 +32,12 @@ func TestAlchemy_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
+		name                string
-		s       Scanner
+		s                   Scanner
-		args    args
+		args                args
-		want    []detectors.Result
+		want                []detectors.Result
-		wantErr bool
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -52,7 +53,8 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -68,7 +70,8 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -78,14 +81,48 @@ func TestAlchemy_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:    nil,
+			want:                nil,
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Alchemy,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantStatusHttpClient(404)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Alchemy,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Alchemy.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -95,6 +132,10 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
+				}
+				got[i].VerificationError = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

pkg/detectors/aws/aws.go

@@ -184,6 +184,8 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
 								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
 							}
+						} else {
+							s1.VerificationError = err
 						}
 						res.Body.Close()
 					} else {

pkg/detectors/dockerhub/dockerhub.go

@@ -0,0 +1,95 @@
+package dockerhub
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Can use email or username for login.
+	usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
+	emailPat    = regexp.MustCompile(common.EmailPattern)
+
+	// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
+	accessTokenPat = regexp.MustCompile(`\bdckr_pat_([a-zA-Z0-9_-]){27}\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"dckr_pat_"}
+}
+
+// FromData will find and optionally verify Dockerhub secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	emailMatches := emailPat.FindAllString(dataStr, -1)
+	dataStr = emailPat.ReplaceAllString(dataStr, "")
+	usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
+
+	accessTokenMatches := accessTokenPat.FindAllString(dataStr, -1)
+
+	userMatches := emailMatches
+	for _, usernameMatch := range usernameMatches {
+		if len(usernameMatch) > 1 {
+			userMatches = append(userMatches, usernameMatch[1])
+		}
+	}
+
+	for _, resUserMatch := range userMatches {
+		for _, resAccessTokenMatch := range accessTokenMatches {
+
+			s1 := detectors.Result{
+				DetectorType: detectorspb.DetectorType_Dockerhub,
+				Raw:          []byte(fmt.Sprintf("%s: %s", resUserMatch, resAccessTokenMatch)),
+			}
+
+			if verify {
+				payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, resUserMatch, resAccessTokenMatch))
+
+				req, err := http.NewRequestWithContext(ctx, "GET", "https://hub.docker.com/v2/users/login", payload)
+				if err != nil {
+					continue
+				}
+				req.Header.Add("Content-Type", "application/json")
+				res, err := client.Do(req)
+				if err == nil {
+					defer res.Body.Close()
+					body, err := io.ReadAll(res.Body)
+					if err != nil {
+						continue
+					}
+
+					// Valid credentials can still return a 401 status code if 2FA is enabled
+					if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 401 && strings.Contains(string(body), "login_2fa_token")) {
+						s1.Verified = true
+					}
+				}
+			}
+
+			results = append(results, s1)
+		}
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Dockerhub
+}

pkg/detectors/dockerhub/dockerhub_test.go

@@ -0,0 +1,138 @@
+//go:build detectors
+// +build detectors
+
+package dockerhub
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestDockerhub_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	username := testSecrets.MustGetField("DOCKERHUB_USERNAME")
+	email := testSecrets.MustGetField("DOCKERHUB_EMAIL")
+	pat := testSecrets.MustGetField("DOCKERHUB_PAT")
+	inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Dockerhub,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, verified (email)",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Dockerhub,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Dockerhub,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/envoyapikey/envoyapikey.go

@@ -1,8 +1,8 @@
-package ibmcloud
+package envoyapikey
 
 import (
 	"context"
-	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"strings"
@@ -21,16 +21,16 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\b([A-Za-z0-9_-]{44})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"envoy"}) + `\b([a-zA-Z0-9]{220})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"ibm"}
+	return []string{"envoy"}
 }
 
-// FromData will find and optionally verify IBM Cloud secrets in a given set of bytes.
+// FromData will find and optionally verify Envoy secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
@@ -43,22 +43,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Ibmcloud,
+			DetectorType: detectorspb.DetectorType_EnvoyApiKey,
 			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ibmcloud.com/apps", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.envoy.com/v1/locations", nil)
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Accept", "application/vnd.ibmcloud+json; version=3")
+			req.Header.Add("Accept", "application/vnd.envoy+json; version=3")
-			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+			req.Header.Add("X-Api-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
+				body, _ := io.ReadAll(res.Body)
-					s1.Verified = true
+
+				// Invalid API keys can also return status code 200, so check for presence of 'status 401' in response body.
+				if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == 403 {
+					if !strings.Contains(string(body), `"status":401`) {
+						s1.Verified = true
+					}
 				} else {
 					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -75,5 +80,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Ibmcloud
+	return detectorspb.DetectorType_EnvoyApiKey
 }

pkg/detectors/envoyapikey/envoyapikey_test.go

@@ -1,7 +1,7 @@
 //go:build detectors
 // +build detectors
 
-package ibmcloud
+package envoyapikey
 
 import (
 	"context"
@@ -10,21 +10,22 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestIbmcloud_FromChunk(t *testing.T) {
+func TestEnvoyapikey_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("IBM_CLOUD_KEY")
+	secret := testSecrets.MustGetField("ENVOYAPIKEY")
-	inactiveSecret := testSecrets.MustGetField("IBM_CLOUD_KEY_INACTIVE")
+	inactiveSecret := testSecrets.MustGetField("ENVOYAPIKEY_INACTIVE")
 
 	type args struct {
 		ctx    context.Context
@@ -43,12 +44,12 @@ func TestIbmcloud_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a ibmcloud secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("You can find a envoyapikey secret %s within", secret)),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_IbmClou,
+					DetectorType: detectorspb.DetectorType_EnvoyApiKey,
 					Verified:     true,
 				},
 			},
@@ -59,12 +60,12 @@ func TestIbmcloud_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a ibmcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a envoyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_Ibmcloud,
+					DetectorType: detectorspb.DetectorType_EnvoyApiKey,
 					Verified:     false,
 				},
 			},
@@ -87,7 +88,7 @@ func TestIbmcloud_FromChunk(t *testing.T) {
 			s := Scanner{}
 			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("Ibmcloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Envoyapikey.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			for i := range got {
@@ -97,7 +98,7 @@ func TestIbmcloud_FromChunk(t *testing.T) {
 				got[i].Raw = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Ibmcloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+				t.Errorf("Envoyapikey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/detectors/getemail/getemail.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +20,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5)
+	client = common.SaneHttpClientTimeOut(5 * time.Second)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getemail"}) + `\b([a-zA-Z0-9-]{20})\b`)

pkg/detectors/jdbc/jdbc.go

@@ -86,7 +86,14 @@ matchLoop:
 			}
 			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 			defer cancel()
-			s.Verified = j.ping(ctx)
+			pingRes := j.ping(ctx)
+			s.Verified = pingRes.err == nil
+			// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
+			// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
+			// detectors to use tri-state verification.
+			if pingRes.err != nil && !pingRes.determinate {
+				s.VerificationError = pingRes.err
+			}
 			// TODO: specialized redaction
 		}
 
@@ -101,15 +108,15 @@ matchLoop:
 }
 
 func tryRedactAnonymousJDBC(conn string) string {
-	if s, ok := tryRedactBasicAuth(conn); ok {
-		return s
-	}
 	if s, ok := tryRedactURLParams(conn); ok {
 		return s
 	}
 	if s, ok := tryRedactODBC(conn); ok {
 		return s
 	}
+	if s, ok := tryRedactBasicAuth(conn); ok {
+		return s
+	}
 	if s, ok := tryRedactRegex(conn); ok {
 		return s
 	}
@@ -158,8 +165,8 @@ func tryRedactODBC(conn string) (string, bool) {
 	var found bool
 	var newParams []string
 	for _, param := range strings.Split(conn, ";") {
-		key, val, _ := strings.Cut(param, "=")
+		key, val, isKvp := strings.Cut(param, "=")
-		if strings.Contains(strings.ToLower(key), "pass") {
+		if isKvp && strings.Contains(strings.ToLower(key), "pass") {
 			newParams = append(newParams, key+"="+strings.Repeat("*", len(val)))
 			found = true
 			continue
@@ -198,8 +205,13 @@ var supportedSubprotocols = map[string]func(string) (jdbc, error){
 	"sqlserver":  parseSqlServer,
 }
 
+type pingResult struct {
+	err         error
+	determinate bool
+}
+
 type jdbc interface {
-	ping(context.Context) bool
+	ping(context.Context) pingResult
 }
 
 func newJDBC(conn string) (jdbc, error) {
@@ -220,11 +232,16 @@ func newJDBC(conn string) (jdbc, error) {
 	return parser(subname)
 }
 
-func ping(ctx context.Context, driverName, conn string) bool {
+func ping(ctx context.Context, driverName string, isDeterminate func(error) bool, candidateConns ...string) pingResult {
-	if err := pingErr(ctx, driverName, conn); err != nil {
+	var indeterminateErrors []error
-		return false
+	for _, c := range candidateConns {
+		err := pingErr(ctx, driverName, c)
+		if err == nil || isDeterminate(err) {
+			return pingResult{err, true}
+		}
+		indeterminateErrors = append(indeterminateErrors, err)
 	}
-	return true
+	return pingResult{errors.Join(indeterminateErrors...), false}
 }
 
 func pingErr(ctx context.Context, driverName, conn string) error {

pkg/detectors/jdbc/jdbc_integration_test.go

@@ -105,6 +105,22 @@ func TestJdbcVerified(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "sql server verified",
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=localhost;database=%s;password=%s", sqlServerDatabase, sqlServerPass)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_JDBC,
+					Verified:     true,
+					Redacted:     "jdbc:sqlserver://odbc:server=localhost;database=master;password=**************",
+				},
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/jdbc/jdbc_test.go

@@ -5,6 +5,7 @@ package jdbc
 
 import (
 	"context"
+	"github.com/stretchr/testify/assert"
 	"os"
 	"testing"
 
@@ -207,6 +208,51 @@ func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
 	}
 }
 
+func TestJdbc_Redact(t *testing.T) {
+	tests := []struct {
+		name string
+		conn string
+		want string
+	}{
+		{
+			name: "basic auth'",
+			conn: "//user:secret@tcp(127.0.0.1:3306)/",
+			want: "//user:******@tcp(127.0.0.1:3306)/",
+		},
+		{
+			name: "basic auth including raw string 'pass'",
+			conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/",
+			want: "//wrongUser:*********@tcp(127.0.0.1:3306)/",
+		},
+		{
+			name: "basic auth including raw string 'pass' with unfortunate db name",
+			conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/passwords",
+			want: "//wrongUser:*********@tcp(127.0.0.1:3306)/passwords",
+		},
+		{
+			name: "url param-style",
+			conn: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=p@ssw04d",
+			want: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=********",
+		},
+		{
+			name: "odbc-style without server",
+			conn: "//odbc:server=localhost;user id=sa;database=master;password=/p?s=sw&rd",
+			want: "//odbc:server=localhost;user id=sa;database=master;password=**********",
+		},
+		{
+			name: "odbc-style with server",
+			conn: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=super-secret-password",
+			want: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=*********************",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tryRedactAnonymousJDBC(tt.conn)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}

pkg/detectors/jdbc/mysql.go

@@ -3,9 +3,8 @@ package jdbc
 import (
 	"context"
 	"errors"
+	"github.com/go-sql-driver/mysql"
 	"strings"
-
-	_ "github.com/go-sql-driver/mysql"
 )
 
 type mysqlJDBC struct {
@@ -16,30 +15,40 @@ type mysqlJDBC struct {
 	params   string
 }
 
-func (s *mysqlJDBC) ping(ctx context.Context) bool {
+func (s *mysqlJDBC) ping(ctx context.Context) pingResult {
-	if ping(ctx, "mysql", s.conn) {
+	return ping(ctx, "mysql", isMySQLErrorDeterminate,
-		return true
+		s.conn,
-	}
+		buildMySQLConnectionString(s.host, s.database, s.userPass, s.params),
-	// try building connection string (should be same as s.conn though)
+		buildMySQLConnectionString(s.host, "", s.userPass, s.params))
-	if ping(ctx, "mysql", s.build()) {
-		return true
-	}
-	// try removing database
-	s.database = ""
-	return ping(ctx, "mysql", s.build())
 }
 
-func (s *mysqlJDBC) build() string {
+func buildMySQLConnectionString(host, database, userPass, params string) string {
-	conn := s.host + "/" + s.database
+	conn := host + "/" + database
-	if s.userPass != "" {
+	if userPass != "" {
-		conn = s.userPass + "@" + conn
+		conn = userPass + "@" + conn
 	}
-	if s.params != "" {
+	if params != "" {
-		conn = conn + "?" + s.params
+		conn = conn + "?" + params
 	}
 	return conn
 }
 
+func isMySQLErrorDeterminate(err error) bool {
+	// MySQL error numbers from https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
+	if mySQLErr, isMySQLErr := err.(*mysql.MySQLError); isMySQLErr {
+		switch mySQLErr.Number {
+		case 1044:
+			// User access denied to a particular database
+			return false // "Indeterminate" so that other connection variations will be tried
+		case 1045:
+			// User access denied
+			return true
+		}
+	}
+
+	return false
+}
+
 func parseMySQL(subname string) (jdbc, error) {
 	// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
 	hostAndDB, params, _ := strings.Cut(subname, "?")

pkg/detectors/jdbc/mysql_integration_test.go

@@ -21,45 +21,54 @@ const (
 )
 
 func TestMySQL(t *testing.T) {
+	type result struct {
+		parseErr        bool
+		pingOk          bool
+		pingDeterminate bool
+	}
 	tests := []struct {
-		input    string
+		input string
-		wantErr  bool
+		want  result
-		wantPing bool
 	}{
 		{
-			input:   "",
+			input: "",
-			wantErr: true,
+			want:  result{parseErr: true},
 		},
 		{
-			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			wantPing: true,
+			want:  result{pingOk: true, pingDeterminate: true},
 		},
 		{
-			input:    "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			wantPing: false,
+			want:  result{pingOk: false, pingDeterminate: true},
 		},
 		{
-			input:    "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			wantPing: false,
+			want:  result{pingOk: false, pingDeterminate: true},
 		},
 		{
-			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
+			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
-			wantPing: true,
+			want:  result{pingOk: true, pingDeterminate: true},
 		},
 		{
-			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
+			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
-			wantPing: true,
+			want:  result{pingOk: true, pingDeterminate: true},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parseMySQL(tt.input)
-			if tt.wantErr {
+
-				assert.Error(t, err)
+			if err != nil {
+				got := result{parseErr: true}
+				assert.Equal(t, tt.want, got)
 				return
 			}
-			assert.NoError(t, err)
+
-			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
+			pr := j.ping(context.Background())
+
+			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
+			assert.Equal(t, tt.want, got)
 		})
 	}
 }

pkg/detectors/jdbc/postgres.go

@@ -4,9 +4,8 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/lib/pq"
 	"strings"
-
-	_ "github.com/lib/pq"
 )
 
 type postgresJDBC struct {
@@ -14,37 +13,35 @@ type postgresJDBC struct {
 	params map[string]string
 }
 
-func (s *postgresJDBC) ping(ctx context.Context) bool {
+func (s *postgresJDBC) ping(ctx context.Context) pingResult {
-	// try the provided connection string directly
+	// It is crucial that we try to build a connection string ourselves before using the one we found. This is because
-	if ping(ctx, "postgres", s.conn) {
+	// if the found connection string doesn't include a username, the driver will attempt to connect using the current
-		return true
+	// user's name, which will fail in a way that looks like a determinate failure, thus terminating the waterfall. In
-	}
+	// contrast, when we build a connection string ourselves, if there's no username, we try 'postgres' instead, which
-	// try as a URL
+	// actually has a chance of working.
-	if ping(ctx, "postgres", "postgres://"+s.conn) {
+	return ping(ctx, "postgres", isPostgresErrorDeterminate,
-		return true
+		buildPostgresConnectionString(s.params, true),
-	}
+		buildPostgresConnectionString(s.params, false),
-	// build a connection string
+		s.conn,
-	data := map[string]string{
+		"postgres://"+s.conn)
-		// default user
+}
-		"user": "postgres",
+
-	}
+func isPostgresErrorDeterminate(err error) bool {
-	for key, val := range s.params {
+	// Postgres codes from https://www.postgresql.org/docs/current/errcodes-appendix.html
-		if key == "host" {
+	if pqErr, isPostgresError := err.(*pq.Error); isPostgresError {
-			if h, p, found := strings.Cut(val, ":"); found {
+		switch pqErr.Code {
-				data["host"] = h
+		case "28P01":
-				data["port"] = p
+			// Invalid username/password
-				continue
+			return true
-			}
+		case "3D000":
+			// Unknown database
+			return false // "Indeterminate" so that other connection variations will be tried
+		case "3F000":
+			// Unknown schema
+			return false // "Indeterminate" so that other connection variations will be tried
 		}
-		data[key] = val
-	}
-	if ping(ctx, "postgres", joinKeyValues(data, " ")) {
-		return true
-	}
-	if s.params["dbname"] != "" {
-		delete(s.params, "dbname")
-		return s.ping(ctx)
 	}
+
 	return false
 }
 
@@ -79,3 +76,25 @@ func parsePostgres(subname string) (jdbc, error) {
 
 	return &postgresJDBC{subname[2:], params}, nil
 }
+
+func buildPostgresConnectionString(params map[string]string, includeDbName bool) string {
+	data := map[string]string{
+		// default user
+		"user": "postgres",
+	}
+	for key, val := range params {
+		if key == "host" {
+			if h, p, found := strings.Cut(val, ":"); found {
+				data["host"] = h
+				data["port"] = p
+				continue
+			}
+		}
+		if key == "dbname" && !includeDbName {
+			continue
+		}
+		data[key] = val
+	}
+
+	return joinKeyValues(data, " ")
+}

pkg/detectors/jdbc/postgres_integration_test.go

@@ -20,49 +20,62 @@ const (
 )
 
 func TestPostgres(t *testing.T) {
+	type result struct {
+		parseErr        bool
+		pingOk          bool
+		pingDeterminate bool
+	}
 	tests := []struct {
-		input    string
+		input string
-		wantErr  bool
+		want  result
-		wantPing bool
 	}{
 		{
-			input:    "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
+			input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
-			wantPing: true,
+			want:  result{pingOk: true, pingDeterminate: true},
 		},
 		{
-			input:    "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
+			input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
-			wantPing: true,
+			want:  result{pingOk: true, pingDeterminate: true},
 		},
 		{
-			input:    "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
+			input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
-			wantPing: true,
+			want:  result{pingOk: true, pingDeterminate: true},
 		},
 		{
-			input:    "//localhost:5432/foo?password=" + postgresPass,
+			input: "//localhost:5432/foo?password=" + postgresPass,
-			wantPing: false,
+			want:  result{pingOk: false, pingDeterminate: false},
 		},
 		{
-			input:    "//localhost:5432/foo?sslmode=disable&password=foo",
+			input: "//localhost:5432/foo?sslmode=disable&password=foo",
-			wantPing: false,
+			want:  result{pingOk: false, pingDeterminate: true},
 		},
 		{
-			input:    "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
+			input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
-			wantPing: false,
+			want:  result{pingOk: false, pingDeterminate: true},
 		},
 		{
-			input:   "invalid",
+			input: "//badhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
-			wantErr: true,
+			want:  result{pingOk: false, pingDeterminate: false},
+		},
+		{
+			input: "invalid",
+			want:  result{parseErr: true},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parsePostgres(tt.input)
-			if tt.wantErr {
+
-				assert.Error(t, err)
+			if err != nil {
+				got := result{parseErr: true}
+				assert.Equal(t, tt.want, got)
 				return
 			}
-			assert.NoError(t, err)
+
-			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
+			pr := j.ping(context.Background())
+
+			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
+			assert.Equal(t, tt.want, got)
 		})
 	}
 }

pkg/detectors/jdbc/sqlite.go

@@ -14,12 +14,18 @@ type sqliteJDBC struct {
 	testing  bool
 }
 
-func (s *sqliteJDBC) ping(ctx context.Context) bool {
+var cannotVerifySqliteError error = errors.New("sqlite credentials cannot be verified")
+
+func (s *sqliteJDBC) ping(ctx context.Context) pingResult {
 	if !s.testing {
 		// sqlite is not a networked database, so we cannot verify
-		return false
+		return pingResult{cannotVerifySqliteError, true}
 	}
-	return ping(ctx, "sqlite3", s.filename)
+	return ping(ctx, "sqlite3", isSqliteErrorDeterminate, s.filename)
+}
+
+func isSqliteErrorDeterminate(err error) bool {
+	return true
 }
 
 func parseSqlite(subname string) (jdbc, error) {

pkg/detectors/jdbc/sqlite_test.go

@@ -39,7 +39,7 @@ func TestParseSqlite(t *testing.T) {
 				assert.Error(t, err)
 			} else {
 				assert.NoError(t, err)
-				assert.True(t, j.ping(context.Background()))
+				assert.True(t, j.ping(context.Background()).err == nil)
 			}
 		})
 	}

pkg/detectors/jdbc/sqlserver.go

@@ -3,9 +3,8 @@ package jdbc
 import (
 	"context"
 	"errors"
+	mssql "github.com/denisenkom/go-mssqldb"
 	"strings"
-
-	_ "github.com/denisenkom/go-mssqldb"
 )
 
 type sqlServerJDBC struct {
@@ -13,15 +12,25 @@ type sqlServerJDBC struct {
 	params map[string]string
 }
 
-func (s *sqlServerJDBC) ping(ctx context.Context) bool {
+func (s *sqlServerJDBC) ping(ctx context.Context) pingResult {
-	if ping(ctx, "mssql", s.conn) {
+	return ping(ctx, "mssql", isSqlServerErrorDeterminate,
-		return true
+		joinKeyValues(s.params, ";"),
+		s.conn,
+		"sqlserver://"+s.conn)
+}
+
+func isSqlServerErrorDeterminate(err error) bool {
+	// Error numbers from https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors?view=sql-server-ver16
+	if mssqlError, isMssqlError := err.(mssql.Error); isMssqlError {
+		switch mssqlError.Number {
+		case 18456:
+			// Login failed
+			// This is a determinate failure iff we tried to use a real user
+			return mssqlError.Message != "login error: Login failed for user ''."
+		}
 	}
-	if ping(ctx, "mssql", joinKeyValues(s.params, ";")) {
+
-		return true
+	return false
-	}
-	// try URL format
-	return ping(ctx, "mssql", "sqlserver://"+s.conn)
 }
 
 func parseSqlServer(subname string) (jdbc, error) {

pkg/detectors/jdbc/sqlserver_integration_test.go

@@ -21,33 +21,50 @@ const (
 )
 
 func TestSqlServer(t *testing.T) {
+	type result struct {
+		parseErr        bool
+		pingOk          bool
+		pingDeterminate bool
+	}
 	tests := []struct {
-		input    string
+		input string
-		wantErr  bool
+		want  result
-		wantPing bool
 	}{
 		{
-			input:   "",
+			input: "",
-			wantErr: true,
+			want:  result{parseErr: true},
 		},
 		{
-			input:    "//odbc:server=localhost;user id=sa;database=master;password=" + sqlServerPass,
+			input: "//server=localhost;user id=sa;database=master;password=" + sqlServerPass,
-			wantPing: true,
+			want:  result{pingOk: true, pingDeterminate: true},
 		},
 		{
-			input:    "//localhost;database= master;spring.datasource.password=" + sqlServerPass,
+			input: "//server=badhost;user id=sa;database=master;password=" + sqlServerPass,
-			wantPing: true,
+			want:  result{pingOk: false, pingDeterminate: false},
+		},
+		{
+			input: "//localhost;database=master;spring.datasource.password=" + sqlServerPass,
+			want:  result{pingOk: true, pingDeterminate: true},
+		},
+		{
+			input: "//localhost;database=master;spring.datasource.password=badpassword",
+			want:  result{pingOk: false, pingDeterminate: true},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parseSqlServer(tt.input)
-			if tt.wantErr {
+
-				assert.Error(t, err)
+			if err != nil {
+				got := result{parseErr: true}
+				assert.Equal(t, tt.want, got)
 				return
 			}
-			assert.NoError(t, err)
+
-			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
+			pr := j.ping(context.Background())
+
+			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
+			assert.Equal(t, tt.want, got)
 		})
 	}
 }

pkg/detectors/launchdarkly/launchdarkly.go

@@ -20,13 +20,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly"}) + `\b([a-z0-9-]{40})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"launchdarkly"}
+	return []string{"launchdarkly", "launch_darkly"}
 }
 
 // FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.

pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -18,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5)
+	client = common.SaneHttpClientTimeOut(5 * time.Second)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)

pkg/detectors/okta/okta.go

@@ -19,8 +19,8 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
+	domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
-	tokenPat  = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
+	tokenPat  = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
 	// TODO: Oauth client secrets
 )
 

pkg/detectors/postgres/postgres.go

@@ -0,0 +1,105 @@
+package postgres
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct {
+	client *http.Client
+}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	defaultClient = common.SaneHttpClient()
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"postgres"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
+)
+
+// find various connection strings
+
+// postgresql://user:secret@localhost:5432/mydb
+// "host=localhost dbname=publishing user=www password=foo"
+
+// detectors.PrefixRegex([]string{"nytimes"})
+
+//var (
+//	connectionStringPats = map[string]*regexp.Regexp{
+//		"KeyValue":               regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
+//		"":              regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
+//		"Slack Workspace Access Token":  regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
+//		"Slack Workspace Refresh Token": regexp.MustCompile(`xoxr\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
+//	}
+//	verifyURL = "https://slack.com/api/auth.test"
+//)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"postgres://", ""}
+}
+
+// FromData will find and optionally verify Postgres secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, match := range matches {
+		if len(match) != 2 {
+			continue
+		}
+		resMatch := strings.TrimSpace(match[1])
+
+		s1 := detectors.Result{
+			DetectorType: detectorspb.DetectorType_Postgres,
+			Raw:          []byte(resMatch),
+		}
+
+		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.postgres.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
+			if err != nil {
+				continue
+			}
+			res, err := client.Do(req)
+			if err == nil {
+				defer res.Body.Close()
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
+					s1.Verified = true
+				} else {
+					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						continue
+					}
+
+					if res.StatusCode != 401 {
+						s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+					}
+				}
+			} else {
+				s1.VerificationError = err
+			}
+		}
+
+		results = append(results, s1)
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_Postgres
+}

pkg/detectors/postgres/postgres_test.go

@@ -0,0 +1,160 @@
+//go:build detectors
+// +build detectors
+
+package postgres
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestPostgres_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secret := testSecrets.MustGetField("POSTGRES")
+	inactiveSecret := testSecrets.MustGetField("POSTGRES_INACTIVE")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:                nil,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantStatusHttpClient(404)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_Postgres,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
+				}
+				got[i].VerificationError = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/scrapersite/scrapersite.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +20,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(10)
+	client = common.SaneHttpClientTimeOut(10 * time.Second)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)

pkg/detectors/screenshotlayer/screenshotlayer.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +20,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(10)
+	client = common.SaneHttpClientTimeOut(10 * time.Second)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"screenshotlayer"}) + `\b([a-zA-Z0-9_]{32})\b`)

pkg/detectors/trufflehogenterprise/trufflehogenterprise.go

@@ -0,0 +1,108 @@
+package trufflehogenterprise
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+
+var (
+	client = common.SaneHttpClient()
+
+	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+	keyPat      = regexp.MustCompile(`\bthog-key-[0-9a-f]{16}\b`)
+	secretPat   = regexp.MustCompile(`\bthog-secret-[0-9a-f]{32}\b`)
+	hostnamePat = regexp.MustCompile(`\b[a-z]+-[a-z]+-[a-z]+\.[a-z][0-9]\.[a-z]+\.trufflehog\.org\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+	return []string{"thog"}
+}
+
+// FromData will find and optionally verify Trufflehog Enterprise secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+	dataStr := string(data)
+
+	keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
+	hostnameMatches := hostnamePat.FindAllStringSubmatch(dataStr, -1)
+
+	for _, keyMatch := range keyMatches {
+		if len(keyMatch) != 1 {
+			continue
+		}
+		resKeyMatch := strings.TrimSpace(keyMatch[0])
+		for _, secretMatch := range secretMatches {
+
+			if len(secretMatch) != 1 {
+				continue
+			}
+			resSecretMatch := strings.TrimSpace(secretMatch[0])
+
+			for _, hostnameMatch := range hostnameMatches {
+				if len(hostnameMatch) != 1 {
+					continue
+				}
+
+				resHostnameMatch := strings.TrimSpace(hostnameMatch[0])
+
+				s1 := detectors.Result{
+					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
+					Raw:          []byte(resKeyMatch),
+				}
+
+				if verify {
+					endpoint := fmt.Sprintf("https://%s/api/v1/sources", resHostnameMatch)
+					req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
+					if err != nil {
+						continue
+					}
+
+					req.Header.Add("Accept", "application/vnd.trufflehogenterprise+json; version=3")
+					req.Header.Add("X-Thog-Secret", resSecretMatch)
+					req.Header.Add("X-Thog-Key", resKeyMatch)
+
+					res, err := client.Do(req)
+					if err == nil {
+						verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "data")
+						if err != nil {
+							return nil, err
+						}
+
+						defer res.Body.Close()
+
+						if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
+							s1.Verified = true
+						} else {
+							// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+							if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
+								continue
+							}
+						}
+					}
+				}
+
+				results = append(results, s1)
+			}
+		}
+	}
+
+	return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+	return detectorspb.DetectorType_TrufflehogEnterprise
+}

pkg/detectors/trufflehogenterprise/trufflehogenterprise_test.go

@@ -0,0 +1,121 @@
+//go:build detectors
+// +build detectors
+
+package trufflehogenterprise
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestTrufflehogenterprise_FromChunk(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	host := testSecrets.MustGetField("THOG_HOSTNAME")
+	key := testSecrets.MustGetField("THOG_WEB_KEY")
+	secret := testSecrets.MustGetField("THOG_WEB_SECRET")
+	inactiveSecret := testSecrets.MustGetField("THOG_WEB_INACTIVE_SECRET")
+
+	type args struct {
+		ctx    context.Context
+		data   []byte
+		verify bool
+	}
+	tests := []struct {
+		name    string
+		s       Scanner
+		args    args
+		want    []detectors.Result
+		wantErr bool
+	}{
+		{
+			name: "found, verified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a thog secret %s for %s with key %s within", secret, host, key)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
+					Verified:     true,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "found, unverified",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a trufflehogenterprise secret %s for %s with key %s within but not valid", inactiveSecret, host, key)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
+					Verified:     false,
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "not found",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte("You cannot find the secret within"),
+				verify: true,
+			},
+			want:    nil,
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Trufflehogenterprise.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			for i := range got {
+				if len(got[i].Raw) == 0 {
+					t.Fatalf("no raw secret present: \n %+v", got[i])
+				}
+				got[i].Raw = nil
+			}
+			if diff := pretty.Compare(got, tt.want); diff != "" {
+				t.Errorf("Trufflehogenterprise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			}
+		})
+	}
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+	ctx := context.Background()
+	s := Scanner{}
+	for name, data := range detectors.MustGetBenchmarkData() {
+		benchmark.Run(name, func(b *testing.B) {
+			for n := 0; n < b.N; n++ {
+				_, err := s.FromData(ctx, false, data)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		})
+	}
+}

pkg/detectors/twilio/twilio.go

@@ -3,9 +3,7 @@ package twilio
 import (
 	"context"
 	"net/http"
-	"net/url"
 	"regexp"
-	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -53,14 +51,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
-
-				form := url.Values{}
-				form.Add("FriendlyName", "MyServiceName")
 				req, err := http.NewRequestWithContext(
-					ctx, "POST", "https://verify.twilio.com/v2/Services",
+					ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
-
-					strings.NewReader(form.Encode()),
-				)
 				if err != nil {
 					continue
 				}

pkg/detectors/uri/uri.go

@@ -53,7 +53,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		password := match[1]
 
 		// Skip findings where the password only has "*" characters, this is a redacted password
-		if strings.Trim(password, "*") == "" {
+		// Also include the url encoded "*" characters: "%2A"
+		if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
 			continue
 		}
 

pkg/detectors/uri/uri_test.go

@@ -31,14 +31,14 @@ func TestURI_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
+				data:   []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_URI,
 					Verified:     false,
-					Redacted:     "https://user:********@www.httpwatch.com",
+					Redacted:     "https://user:********@httpwatch.com",
 				},
 			},
 			wantErr: false,
@@ -87,6 +87,17 @@ func TestURI_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "nothing found, password was redacted two different ways",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("Both %s and %s have been redacted within", "https://httpwatch::********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar", "https://httpwatch::%2A%2A%2A%2A%2A%2A@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")),
+				verify: true,
+			},
+			want:    []detectors.Result{},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/zenserp/zenserp.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -18,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5)
+	client = common.SaneHttpClientTimeOut(5 * time.Second)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"zenserp"}) + `\b([0-9a-z-]{36})\b`)

pkg/engine/defaults.go

@@ -1,6 +1,8 @@
 package engine
 
 import (
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
@@ -159,6 +161,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convertkit"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convier"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/copper"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/couchbase"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/countrylayer"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/courier"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coveralls"
@@ -198,6 +201,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/disqus"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ditto"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dnscheck"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dockerhub"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docparser"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/documo"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docusign"
@@ -1523,6 +1527,9 @@ func DefaultDetectors() []detectors.Detector {
 		prefect.Scanner{},
 		buildkitev2.Scanner{},
 		opsgenie.Scanner{},
+		dockerhub.Scanner{},
+		couchbase.Scanner{},
+		envoyapikey.Scanner{},
 	}
 
 }

pkg/engine/engine.go

@@ -113,6 +113,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		chunks:          make(chan *sources.Chunk),
 		results:         make(chan detectors.ResultWithMetadata),
 		detectorAvgTime: sync.Map{},
+		sourcesWg:       &errgroup.Group{},
 	}
 
 	for _, option := range options {
@@ -127,10 +128,8 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 	ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
 
-	sourcesWg, egCtx := errgroup.WithContext(ctx)
+	// Limit number of concurrent goroutines dedicated to chunking a source.
-	sourcesWg.SetLimit(e.concurrency)
+	e.sourcesWg.SetLimit(e.concurrency)
-	e.sourcesWg = sourcesWg
-	ctx.SetParent(egCtx)
 
 	if len(e.decoders) == 0 {
 		e.decoders = decoders.DefaultDecoders()
@@ -185,7 +184,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	for i := 0; i < e.concurrency; i++ {
 		e.workersWg.Add(1)
 		go func() {
-			defer common.RecoverWithExit(ctx)
+			defer common.Recover(ctx)
 			defer e.workersWg.Done()
 			e.detectorWorker(ctx)
 		}()
@@ -378,31 +377,23 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 	}
 }
 
-// lineNumberSupportedSources is a list of sources that support line numbers.
+// SupportsLineNumbers determines if a line number can be found for a source type.
-// It is stored this way because slice consts are not supported.
+func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
-func lineNumberSupportedSources() []sourcespb.SourceType {
+	switch sourceType {
-	return []sourcespb.SourceType{
+	case sourcespb.SourceType_SOURCE_TYPE_GIT,
-		sourcespb.SourceType_SOURCE_TYPE_GIT,
 		sourcespb.SourceType_SOURCE_TYPE_GITHUB,
 		sourcespb.SourceType_SOURCE_TYPE_GITLAB,
 		sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
 		sourcespb.SourceType_SOURCE_TYPE_GERRIT,
 		sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
 		sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
-		sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
+		sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
+		return true
+	default:
+		return false
 	}
 }
 
-// SupportsLineNumbers determines if a line number can be found for a source type.
-func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
-	for _, i := range lineNumberSupportedSources() {
-		if i == sourceType {
-			return true
-		}
-	}
-	return false
-}
-
 // FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
 func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
 	lines := bytes.Split(chunk.Data, []byte("\n"))

pkg/engine/engine_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
@@ -72,3 +73,30 @@ func TestDefaultDecoders(t *testing.T) {
 		t.Errorf("DefaultDecoders() = %v, expected UTF8 decoder to be first", ds)
 	}
 }
+
+func TestSupportsLineNumbers(t *testing.T) {
+	testCases := []struct {
+		name     string
+		input    sourcespb.SourceType
+		expected bool
+	}{
+		{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
+		{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
+		{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if result := SupportsLineNumbers(tc.input); result != tc.expected {
+				t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
+			}
+		})
+	}
+}
+
+func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
+	sourceType := sourcespb.SourceType_SOURCE_TYPE_GITHUB
+	for i := 0; i < b.N; i++ {
+		_ = SupportsLineNumbers(sourceType)
+	}
+}

pkg/engine/filesystem.go

@@ -28,6 +28,22 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 	}
 
 	fileSystemSource := filesystem.Source{}
+	fileSystemSource.WithFilter(c.Filter)
+
+	// TODO: This is how it would be used; everything after these comments would be removed.
+	// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
+	// 	func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
+	// 		fileSystemSource := filesystem.Source{}
+	// 		fileSystemSource.WithFilter(c.Filter)
+	// 		if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
+	// 			return nil, err
+	// 		}
+	// 		return &fileSystemSource, nil
+	// 	})
+	// if err != nil {
+	// 	return err
+	// }
+	// return e.sourceManager.ScheduleRun(ctx, handle)
 
 	ctx = context.WithValues(ctx,
 		"source_type", fileSystemSource.Type().String(),
@@ -37,7 +53,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}
-	fileSystemSource.WithFilter(c.Filter)
 	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
 		err := fileSystemSource.Chunks(ctx, e.ChunksChan())

pkg/engine/git_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -15,6 +16,7 @@ import (
 type expResult struct {
 	B          string
 	LineNumber int64
+	Verified   bool
 }
 
 func TestGitEngine(t *testing.T) {
@@ -38,16 +40,16 @@ func TestGitEngine(t *testing.T) {
 	for tName, tTest := range map[string]testProfile{
 		"all_secrets": {
 			expected: map[string]expResult{
-				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
+				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
-				"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4},
+				"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4, true},
-				"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
+				"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3, false},
-				"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
+				"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7, false},
 			},
 			filter: common.FilterEmpty(),
 		},
 		"base_commit": {
 			expected: map[string]expResult{
-				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
+				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
 			},
 			filter: common.FilterEmpty(),
 			base:   "2f251b8c1e72135a375b659951097ec7749d4af9",
@@ -57,8 +59,12 @@ func TestGitEngine(t *testing.T) {
 			e := Start(ctx,
 				WithConcurrency(1),
 				WithDecoders(decoders.DefaultDecoders()...),
-				WithDetectors(false, DefaultDetectors()...),
+				WithDetectors(true, DefaultDetectors()...),
 			)
+			// Make the channels buffered so Finish returns.
+			e.chunks = make(chan *sources.Chunk, 10)
+			e.results = make(chan detectors.ResultWithMetadata, 10)
+
 			cfg := sources.GitConfig{
 				RepoPath: path,
 				HeadRef:  tTest.branch,
@@ -73,7 +79,8 @@ func TestGitEngine(t *testing.T) {
 			logFatalFunc := func(_ error, _ string, _ ...any) {
 				t.Fatalf("error logging function should not have been called")
 			}
-			go e.Finish(ctx, logFatalFunc)
+			// Wait for all the chunks to be processed.
+			e.Finish(ctx, logFatalFunc)
 			resultCount := 0
 			for result := range e.ResultsChan() {
 				switch meta := result.SourceMetadata.GetData().(type) {
@@ -84,6 +91,9 @@ func TestGitEngine(t *testing.T) {
 					if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
 						t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
 					}
+					if tTest.expected[meta.Git.Commit].Verified != result.Verified {
+						t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
+					}
 				}
 				resultCount++
 

pkg/pb/detectorspb/detectors.pb.go

@@ -992,6 +992,9 @@ const (
 	DetectorType_Prefect                       DetectorType = 918
 	DetectorType_Docusign                      DetectorType = 919
 	DetectorType_Couchbase                     DetectorType = 920
+	DetectorType_Dockerhub                     DetectorType = 921
+	DetectorType_TrufflehogEnterprise          DetectorType = 922
+	DetectorType_EnvoyApiKey                   DetectorType = 923
 )
 
 // Enum value maps for DetectorType.
@@ -1914,6 +1917,9 @@ var (
 		918: "Prefect",
 		919: "Docusign",
 		920: "Couchbase",
+		921: "Dockerhub",
+		922: "TrufflehogEnterprise",
+		923: "EnvoyApiKey",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2833,6 +2839,9 @@ var (
 		"Prefect":                       918,
 		"Docusign":                      919,
 		"Couchbase":                     920,
+		"Dockerhub":                     921,
+		"TrufflehogEnterprise":          922,
+		"EnvoyApiKey":                   923,
 	}
 )
 
@@ -3211,7 +3220,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc6, 0x73, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4132,11 +4141,15 @@ var file_detectors_proto_rawDesc = []byte{
 	0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x72, 0x65,
 	0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63, 0x75, 0x73,
 	0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63, 0x68, 0x62,
-	0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
+	0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72,
-	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75,
+	0x68, 0x75, 0x62, 0x10, 0x99, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x75, 0x66, 0x66, 0x6c,
-	0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f,
+	0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
-	0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
+	0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
-	0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x10, 0x9b, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
+	0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
+	0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
+	0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
+	0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/sources/filesystem/filesystem.go

@@ -46,6 +46,7 @@ type Source struct {
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 var _ sources.SourceUnitEnumerator = (*Source)(nil)
+var _ sources.SourceUnitChunker = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
@@ -220,12 +221,53 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 // Enumerate implements SourceUnitEnumerator interface. This implementation simply
 // passes the configured paths as the source unit, whether it be a single
 // filepath or a directory.
-func (s *Source) Enumerate(ctx context.Context, units chan<- sources.EnumerationResult) error {
+func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
 	for _, path := range s.paths {
-		item := sources.CommonEnumerationOk(path)
+		item := sources.CommonSourceUnit{ID: path}
-		if err := common.CancellableWrite(ctx, units, item); err != nil {
+		if err := reporter.UnitOk(ctx, item); err != nil {
 			return err
 		}
 	}
 	return nil
 }
+
+// ChunkUnit implements SourceUnitChunker interface.
+func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
+	path := unit.SourceUnitID()
+	logger := ctx.Logger().WithValues("path", path)
+
+	cleanPath := filepath.Clean(path)
+	fileInfo, err := os.Stat(cleanPath)
+	if err != nil {
+		return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
+	}
+
+	ch := make(chan *sources.Chunk)
+	var scanErr error
+	go func() {
+		defer close(ch)
+		if fileInfo.IsDir() {
+			// TODO: Finer grain error tracking of individual chunks.
+			scanErr = s.scanDir(ctx, cleanPath, ch)
+		} else {
+			// TODO: Finer grain error tracking of individual
+			// chunks (in the case of archives).
+			scanErr = s.scanFile(ctx, cleanPath, ch)
+		}
+	}()
+
+	for chunk := range ch {
+		if chunk == nil {
+			continue
+		}
+		if err := reporter.ChunkOk(ctx, *chunk); err != nil {
+			return err
+		}
+	}
+
+	if scanErr != nil && scanErr != io.EOF {
+		logger.Info("error scanning filesystem", "error", scanErr)
+		return reporter.ChunkErr(ctx, scanErr)
+	}
+	return nil
+}

pkg/sources/git/git.go

@@ -755,7 +755,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
 		}
 		path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username(), "--shallow-since", timestamp)
 		if err != nil {
-			return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
+			return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
 		}
 	default:
 		ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
@@ -793,7 +793,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
 			}
 			path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username())
 			if err != nil {
-				return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
+				return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
 			}
 		default:
 			ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)

pkg/sources/source_manager.go

@@ -0,0 +1,164 @@
+package sources
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"golang.org/x/sync/errgroup"
+)
+
+// handle uniquely identifies a Source given to the manager to manage. If the
+// SourceManager is connected to the API, it will be equivalent to the unique
+// source ID, otherwise it behaves as a counter.
+type handle int64
+
+// SourceInitFunc is a function that takes a source and job ID and returns an
+// initialized Source.
+type SourceInitFunc func(sourceID int64, jobID int64) (Source, error)
+
+type SourceManager struct {
+	api apiClient
+	// Map of handle to source initializer.
+	handles     map[handle]SourceInitFunc
+	handlesLock sync.Mutex
+	// Pool limiting the amount of concurrent sources running.
+	pool errgroup.Group
+	// Downstream chunks channel to be scanned.
+	outputChunks chan *Chunk
+}
+
+// apiClient is an interface for optionally communicating with an external API.
+type apiClient interface {
+	// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
+	RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
+	// GetJobID queries the API for an existing job or new job ID.
+	GetJobID(ctx context.Context, id int64) (int64, error)
+}
+
+// WithAPI adds an API client to the manager for tracking jobs and progress.
+func WithAPI(api apiClient) func(*SourceManager) {
+	return func(man *SourceManager) { man.api = api }
+}
+
+// WithConcurrency limits the concurrent number of sources a manager can run.
+func WithConcurrency(concurrency int) func(*SourceManager) {
+	return func(man *SourceManager) { man.pool.SetLimit(concurrency) }
+}
+
+// NewManager creates a new manager with the provided options.
+func NewManager(outputChunks chan *Chunk, opts ...func(*SourceManager)) *SourceManager {
+	man := SourceManager{
+		// Default to the headless API. Can be overwritten by the WithAPI option.
+		api:          &headlessAPI{},
+		handles:      make(map[handle]SourceInitFunc),
+		outputChunks: outputChunks,
+	}
+	for _, opt := range opts {
+		opt(&man)
+	}
+	return &man
+}
+
+// Enroll informs the SourceManager to track and manage a Source.
+func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
+	id, err := s.api.RegisterSource(ctx, name, kind)
+	if err != nil {
+		return 0, err
+	}
+	handleID := handle(id)
+	s.handlesLock.Lock()
+	defer s.handlesLock.Unlock()
+	if _, ok := s.handles[handleID]; ok {
+		// TODO: smartly handle this?
+		return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
+	}
+	s.handles[handleID] = f
+	return 0, nil
+}
+
+// Run blocks until a resource is available to run the source, then synchronously runs it.
+func (s *SourceManager) Run(ctx context.Context, handle handle) error {
+	// Check the handle is valid before waiting on the pool.
+	if _, ok := s.getInitFunc(handle); !ok {
+		return fmt.Errorf("unrecognized handle")
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ch := make(chan error)
+	s.pool.Go(func() error {
+		defer common.Recover(ctx)
+		// TODO: The manager should record these errors.
+		ch <- s.run(ctx, handle)
+		return nil
+	})
+	return <-ch
+}
+
+// ScheduleRun blocks until a resource is available to run the source, then
+// asynchronously runs it. Error information is lost in this case.
+func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
+	// Check the handle is valid before waiting on the pool.
+	if _, ok := s.getInitFunc(handle); !ok {
+		return fmt.Errorf("unrecognized handle")
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	s.pool.Go(func() error {
+		defer common.Recover(ctx)
+		// TODO: The manager should record these errors.
+		_ = s.run(ctx, handle)
+		return nil
+	})
+	// TODO: Maybe wait for a signal here that initialization was successful?
+	return nil
+}
+
+// run is a helper method to sychronously run the source. It does not check for
+// acquired resources.
+func (s *SourceManager) run(ctx context.Context, handle handle) error {
+	jobID, err := s.api.GetJobID(ctx, int64(handle))
+	if err != nil {
+		return err
+	}
+	initFunc, ok := s.getInitFunc(handle)
+	if !ok {
+		return fmt.Errorf("unrecognized handle")
+	}
+	source, err := initFunc(jobID, int64(handle))
+	if err != nil {
+		return err
+	}
+	// TODO: Support UnitChunker and SourceUnitEnumerator.
+	// TODO: This is where we can introspect on the chunks collected.
+	return source.Chunks(ctx, s.outputChunks)
+}
+
+// getInitFunc is a helper method for safe concurrent access to the
+// map[handle]SourceInitFunc map.
+func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
+	s.handlesLock.Lock()
+	defer s.handlesLock.Unlock()
+	f, ok := s.handles[handle]
+	return f, ok
+}
+
+// headlessAPI implements the apiClient interface locally.
+type headlessAPI struct {
+	// Counters for assigning handle and job IDs.
+	sourceIDCounter int64
+	jobIDCounter    int64
+}
+
+func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
+	return atomic.AddInt64(&api.sourceIDCounter, 1), nil
+}
+
+func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
+	return atomic.AddInt64(&api.jobIDCounter, 1), nil
+}

pkg/sources/sources.go

@@ -53,21 +53,41 @@ type SourceUnitUnmarshaller interface {
 // SourceUnitEnumerator defines an optional interface a Source can implement to
 // support enumerating an initialized Source into SourceUnits.
 type SourceUnitEnumerator interface {
-	// Enumerate enumerates the initialized Source, outputting units. This
+	// Enumerate creates 0 or more units from an initialized source,
-	// method is synchronous but can be called in a goroutine to support
+	// reporting them or any errors to the UnitReporter. This method is
-	// concurrent enumeration and chunking. An error should only be
+	// synchronous but can be called in a goroutine to support concurrent
-	// returned from this method in the case of context cancellation. All
+	// enumeration and chunking. An error should only be returned from this
-	// other errors related to unit enumeration are tracked in the
+	// method in the case of context cancellation, fatal source errors, or
-	// EnumerationResult.
+	// errors returned by the reporter All other errors related to unit
-	Enumerate(ctx context.Context, units chan<- EnumerationResult) error
+	// enumeration are tracked by the UnitReporter.
+	Enumerate(ctx context.Context, reporter UnitReporter) error
 }
 
-// EnumerationResult is the result of an enumeration, containing the unit and
+// UnitReporter defines the interface a source will use to report whether a
-// error if any. Unit and Error are mutually exclusive (only one will be
+// unit was found during enumeration. Either method may be called any number of
-// non-nil).
+// times. Implementors of this interface should allow for concurrent calls.
-type EnumerationResult struct {
+type UnitReporter interface {
-	Unit  SourceUnit
+	UnitOk(ctx context.Context, unit SourceUnit) error
-	Error error
+	UnitErr(ctx context.Context, err error) error
+}
+
+// SourceUnitChunker defines an optional interface a Source can implement to
+// support chunking a single SourceUnit.
+type SourceUnitChunker interface {
+	// ChunkUnit creates 0 or more chunks from a unit, reporting them or
+	// any errors to the ChunkReporter. An error should only be returned
+	// from this method in the case of context cancellation, fatal source
+	// errors, or errors returned by the reporter. All other errors related
+	// to unit chunking are tracked by the ChunkReporter.
+	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
+}
+
+// ChunkReporter defines the interface a source will use to report whether a
+// chunk was found during unit chunking. Either method may be called any number
+// of times. Implementors of this interface should allow for concurrent calls.
+type ChunkReporter interface {
+	ChunkOk(ctx context.Context, chunk Chunk) error
+	ChunkErr(ctx context.Context, err error) error
 }
 
 // SourceUnit is an object that represents a Source's unit of work. This is
@@ -243,16 +263,3 @@ func (p *Progress) GetProgress() *Progress {
 	defer p.mut.Unlock()
 	return p
 }
-
-// CommonEnumerationOk is a helper function to construct an EnumerationResult
-// using a CommonSourceUnit.
-func CommonEnumerationOk(id string) EnumerationResult {
-	unit := CommonSourceUnit{ID: id}
-	return EnumerationResult{Unit: unit}
-}
-
-// EnumerationErr is a helper function to construct an EnumerationResult from
-// an error.
-func EnumerationErr(err error) EnumerationResult {
-	return EnumerationResult{Error: err}
-}

proto/detectors.proto

@@ -929,7 +929,10 @@ enum DetectorType {
   Prefect = 918;
   Docusign = 919;
   Couchbase = 920;
-  IBMCloud = 923; // 921 and 922 are reserved for Trufflehog Web and Scanner Detectors
+  Dockerhub = 921;
+  TrufflehogEnterprise = 922;
+  EnvoyApiKey = 923;
+  Postgres = 925;
 }
 
 message Result {