save ibm progress

.github/dependabot.yml

@@ -8,8 +8,9 @@ updates:
   - package-ecosystem: "gomod" # See documentation for possible values
     directory: "/" # Location of package manifests
     schedule:
-      interval: "monthly"
+      interval: "weekly"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "monthly"
+      # Check for updates to GitHub Actions every weekday
+      interval: "weekly"

.github/workflows/performance.yml

@@ -13,7 +13,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.18'
 
     - name: Checkout code
       uses: actions/checkout@v3

.github/workflows/smoke.yml

@@ -11,7 +11,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.18'
     - name: Checkout code
       uses: actions/checkout@v3
     - name: Smoke

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/felixge/fgprof v0.9.3
 	github.com/getsentry/sentry-go v0.22.0
 	github.com/go-errors/errors v1.4.2
-	github.com/go-git/go-git/v5 v5.8.0
+	github.com/go-git/go-git/v5 v5.7.0
 	github.com/go-ldap/ldap/v3 v3.4.5
 	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zapr v1.2.4
@@ -33,7 +33,7 @@ require (
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-containerregistry v0.14.0
 	github.com/google/go-github/v42 v42.0.0
-	github.com/googleapis/gax-go/v2 v2.12.0
+	github.com/googleapis/gax-go/v2 v2.11.0
 	github.com/hashicorp/go-retryablehttp v0.7.4
 	github.com/jlaffaye/ftp v0.2.0
 	github.com/joho/godotenv v1.5.1
@@ -56,10 +56,10 @@ require (
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.11.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
-	golang.org/x/oauth2 v0.10.0
+	golang.org/x/oauth2 v0.9.0
 	golang.org/x/sync v0.3.0
 	golang.org/x/text v0.11.0
-	google.golang.org/api v0.131.0
+	google.golang.org/api v0.130.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
@@ -68,7 +68,7 @@ require (
 
 require (
 	cloud.google.com/go v0.110.2 // indirect
-	cloud.google.com/go/compute v1.20.1 // indirect
+	cloud.google.com/go/compute v1.19.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -170,10 +170,10 @@ require (
 	golang.org/x/tools v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
+	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
-	google.golang.org/grpc v1.56.2 // indirect
+	google.golang.org/grpc v1.56.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -11,8 +11,8 @@ cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
 cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
+cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
-cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
+cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
@@ -178,8 +178,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
 github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
 github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
 github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
-github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
+github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
-github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
+github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
@@ -281,8 +281,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki
 github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
+github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
-github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
+github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@@ -574,8 +574,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
+golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -698,8 +698,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
+google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ=
-google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
+google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -722,12 +722,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
+google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
-google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
+google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
-google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
+google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
-google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
+google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -740,8 +740,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
 google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
+google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
-google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=

hack/docs/Adding_Detectors_Internal.md

@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
    1. Add the test secret to GCP Secrets. See [managing test secrets](#managing-test-secrets)
    2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
    3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
-      * Make sure you understand [verification indeterminacy](#verification-indeterminacy).
    4. Update the tests with these test cases at minimum:
       1. Found and verified (using a credential loaded from GCP Secrets)
-      2. Found and unverified (determinately, i.e. the secret is invalid)
+      2. Found and unverified
-      3. Found and unverified (indeterminately due to timeout)
+      3. Not found
-      4. Found and unverified (indeterminately due to an unexpected API response)
+      4. Any false positive cases that you come across
-      5. Not found
-      6. Any false positive cases that you come across
    5. Create a merge request for review. CI tests must be passing.
 
 ## Addendum
 
-### Verification indeterminacy
-
-There are two types of reasons that secret verification can fail:
-* The candidate secret is not actually a valid secret.
-* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
-
-In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
-
-For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
-* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
-* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
-* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
-
 ### Managing Test Secrets
 
 Do not embed test credentials in the test code. Instead, use GCP Secrets Manager.

hack/docs/Adding_Detectors_external.md

@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
    1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
    2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
    3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
-      * Make sure you understand [verification indeterminacy](#verification-indeterminacy).
    4. Update the tests with these test cases at minimum:
       1. Found and verified (using a credential loaded from GCP Secrets)
-      2. Found and unverified (determinately, i.e. the secret is invalid)
+      2. Found and unverified
-      3. Found and unverified (indeterminately due to timeout)
+      3. Not found
-      4. Found and unverified (indeterminately due to an unexpected API response)
+      4. Any false positive cases that you come across
-      5. Not found
-      6. Any false positive cases that you come across
    5. Create a pull request for review.
 
 ## Addendum
 
-### Verification indeterminacy
-
-There are two types of reasons that secret verification can fail:
-* The candidate secret is not actually a valid secret.
-* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
-
-In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
-
-For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
-* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
-* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
-* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
-
 ### Using a test secret file
 
 1. Create a file called `.env` with this env file format:

hack/generate/generate.go

@@ -34,14 +34,14 @@ func main() {
 	case "detector":
 		mustWriteTemplates([]templateJob{
 			{
-				TemplatePath:  "pkg/detectors/alchemy/alchemy.go",
+				TemplatePath:  "pkg/detectors/heroku/heroku.go",
 				WritePath:     filepath.Join(folderPath(), nameLower+".go"),
-				ReplaceString: []string{"alchemy"},
+				ReplaceString: []string{"heroku"},
 			},
 			{
-				TemplatePath:  "pkg/detectors/alchemy/alchemy_test.go",
+				TemplatePath:  "pkg/detectors/heroku/heroku_test.go",
 				WritePath:     filepath.Join(folderPath(), nameLower+"_test.go"),
-				ReplaceString: []string{"alchemy"},
+				ReplaceString: []string{"heroku"},
 			},
 		})
 		// case "source":

main.go

@@ -207,7 +207,7 @@ func run(state overseer.State) {
 	if *profile {
 		go func() {
 			router := http.NewServeMux()
-			router.Handle("/debug/pprof/", http.DefaultServeMux)
+			router.Handle("/debug/pprof", http.DefaultServeMux)
 			router.Handle("/debug/fgprof", fgprof.Handler())
 			logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
 			if err := http.ListenAndServe(":18066", router); err != nil {

pkg/common/http.go

@@ -3,7 +3,6 @@ package common
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"io"
 	"net"
 	"net/http"
 	"strings"
@@ -76,14 +75,6 @@ func PinnedCertPool() *x509.CertPool {
 	return trustedCerts
 }
 
-type FakeTransport struct {
-	CreateResponse func(req *http.Request) (*http.Response, error)
-}
-
-func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	return t.CreateResponse(req)
-}
-
 type CustomTransport struct {
 	T http.RoundTripper
 }
@@ -100,21 +91,6 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
 	return &CustomTransport{T}
 }
 
-func ConstantStatusHttpClient(statusCode int) *http.Client {
-	return &http.Client{
-		Timeout: DefaultResponseTimeout,
-		Transport: FakeTransport{
-			CreateResponse: func(req *http.Request) (*http.Response, error) {
-				return &http.Response{
-					Request:    req,
-					Body:       io.NopCloser(strings.NewReader("")),
-					StatusCode: statusCode,
-				}, nil
-			},
-		},
-	}
-}
-
 func PinnedRetryableHttpClient() *http.Client {
 	httpClient := retryablehttp.NewClient()
 	httpClient.Logger = nil
@@ -176,9 +152,9 @@ func SaneHttpClient() *http.Client {
 }
 
 // SaneHttpClientTimeOut adds a custom timeout for some scanners
-func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
+func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
 	httpClient := &http.Client{}
-	httpClient.Timeout = timeout
+	httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
 	httpClient.Transport = NewCustomTransport(nil)
 	return httpClient
 }

pkg/context/context.go

@@ -18,11 +18,22 @@ var (
 type Context interface {
 	context.Context
 	Logger() logr.Logger
+	Parent() context.Context
+	SetParent(ctx context.Context) Context
 }
 
-// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
+// Parent returns the parent context.
-// the same types.
+func (l logCtx) Parent() context.Context {
-type CancelFunc = context.CancelFunc
+	return l.Context
+}
+
+// SetParent sets the parent context on the context.
+func (l logCtx) SetParent(ctx context.Context) Context {
+	l.Context = ctx
+	return l
+}
+
+type CancelFunc context.CancelFunc
 
 // logCtx implements Context.
 type logCtx struct {

pkg/detectors/alchemy/alchemy.go

@@ -2,7 +2,6 @@ package alchemy
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -12,15 +11,14 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-type Scanner struct {
+type Scanner struct{}
-	client *http.Client
-}
 
 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	defaultClient = common.SaneHttpClient()
+	client = common.SaneHttpClient()
+
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alchemy"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
 )
@@ -49,10 +47,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}
 
 		if verify {
-			client := s.client
-			if client == nil {
-				client = defaultClient
-			}
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.alchemy.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
 			if err != nil {
 				continue
@@ -67,13 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 						continue
 					}
-
-					if res.StatusCode != 401 {
-						s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
-					}
 				}
-			} else {
-				s1.VerificationError = err
 			}
 		}
 

pkg/detectors/alchemy/alchemy_test.go

@@ -32,12 +32,11 @@ func TestAlchemy_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name                string
+		name    string
-		s                   Scanner
+		s       Scanner
-		args                args
+		args    args
-		want                []detectors.Result
+		want    []detectors.Result
-		wantErr             bool
+		wantErr bool
-		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -53,8 +52,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr:             false,
+			wantErr: false,
-			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -70,8 +68,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr:             false,
+			wantErr: false,
-			wantVerificationErr: false,
 		},
 		{
 			name: "not found",
@@ -81,48 +78,14 @@ func TestAlchemy_FromChunk(t *testing.T) {
 				data:   []byte("You cannot find the secret within"),
 				verify: true,
 			},
-			want:                nil,
+			want:    nil,
-			wantErr:             false,
+			wantErr: false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Alchemy,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantStatusHttpClient(404)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Alchemy,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			s := Scanner{}
+			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("Alchemy.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -132,10 +95,6 @@ func TestAlchemy_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
-				}
-				got[i].VerificationError = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)

pkg/detectors/aws/aws.go

@@ -184,8 +184,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
 								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
 							}
-						} else {
-							s1.VerificationError = err
 						}
 						res.Body.Close()
 					} else {

pkg/detectors/dockerhub/dockerhub.go

@@ -1,95 +0,0 @@
-package dockerhub
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	client = common.SaneHttpClient()
-
-	// Can use email or username for login.
-	usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
-	emailPat    = regexp.MustCompile(common.EmailPattern)
-
-	// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
-	accessTokenPat = regexp.MustCompile(`\bdckr_pat_([a-zA-Z0-9_-]){27}\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"dckr_pat_"}
-}
-
-// FromData will find and optionally verify Dockerhub secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	emailMatches := emailPat.FindAllString(dataStr, -1)
-	dataStr = emailPat.ReplaceAllString(dataStr, "")
-	usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
-
-	accessTokenMatches := accessTokenPat.FindAllString(dataStr, -1)
-
-	userMatches := emailMatches
-	for _, usernameMatch := range usernameMatches {
-		if len(usernameMatch) > 1 {
-			userMatches = append(userMatches, usernameMatch[1])
-		}
-	}
-
-	for _, resUserMatch := range userMatches {
-		for _, resAccessTokenMatch := range accessTokenMatches {
-
-			s1 := detectors.Result{
-				DetectorType: detectorspb.DetectorType_Dockerhub,
-				Raw:          []byte(fmt.Sprintf("%s: %s", resUserMatch, resAccessTokenMatch)),
-			}
-
-			if verify {
-				payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, resUserMatch, resAccessTokenMatch))
-
-				req, err := http.NewRequestWithContext(ctx, "GET", "https://hub.docker.com/v2/users/login", payload)
-				if err != nil {
-					continue
-				}
-				req.Header.Add("Content-Type", "application/json")
-				res, err := client.Do(req)
-				if err == nil {
-					defer res.Body.Close()
-					body, err := io.ReadAll(res.Body)
-					if err != nil {
-						continue
-					}
-
-					// Valid credentials can still return a 401 status code if 2FA is enabled
-					if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 401 && strings.Contains(string(body), "login_2fa_token")) {
-						s1.Verified = true
-					}
-				}
-			}
-
-			results = append(results, s1)
-		}
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Dockerhub
-}

pkg/detectors/dockerhub/dockerhub_test.go

@@ -1,138 +0,0 @@
-//go:build detectors
-// +build detectors
-
-package dockerhub
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestDockerhub_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	username := testSecrets.MustGetField("DOCKERHUB_USERNAME")
-	email := testSecrets.MustGetField("DOCKERHUB_EMAIL")
-	pat := testSecrets.MustGetField("DOCKERHUB_PAT")
-	inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Dockerhub,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, verified (email)",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Dockerhub,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Dockerhub,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/detectors/getemail/getemail.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5 * time.Second)
+	client = common.SaneHttpClientTimeOut(5)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getemail"}) + `\b([a-zA-Z0-9-]{20})\b`)

pkg/detectors/ibmcloud/ibmcloud.go

@@ -1,8 +1,8 @@
-package envoyapikey
+package ibmcloud
 
 import (
 	"context"
-	"io"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -21,16 +21,16 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"envoy"}) + `\b([a-zA-Z0-9]{220})\b`)
+	keyPat = regexp.MustCompile(`\b([A-Za-z0-9_-]{44})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"envoy"}
+	return []string{"ibm"}
 }
 
-// FromData will find and optionally verify Envoy secrets in a given set of bytes.
+// FromData will find and optionally verify IBM Cloud secrets in a given set of bytes.
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
@@ -43,27 +43,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		resMatch := strings.TrimSpace(match[1])
 
 		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_EnvoyApiKey,
+			DetectorType: detectorspb.DetectorType_Ibmcloud,
 			Raw:          []byte(resMatch),
 		}
 
 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.envoy.com/v1/locations", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ibmcloud.com/apps", nil)
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Accept", "application/vnd.envoy+json; version=3")
+			req.Header.Add("Accept", "application/vnd.ibmcloud+json; version=3")
-			req.Header.Add("X-Api-Key", resMatch)
+			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				body, _ := io.ReadAll(res.Body)
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
-
+					s1.Verified = true
-				// Invalid API keys can also return status code 200, so check for presence of 'status 401' in response body.
-				if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == 403 {
-					if !strings.Contains(string(body), `"status":401`) {
-						s1.Verified = true
-					}
 				} else {
 					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -80,5 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 }
 
 func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_EnvoyApiKey
+	return detectorspb.DetectorType_Ibmcloud
 }

pkg/detectors/ibmcloud/ibmcloud_test.go

@@ -1,7 +1,7 @@
 //go:build detectors
 // +build detectors
 
-package envoyapikey
+package ibmcloud
 
 import (
 	"context"
@@ -10,22 +10,21 @@ import (
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
-func TestEnvoyapikey_FromChunk(t *testing.T) {
+func TestIbmcloud_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
-	secret := testSecrets.MustGetField("ENVOYAPIKEY")
+	secret := testSecrets.MustGetField("IBM_CLOUD_KEY")
-	inactiveSecret := testSecrets.MustGetField("ENVOYAPIKEY_INACTIVE")
+	inactiveSecret := testSecrets.MustGetField("IBM_CLOUD_KEY_INACTIVE")
 
 	type args struct {
 		ctx    context.Context
@@ -44,12 +43,12 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a envoyapikey secret %s within", secret)),
+				data:   []byte(fmt.Sprintf("You can find a ibmcloud secret %s within", secret)),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_EnvoyApiKey,
+					DetectorType: detectorspb.DetectorType_IbmClou,
 					Verified:     true,
 				},
 			},
@@ -60,12 +59,12 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a envoyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
+				data:   []byte(fmt.Sprintf("You can find a ibmcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
-					DetectorType: detectorspb.DetectorType_EnvoyApiKey,
+					DetectorType: detectorspb.DetectorType_Ibmcloud,
 					Verified:     false,
 				},
 			},
@@ -88,7 +87,7 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
 			s := Scanner{}
 			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("Envoyapikey.FromData() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("Ibmcloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			for i := range got {
@@ -98,7 +97,7 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
 				got[i].Raw = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Envoyapikey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+				t.Errorf("Ibmcloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}

pkg/detectors/jdbc/jdbc.go

@@ -86,14 +86,7 @@ matchLoop:
 			}
 			ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 			defer cancel()
-			pingRes := j.ping(ctx)
+			s.Verified = j.ping(ctx)
-			s.Verified = pingRes.err == nil
-			// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
-			// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
-			// detectors to use tri-state verification.
-			if pingRes.err != nil && !pingRes.determinate {
-				s.VerificationError = pingRes.err
-			}
 			// TODO: specialized redaction
 		}
 
@@ -108,15 +101,15 @@ matchLoop:
 }
 
 func tryRedactAnonymousJDBC(conn string) string {
+	if s, ok := tryRedactBasicAuth(conn); ok {
+		return s
+	}
 	if s, ok := tryRedactURLParams(conn); ok {
 		return s
 	}
 	if s, ok := tryRedactODBC(conn); ok {
 		return s
 	}
-	if s, ok := tryRedactBasicAuth(conn); ok {
-		return s
-	}
 	if s, ok := tryRedactRegex(conn); ok {
 		return s
 	}
@@ -165,8 +158,8 @@ func tryRedactODBC(conn string) (string, bool) {
 	var found bool
 	var newParams []string
 	for _, param := range strings.Split(conn, ";") {
-		key, val, isKvp := strings.Cut(param, "=")
+		key, val, _ := strings.Cut(param, "=")
-		if isKvp && strings.Contains(strings.ToLower(key), "pass") {
+		if strings.Contains(strings.ToLower(key), "pass") {
 			newParams = append(newParams, key+"="+strings.Repeat("*", len(val)))
 			found = true
 			continue
@@ -205,13 +198,8 @@ var supportedSubprotocols = map[string]func(string) (jdbc, error){
 	"sqlserver":  parseSqlServer,
 }
 
-type pingResult struct {
-	err         error
-	determinate bool
-}
-
 type jdbc interface {
-	ping(context.Context) pingResult
+	ping(context.Context) bool
 }
 
 func newJDBC(conn string) (jdbc, error) {
@@ -232,16 +220,11 @@ func newJDBC(conn string) (jdbc, error) {
 	return parser(subname)
 }
 
-func ping(ctx context.Context, driverName string, isDeterminate func(error) bool, candidateConns ...string) pingResult {
+func ping(ctx context.Context, driverName, conn string) bool {
-	var indeterminateErrors []error
+	if err := pingErr(ctx, driverName, conn); err != nil {
-	for _, c := range candidateConns {
+		return false
-		err := pingErr(ctx, driverName, c)
-		if err == nil || isDeterminate(err) {
-			return pingResult{err, true}
-		}
-		indeterminateErrors = append(indeterminateErrors, err)
 	}
-	return pingResult{errors.Join(indeterminateErrors...), false}
+	return true
 }
 
 func pingErr(ctx context.Context, driverName, conn string) error {

pkg/detectors/jdbc/jdbc_integration_test.go

@@ -105,22 +105,6 @@ func TestJdbcVerified(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "sql server verified",
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=localhost;database=%s;password=%s", sqlServerDatabase, sqlServerPass)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_JDBC,
-					Verified:     true,
-					Redacted:     "jdbc:sqlserver://odbc:server=localhost;database=master;password=**************",
-				},
-			},
-			wantErr: false,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/jdbc/jdbc_test.go

@@ -5,7 +5,6 @@ package jdbc
 
 import (
 	"context"
-	"github.com/stretchr/testify/assert"
 	"os"
 	"testing"
 
@@ -208,51 +207,6 @@ func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
 	}
 }
 
-func TestJdbc_Redact(t *testing.T) {
-	tests := []struct {
-		name string
-		conn string
-		want string
-	}{
-		{
-			name: "basic auth'",
-			conn: "//user:secret@tcp(127.0.0.1:3306)/",
-			want: "//user:******@tcp(127.0.0.1:3306)/",
-		},
-		{
-			name: "basic auth including raw string 'pass'",
-			conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/",
-			want: "//wrongUser:*********@tcp(127.0.0.1:3306)/",
-		},
-		{
-			name: "basic auth including raw string 'pass' with unfortunate db name",
-			conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/passwords",
-			want: "//wrongUser:*********@tcp(127.0.0.1:3306)/passwords",
-		},
-		{
-			name: "url param-style",
-			conn: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=p@ssw04d",
-			want: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=********",
-		},
-		{
-			name: "odbc-style without server",
-			conn: "//odbc:server=localhost;user id=sa;database=master;password=/p?s=sw&rd",
-			want: "//odbc:server=localhost;user id=sa;database=master;password=**********",
-		},
-		{
-			name: "odbc-style with server",
-			conn: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=super-secret-password",
-			want: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=*********************",
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got := tryRedactAnonymousJDBC(tt.conn)
-			assert.Equal(t, tt.want, got)
-		})
-	}
-}
-
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}

pkg/detectors/jdbc/mysql.go

@@ -3,8 +3,9 @@ package jdbc
 import (
 	"context"
 	"errors"
-	"github.com/go-sql-driver/mysql"
 	"strings"
+
+	_ "github.com/go-sql-driver/mysql"
 )
 
 type mysqlJDBC struct {
@@ -15,40 +16,30 @@ type mysqlJDBC struct {
 	params   string
 }
 
-func (s *mysqlJDBC) ping(ctx context.Context) pingResult {
+func (s *mysqlJDBC) ping(ctx context.Context) bool {
-	return ping(ctx, "mysql", isMySQLErrorDeterminate,
+	if ping(ctx, "mysql", s.conn) {
-		s.conn,
+		return true
-		buildMySQLConnectionString(s.host, s.database, s.userPass, s.params),
+	}
-		buildMySQLConnectionString(s.host, "", s.userPass, s.params))
+	// try building connection string (should be same as s.conn though)
+	if ping(ctx, "mysql", s.build()) {
+		return true
+	}
+	// try removing database
+	s.database = ""
+	return ping(ctx, "mysql", s.build())
 }
 
-func buildMySQLConnectionString(host, database, userPass, params string) string {
+func (s *mysqlJDBC) build() string {
-	conn := host + "/" + database
+	conn := s.host + "/" + s.database
-	if userPass != "" {
+	if s.userPass != "" {
-		conn = userPass + "@" + conn
+		conn = s.userPass + "@" + conn
 	}
-	if params != "" {
+	if s.params != "" {
-		conn = conn + "?" + params
+		conn = conn + "?" + s.params
 	}
 	return conn
 }
 
-func isMySQLErrorDeterminate(err error) bool {
-	// MySQL error numbers from https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
-	if mySQLErr, isMySQLErr := err.(*mysql.MySQLError); isMySQLErr {
-		switch mySQLErr.Number {
-		case 1044:
-			// User access denied to a particular database
-			return false // "Indeterminate" so that other connection variations will be tried
-		case 1045:
-			// User access denied
-			return true
-		}
-	}
-
-	return false
-}
-
 func parseMySQL(subname string) (jdbc, error) {
 	// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
 	hostAndDB, params, _ := strings.Cut(subname, "?")

pkg/detectors/jdbc/mysql_integration_test.go

@@ -21,54 +21,45 @@ const (
 )
 
 func TestMySQL(t *testing.T) {
-	type result struct {
-		parseErr        bool
-		pingOk          bool
-		pingDeterminate bool
-	}
 	tests := []struct {
-		input string
+		input    string
-		want  result
+		wantErr  bool
+		wantPing bool
 	}{
 		{
-			input: "",
+			input:   "",
-			want:  result{parseErr: true},
+			wantErr: true,
 		},
 		{
-			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			want:  result{pingOk: true, pingDeterminate: true},
+			wantPing: true,
 		},
 		{
-			input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			input:    "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			want:  result{pingOk: false, pingDeterminate: true},
+			wantPing: false,
 		},
 		{
-			input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+			input:    "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
-			want:  result{pingOk: false, pingDeterminate: true},
+			wantPing: false,
 		},
 		{
-			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
+			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
-			want:  result{pingOk: true, pingDeterminate: true},
+			wantPing: true,
 		},
 		{
-			input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
+			input:    "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
-			want:  result{pingOk: true, pingDeterminate: true},
+			wantPing: true,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parseMySQL(tt.input)
-
+			if tt.wantErr {
-			if err != nil {
+				assert.Error(t, err)
-				got := result{parseErr: true}
-				assert.Equal(t, tt.want, got)
 				return
 			}
-
+			assert.NoError(t, err)
-			pr := j.ping(context.Background())
+			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
-
-			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
-			assert.Equal(t, tt.want, got)
 		})
 	}
 }

pkg/detectors/jdbc/postgres.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/lib/pq"
 	"strings"
+
+	_ "github.com/lib/pq"
 )
 
 type postgresJDBC struct {
@@ -13,35 +14,37 @@ type postgresJDBC struct {
 	params map[string]string
 }
 
-func (s *postgresJDBC) ping(ctx context.Context) pingResult {
+func (s *postgresJDBC) ping(ctx context.Context) bool {
-	// It is crucial that we try to build a connection string ourselves before using the one we found. This is because
+	// try the provided connection string directly
-	// if the found connection string doesn't include a username, the driver will attempt to connect using the current
+	if ping(ctx, "postgres", s.conn) {
-	// user's name, which will fail in a way that looks like a determinate failure, thus terminating the waterfall. In
+		return true
-	// contrast, when we build a connection string ourselves, if there's no username, we try 'postgres' instead, which
+	}
-	// actually has a chance of working.
+	// try as a URL
-	return ping(ctx, "postgres", isPostgresErrorDeterminate,
+	if ping(ctx, "postgres", "postgres://"+s.conn) {
-		buildPostgresConnectionString(s.params, true),
+		return true
-		buildPostgresConnectionString(s.params, false),
+	}
-		s.conn,
+	// build a connection string
-		"postgres://"+s.conn)
+	data := map[string]string{
-}
+		// default user
-
+		"user": "postgres",
-func isPostgresErrorDeterminate(err error) bool {
+	}
-	// Postgres codes from https://www.postgresql.org/docs/current/errcodes-appendix.html
+	for key, val := range s.params {
-	if pqErr, isPostgresError := err.(*pq.Error); isPostgresError {
+		if key == "host" {
-		switch pqErr.Code {
+			if h, p, found := strings.Cut(val, ":"); found {
-		case "28P01":
+				data["host"] = h
-			// Invalid username/password
+				data["port"] = p
-			return true
+				continue
-		case "3D000":
+			}
-			// Unknown database
+		}
-			return false // "Indeterminate" so that other connection variations will be tried
+		data[key] = val
-		case "3F000":
+	}
-			// Unknown schema
+	if ping(ctx, "postgres", joinKeyValues(data, " ")) {
-			return false // "Indeterminate" so that other connection variations will be tried
+		return true
-		}
+	}
+	if s.params["dbname"] != "" {
+		delete(s.params, "dbname")
+		return s.ping(ctx)
 	}
-
 	return false
 }
 
@@ -76,25 +79,3 @@ func parsePostgres(subname string) (jdbc, error) {
 
 	return &postgresJDBC{subname[2:], params}, nil
 }
-
-func buildPostgresConnectionString(params map[string]string, includeDbName bool) string {
-	data := map[string]string{
-		// default user
-		"user": "postgres",
-	}
-	for key, val := range params {
-		if key == "host" {
-			if h, p, found := strings.Cut(val, ":"); found {
-				data["host"] = h
-				data["port"] = p
-				continue
-			}
-		}
-		if key == "dbname" && !includeDbName {
-			continue
-		}
-		data[key] = val
-	}
-
-	return joinKeyValues(data, " ")
-}

pkg/detectors/jdbc/postgres_integration_test.go

@@ -20,62 +20,49 @@ const (
 )
 
 func TestPostgres(t *testing.T) {
-	type result struct {
-		parseErr        bool
-		pingOk          bool
-		pingDeterminate bool
-	}
 	tests := []struct {
-		input string
+		input    string
-		want  result
+		wantErr  bool
+		wantPing bool
 	}{
 		{
-			input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
+			input:    "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
-			want:  result{pingOk: true, pingDeterminate: true},
+			wantPing: true,
 		},
 		{
-			input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
+			input:    "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
-			want:  result{pingOk: true, pingDeterminate: true},
+			wantPing: true,
 		},
 		{
-			input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
+			input:    "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
-			want:  result{pingOk: true, pingDeterminate: true},
+			wantPing: true,
 		},
 		{
-			input: "//localhost:5432/foo?password=" + postgresPass,
+			input:    "//localhost:5432/foo?password=" + postgresPass,
-			want:  result{pingOk: false, pingDeterminate: false},
+			wantPing: false,
 		},
 		{
-			input: "//localhost:5432/foo?sslmode=disable&password=foo",
+			input:    "//localhost:5432/foo?sslmode=disable&password=foo",
-			want:  result{pingOk: false, pingDeterminate: true},
+			wantPing: false,
 		},
 		{
-			input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
+			input:    "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
-			want:  result{pingOk: false, pingDeterminate: true},
+			wantPing: false,
 		},
 		{
-			input: "//badhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
+			input:   "invalid",
-			want:  result{pingOk: false, pingDeterminate: false},
+			wantErr: true,
-		},
-		{
-			input: "invalid",
-			want:  result{parseErr: true},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parsePostgres(tt.input)
-
+			if tt.wantErr {
-			if err != nil {
+				assert.Error(t, err)
-				got := result{parseErr: true}
-				assert.Equal(t, tt.want, got)
 				return
 			}
-
+			assert.NoError(t, err)
-			pr := j.ping(context.Background())
+			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
-
-			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
-			assert.Equal(t, tt.want, got)
 		})
 	}
 }

pkg/detectors/jdbc/sqlite.go

@@ -14,18 +14,12 @@ type sqliteJDBC struct {
 	testing  bool
 }
 
-var cannotVerifySqliteError error = errors.New("sqlite credentials cannot be verified")
+func (s *sqliteJDBC) ping(ctx context.Context) bool {
-
-func (s *sqliteJDBC) ping(ctx context.Context) pingResult {
 	if !s.testing {
 		// sqlite is not a networked database, so we cannot verify
-		return pingResult{cannotVerifySqliteError, true}
+		return false
 	}
-	return ping(ctx, "sqlite3", isSqliteErrorDeterminate, s.filename)
+	return ping(ctx, "sqlite3", s.filename)
-}
-
-func isSqliteErrorDeterminate(err error) bool {
-	return true
 }
 
 func parseSqlite(subname string) (jdbc, error) {

pkg/detectors/jdbc/sqlite_test.go

@@ -39,7 +39,7 @@ func TestParseSqlite(t *testing.T) {
 				assert.Error(t, err)
 			} else {
 				assert.NoError(t, err)
-				assert.True(t, j.ping(context.Background()).err == nil)
+				assert.True(t, j.ping(context.Background()))
 			}
 		})
 	}

pkg/detectors/jdbc/sqlserver.go

@@ -3,8 +3,9 @@ package jdbc
 import (
 	"context"
 	"errors"
-	mssql "github.com/denisenkom/go-mssqldb"
 	"strings"
+
+	_ "github.com/denisenkom/go-mssqldb"
 )
 
 type sqlServerJDBC struct {
@@ -12,25 +13,15 @@ type sqlServerJDBC struct {
 	params map[string]string
 }
 
-func (s *sqlServerJDBC) ping(ctx context.Context) pingResult {
+func (s *sqlServerJDBC) ping(ctx context.Context) bool {
-	return ping(ctx, "mssql", isSqlServerErrorDeterminate,
+	if ping(ctx, "mssql", s.conn) {
-		joinKeyValues(s.params, ";"),
+		return true
-		s.conn,
-		"sqlserver://"+s.conn)
-}
-
-func isSqlServerErrorDeterminate(err error) bool {
-	// Error numbers from https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors?view=sql-server-ver16
-	if mssqlError, isMssqlError := err.(mssql.Error); isMssqlError {
-		switch mssqlError.Number {
-		case 18456:
-			// Login failed
-			// This is a determinate failure iff we tried to use a real user
-			return mssqlError.Message != "login error: Login failed for user ''."
-		}
 	}
-
+	if ping(ctx, "mssql", joinKeyValues(s.params, ";")) {
-	return false
+		return true
+	}
+	// try URL format
+	return ping(ctx, "mssql", "sqlserver://"+s.conn)
 }
 
 func parseSqlServer(subname string) (jdbc, error) {

pkg/detectors/jdbc/sqlserver_integration_test.go

@@ -21,50 +21,33 @@ const (
 )
 
 func TestSqlServer(t *testing.T) {
-	type result struct {
-		parseErr        bool
-		pingOk          bool
-		pingDeterminate bool
-	}
 	tests := []struct {
-		input string
+		input    string
-		want  result
+		wantErr  bool
+		wantPing bool
 	}{
 		{
-			input: "",
+			input:   "",
-			want:  result{parseErr: true},
+			wantErr: true,
 		},
 		{
-			input: "//server=localhost;user id=sa;database=master;password=" + sqlServerPass,
+			input:    "//odbc:server=localhost;user id=sa;database=master;password=" + sqlServerPass,
-			want:  result{pingOk: true, pingDeterminate: true},
+			wantPing: true,
 		},
 		{
-			input: "//server=badhost;user id=sa;database=master;password=" + sqlServerPass,
+			input:    "//localhost;database= master;spring.datasource.password=" + sqlServerPass,
-			want:  result{pingOk: false, pingDeterminate: false},
+			wantPing: true,
-		},
-		{
-			input: "//localhost;database=master;spring.datasource.password=" + sqlServerPass,
-			want:  result{pingOk: true, pingDeterminate: true},
-		},
-		{
-			input: "//localhost;database=master;spring.datasource.password=badpassword",
-			want:  result{pingOk: false, pingDeterminate: true},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.input, func(t *testing.T) {
 			j, err := parseSqlServer(tt.input)
-
+			if tt.wantErr {
-			if err != nil {
+				assert.Error(t, err)
-				got := result{parseErr: true}
-				assert.Equal(t, tt.want, got)
 				return
 			}
-
+			assert.NoError(t, err)
-			pr := j.ping(context.Background())
+			assert.Equal(t, tt.wantPing, j.ping(context.Background()))
-
-			got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
-			assert.Equal(t, tt.want, got)
 		})
 	}
 }

pkg/detectors/launchdarkly/launchdarkly.go

@@ -20,13 +20,13 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly"}) + `\b([a-z0-9-]{40})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"launchdarkly", "launch_darkly"}
+	return []string{"launchdarkly"}
 }
 
 // FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.

pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5 * time.Second)
+	client = common.SaneHttpClientTimeOut(5)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)

pkg/detectors/okta/okta.go

@@ -19,8 +19,8 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
+	domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
-	tokenPat  = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
+	tokenPat  = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
 	// TODO: Oauth client secrets
 )
 

pkg/detectors/postgres/postgres.go

@@ -1,105 +0,0 @@
-package postgres
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct {
-	client *http.Client
-}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	defaultClient = common.SaneHttpClient()
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"postgres"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
-)
-
-// find various connection strings
-
-// postgresql://user:secret@localhost:5432/mydb
-// "host=localhost dbname=publishing user=www password=foo"
-
-// detectors.PrefixRegex([]string{"nytimes"})
-
-//var (
-//	connectionStringPats = map[string]*regexp.Regexp{
-//		"KeyValue":               regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
-//		"":              regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
-//		"Slack Workspace Access Token":  regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
-//		"Slack Workspace Refresh Token": regexp.MustCompile(`xoxr\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
-//	}
-//	verifyURL = "https://slack.com/api/auth.test"
-//)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"postgres://", ""}
-}
-
-// FromData will find and optionally verify Postgres secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, match := range matches {
-		if len(match) != 2 {
-			continue
-		}
-		resMatch := strings.TrimSpace(match[1])
-
-		s1 := detectors.Result{
-			DetectorType: detectorspb.DetectorType_Postgres,
-			Raw:          []byte(resMatch),
-		}
-
-		if verify {
-			client := s.client
-			if client == nil {
-				client = defaultClient
-			}
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.postgres.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
-			if err != nil {
-				continue
-			}
-			res, err := client.Do(req)
-			if err == nil {
-				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
-					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
-
-					if res.StatusCode != 401 {
-						s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
-					}
-				}
-			} else {
-				s1.VerificationError = err
-			}
-		}
-
-		results = append(results, s1)
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_Postgres
-}

pkg/detectors/postgres/postgres_test.go

@@ -1,160 +0,0 @@
-//go:build detectors
-// +build detectors
-
-package postgres
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestPostgres_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	secret := testSecrets.MustGetField("POSTGRES")
-	inactiveSecret := testSecrets.MustGetField("POSTGRES_INACTIVE")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name                string
-		s                   Scanner
-		args                args
-		want                []detectors.Result
-		wantErr             bool
-		wantVerificationErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     true,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:                nil,
-			wantErr:             false,
-			wantVerificationErr: false,
-		},
-		{
-			name: "found, would be verified if not for timeout",
-			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-		{
-			name: "found, verified but unexpected api surface",
-			s:    Scanner{client: common.ConstantStatusHttpClient(404)},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_Postgres,
-					Verified:     false,
-				},
-			},
-			wantErr:             false,
-			wantVerificationErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
-					t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
-				}
-				got[i].VerificationError = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/detectors/scrapersite/scrapersite.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(10 * time.Second)
+	client = common.SaneHttpClientTimeOut(10)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)

pkg/detectors/screenshotlayer/screenshotlayer.go

@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(10 * time.Second)
+	client = common.SaneHttpClientTimeOut(10)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"screenshotlayer"}) + `\b([a-zA-Z0-9_]{32})\b`)

pkg/detectors/trufflehogenterprise/trufflehogenterprise.go

@@ -1,108 +0,0 @@
-package trufflehogenterprise
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"regexp"
-	"strings"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type Scanner struct{}
-
-// Ensure the Scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-
-var (
-	client = common.SaneHttpClient()
-
-	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat      = regexp.MustCompile(`\bthog-key-[0-9a-f]{16}\b`)
-	secretPat   = regexp.MustCompile(`\bthog-secret-[0-9a-f]{32}\b`)
-	hostnamePat = regexp.MustCompile(`\b[a-z]+-[a-z]+-[a-z]+\.[a-z][0-9]\.[a-z]+\.trufflehog\.org\b`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s Scanner) Keywords() []string {
-	return []string{"thog"}
-}
-
-// FromData will find and optionally verify Trufflehog Enterprise secrets in a given set of bytes.
-func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
-	dataStr := string(data)
-
-	keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
-	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
-	hostnameMatches := hostnamePat.FindAllStringSubmatch(dataStr, -1)
-
-	for _, keyMatch := range keyMatches {
-		if len(keyMatch) != 1 {
-			continue
-		}
-		resKeyMatch := strings.TrimSpace(keyMatch[0])
-		for _, secretMatch := range secretMatches {
-
-			if len(secretMatch) != 1 {
-				continue
-			}
-			resSecretMatch := strings.TrimSpace(secretMatch[0])
-
-			for _, hostnameMatch := range hostnameMatches {
-				if len(hostnameMatch) != 1 {
-					continue
-				}
-
-				resHostnameMatch := strings.TrimSpace(hostnameMatch[0])
-
-				s1 := detectors.Result{
-					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
-					Raw:          []byte(resKeyMatch),
-				}
-
-				if verify {
-					endpoint := fmt.Sprintf("https://%s/api/v1/sources", resHostnameMatch)
-					req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
-					if err != nil {
-						continue
-					}
-
-					req.Header.Add("Accept", "application/vnd.trufflehogenterprise+json; version=3")
-					req.Header.Add("X-Thog-Secret", resSecretMatch)
-					req.Header.Add("X-Thog-Key", resKeyMatch)
-
-					res, err := client.Do(req)
-					if err == nil {
-						verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "data")
-						if err != nil {
-							return nil, err
-						}
-
-						defer res.Body.Close()
-
-						if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
-							s1.Verified = true
-						} else {
-							// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-							if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
-								continue
-							}
-						}
-					}
-				}
-
-				results = append(results, s1)
-			}
-		}
-	}
-
-	return results, nil
-}
-
-func (s Scanner) Type() detectorspb.DetectorType {
-	return detectorspb.DetectorType_TrufflehogEnterprise
-}

pkg/detectors/trufflehogenterprise/trufflehogenterprise_test.go

@@ -1,121 +0,0 @@
-//go:build detectors
-// +build detectors
-
-package trufflehogenterprise
-
-import (
-	"context"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kylelemons/godebug/pretty"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-func TestTrufflehogenterprise_FromChunk(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
-	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
-	if err != nil {
-		t.Fatalf("could not get test secrets from GCP: %s", err)
-	}
-	host := testSecrets.MustGetField("THOG_HOSTNAME")
-	key := testSecrets.MustGetField("THOG_WEB_KEY")
-	secret := testSecrets.MustGetField("THOG_WEB_SECRET")
-	inactiveSecret := testSecrets.MustGetField("THOG_WEB_INACTIVE_SECRET")
-
-	type args struct {
-		ctx    context.Context
-		data   []byte
-		verify bool
-	}
-	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
-	}{
-		{
-			name: "found, verified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a thog secret %s for %s with key %s within", secret, host, key)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
-					Verified:     true,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "found, unverified",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a trufflehogenterprise secret %s for %s with key %s within but not valid", inactiveSecret, host, key)), // the secret would satisfy the regex but not pass validation
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
-					Verified:     false,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte("You cannot find the secret within"),
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("Trufflehogenterprise.FromData() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			for i := range got {
-				if len(got[i].Raw) == 0 {
-					t.Fatalf("no raw secret present: \n %+v", got[i])
-				}
-				got[i].Raw = nil
-			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Trufflehogenterprise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-			}
-		})
-	}
-}
-
-func BenchmarkFromData(benchmark *testing.B) {
-	ctx := context.Background()
-	s := Scanner{}
-	for name, data := range detectors.MustGetBenchmarkData() {
-		benchmark.Run(name, func(b *testing.B) {
-			for n := 0; n < b.N; n++ {
-				_, err := s.FromData(ctx, false, data)
-				if err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}

pkg/detectors/twilio/twilio.go

@@ -3,7 +3,9 @@ package twilio
 import (
 	"context"
 	"net/http"
+	"net/url"
 	"regexp"
+	"strings"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -51,8 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			if verify {
+
+				form := url.Values{}
+				form.Add("FriendlyName", "MyServiceName")
 				req, err := http.NewRequestWithContext(
-					ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
+					ctx, "POST", "https://verify.twilio.com/v2/Services",
+
+					strings.NewReader(form.Encode()),
+				)
 				if err != nil {
 					continue
 				}

pkg/detectors/uri/uri.go

@@ -53,8 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		password := match[1]
 
 		// Skip findings where the password only has "*" characters, this is a redacted password
-		// Also include the url encoded "*" characters: "%2A"
+		if strings.Trim(password, "*") == "" {
-		if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
 			continue
 		}
 

pkg/detectors/uri/uri_test.go

@@ -31,14 +31,14 @@ func TestURI_FromChunk(t *testing.T) {
 			s:    Scanner{},
 			args: args{
 				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
+				data:   []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
 				verify: true,
 			},
 			want: []detectors.Result{
 				{
 					DetectorType: detectorspb.DetectorType_URI,
 					Verified:     false,
-					Redacted:     "https://user:********@httpwatch.com",
+					Redacted:     "https://user:********@www.httpwatch.com",
 				},
 			},
 			wantErr: false,
@@ -87,17 +87,6 @@ func TestURI_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
-		{
-			name: "nothing found, password was redacted two different ways",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("Both %s and %s have been redacted within", "https://httpwatch::********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar", "https://httpwatch::%2A%2A%2A%2A%2A%2A@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")),
-				verify: true,
-			},
-			want:    []detectors.Result{},
-			wantErr: false,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/detectors/zenserp/zenserp.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +18,7 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 
 var (
-	client = common.SaneHttpClientTimeOut(5 * time.Second)
+	client = common.SaneHttpClientTimeOut(5)
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"zenserp"}) + `\b([0-9a-z-]{36})\b`)

pkg/engine/defaults.go

@@ -1,8 +1,6 @@
 package engine
 
 import (
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
@@ -161,7 +159,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convertkit"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convier"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/copper"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/couchbase"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/countrylayer"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/courier"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coveralls"
@@ -201,7 +198,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/disqus"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ditto"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dnscheck"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dockerhub"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docparser"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/documo"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docusign"
@@ -1527,9 +1523,6 @@ func DefaultDetectors() []detectors.Detector {
 		prefect.Scanner{},
 		buildkitev2.Scanner{},
 		opsgenie.Scanner{},
-		dockerhub.Scanner{},
-		couchbase.Scanner{},
-		envoyapikey.Scanner{},
 	}
 
 }

pkg/engine/engine.go

@@ -113,7 +113,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		chunks:          make(chan *sources.Chunk),
 		results:         make(chan detectors.ResultWithMetadata),
 		detectorAvgTime: sync.Map{},
-		sourcesWg:       &errgroup.Group{},
 	}
 
 	for _, option := range options {
@@ -128,8 +127,10 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 	ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
 
-	// Limit number of concurrent goroutines dedicated to chunking a source.
+	sourcesWg, egCtx := errgroup.WithContext(ctx)
-	e.sourcesWg.SetLimit(e.concurrency)
+	sourcesWg.SetLimit(e.concurrency)
+	e.sourcesWg = sourcesWg
+	ctx.SetParent(egCtx)
 
 	if len(e.decoders) == 0 {
 		e.decoders = decoders.DefaultDecoders()
@@ -184,7 +185,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	for i := 0; i < e.concurrency; i++ {
 		e.workersWg.Add(1)
 		go func() {
-			defer common.Recover(ctx)
+			defer common.RecoverWithExit(ctx)
 			defer e.workersWg.Done()
 			e.detectorWorker(ctx)
 		}()
@@ -377,23 +378,31 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 	}
 }
 
-// SupportsLineNumbers determines if a line number can be found for a source type.
+// lineNumberSupportedSources is a list of sources that support line numbers.
-func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
+// It is stored this way because slice consts are not supported.
-	switch sourceType {
+func lineNumberSupportedSources() []sourcespb.SourceType {
-	case sourcespb.SourceType_SOURCE_TYPE_GIT,
+	return []sourcespb.SourceType{
+		sourcespb.SourceType_SOURCE_TYPE_GIT,
 		sourcespb.SourceType_SOURCE_TYPE_GITHUB,
 		sourcespb.SourceType_SOURCE_TYPE_GITLAB,
 		sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
 		sourcespb.SourceType_SOURCE_TYPE_GERRIT,
 		sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
 		sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
-		sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
+		sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
-		return true
-	default:
-		return false
 	}
 }
 
+// SupportsLineNumbers determines if a line number can be found for a source type.
+func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
+	for _, i := range lineNumberSupportedSources() {
+		if i == sourceType {
+			return true
+		}
+	}
+	return false
+}
+
 // FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
 func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
 	lines := bytes.Split(chunk.Data, []byte("\n"))

pkg/engine/engine_test.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
@@ -73,30 +72,3 @@ func TestDefaultDecoders(t *testing.T) {
 		t.Errorf("DefaultDecoders() = %v, expected UTF8 decoder to be first", ds)
 	}
 }
-
-func TestSupportsLineNumbers(t *testing.T) {
-	testCases := []struct {
-		name     string
-		input    sourcespb.SourceType
-		expected bool
-	}{
-		{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
-		{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
-		{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			if result := SupportsLineNumbers(tc.input); result != tc.expected {
-				t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
-			}
-		})
-	}
-}
-
-func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
-	sourceType := sourcespb.SourceType_SOURCE_TYPE_GITHUB
-	for i := 0; i < b.N; i++ {
-		_ = SupportsLineNumbers(sourceType)
-	}
-}

pkg/engine/filesystem.go

@@ -28,22 +28,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 	}
 
 	fileSystemSource := filesystem.Source{}
-	fileSystemSource.WithFilter(c.Filter)
-
-	// TODO: This is how it would be used; everything after these comments would be removed.
-	// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
-	// 	func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-	// 		fileSystemSource := filesystem.Source{}
-	// 		fileSystemSource.WithFilter(c.Filter)
-	// 		if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-	// 			return nil, err
-	// 		}
-	// 		return &fileSystemSource, nil
-	// 	})
-	// if err != nil {
-	// 	return err
-	// }
-	// return e.sourceManager.ScheduleRun(ctx, handle)
 
 	ctx = context.WithValues(ctx,
 		"source_type", fileSystemSource.Type().String(),
@@ -53,6 +37,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 	if err != nil {
 		return errors.WrapPrefix(err, "could not init filesystem source", 0)
 	}
+	fileSystemSource.WithFilter(c.Filter)
 	e.sourcesWg.Go(func() error {
 		defer common.RecoverWithExit(ctx)
 		err := fileSystemSource.Chunks(ctx, e.ChunksChan())

pkg/engine/git_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -16,7 +15,6 @@ import (
 type expResult struct {
 	B          string
 	LineNumber int64
-	Verified   bool
 }
 
 func TestGitEngine(t *testing.T) {
@@ -40,16 +38,16 @@ func TestGitEngine(t *testing.T) {
 	for tName, tTest := range map[string]testProfile{
 		"all_secrets": {
 			expected: map[string]expResult{
-				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
+				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
-				"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4, true},
+				"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4},
-				"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3, false},
+				"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
-				"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7, false},
+				"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
 			},
 			filter: common.FilterEmpty(),
 		},
 		"base_commit": {
 			expected: map[string]expResult{
-				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
+				"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
 			},
 			filter: common.FilterEmpty(),
 			base:   "2f251b8c1e72135a375b659951097ec7749d4af9",
@@ -59,12 +57,8 @@ func TestGitEngine(t *testing.T) {
 			e := Start(ctx,
 				WithConcurrency(1),
 				WithDecoders(decoders.DefaultDecoders()...),
-				WithDetectors(true, DefaultDetectors()...),
+				WithDetectors(false, DefaultDetectors()...),
 			)
-			// Make the channels buffered so Finish returns.
-			e.chunks = make(chan *sources.Chunk, 10)
-			e.results = make(chan detectors.ResultWithMetadata, 10)
-
 			cfg := sources.GitConfig{
 				RepoPath: path,
 				HeadRef:  tTest.branch,
@@ -79,8 +73,7 @@ func TestGitEngine(t *testing.T) {
 			logFatalFunc := func(_ error, _ string, _ ...any) {
 				t.Fatalf("error logging function should not have been called")
 			}
-			// Wait for all the chunks to be processed.
+			go e.Finish(ctx, logFatalFunc)
-			e.Finish(ctx, logFatalFunc)
 			resultCount := 0
 			for result := range e.ResultsChan() {
 				switch meta := result.SourceMetadata.GetData().(type) {
@@ -91,9 +84,6 @@ func TestGitEngine(t *testing.T) {
 					if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
 						t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
 					}
-					if tTest.expected[meta.Git.Commit].Verified != result.Verified {
-						t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
-					}
 				}
 				resultCount++
 

pkg/pb/detectorspb/detectors.pb.go

@@ -992,9 +992,6 @@ const (
 	DetectorType_Prefect                       DetectorType = 918
 	DetectorType_Docusign                      DetectorType = 919
 	DetectorType_Couchbase                     DetectorType = 920
-	DetectorType_Dockerhub                     DetectorType = 921
-	DetectorType_TrufflehogEnterprise          DetectorType = 922
-	DetectorType_EnvoyApiKey                   DetectorType = 923
 )
 
 // Enum value maps for DetectorType.
@@ -1917,9 +1914,6 @@ var (
 		918: "Prefect",
 		919: "Docusign",
 		920: "Couchbase",
-		921: "Dockerhub",
-		922: "TrufflehogEnterprise",
-		923: "EnvoyApiKey",
 	}
 	DetectorType_value = map[string]int32{
 		"Alibaba":                       0,
@@ -2839,9 +2833,6 @@ var (
 		"Prefect":                       918,
 		"Docusign":                      919,
 		"Couchbase":                     920,
-		"Dockerhub":                     921,
-		"TrufflehogEnterprise":          922,
-		"EnvoyApiKey":                   923,
 	}
 )
 
@@ -3220,7 +3211,7 @@ var file_detectors_proto_rawDesc = []byte{
 	0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
 	0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
 	0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
-	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc6, 0x73, 0x0a, 0x0c, 0x44,
+	0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a, 0x0c, 0x44,
 	0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
 	0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
 	0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4141,15 +4132,11 @@ var file_detectors_proto_rawDesc = []byte{
 	0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x72, 0x65,
 	0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63, 0x75, 0x73,
 	0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63, 0x68, 0x62,
-	0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72,
+	0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
-	0x68, 0x75, 0x62, 0x10, 0x99, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x75, 0x66, 0x66, 0x6c,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75,
-	0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
+	0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f,
-	0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
+	0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
-	0x10, 0x9b, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
+	0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-	0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
-	0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
-	0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
-	0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (

pkg/sources/filesystem/filesystem.go

@@ -46,7 +46,6 @@ type Source struct {
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 var _ sources.SourceUnitEnumerator = (*Source)(nil)
-var _ sources.SourceUnitChunker = (*Source)(nil)
 
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
@@ -221,53 +220,12 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
 // Enumerate implements SourceUnitEnumerator interface. This implementation simply
 // passes the configured paths as the source unit, whether it be a single
 // filepath or a directory.
-func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
+func (s *Source) Enumerate(ctx context.Context, units chan<- sources.EnumerationResult) error {
 	for _, path := range s.paths {
-		item := sources.CommonSourceUnit{ID: path}
+		item := sources.CommonEnumerationOk(path)
-		if err := reporter.UnitOk(ctx, item); err != nil {
+		if err := common.CancellableWrite(ctx, units, item); err != nil {
 			return err
 		}
 	}
 	return nil
 }
-
-// ChunkUnit implements SourceUnitChunker interface.
-func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
-	path := unit.SourceUnitID()
-	logger := ctx.Logger().WithValues("path", path)
-
-	cleanPath := filepath.Clean(path)
-	fileInfo, err := os.Stat(cleanPath)
-	if err != nil {
-		return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
-	}
-
-	ch := make(chan *sources.Chunk)
-	var scanErr error
-	go func() {
-		defer close(ch)
-		if fileInfo.IsDir() {
-			// TODO: Finer grain error tracking of individual chunks.
-			scanErr = s.scanDir(ctx, cleanPath, ch)
-		} else {
-			// TODO: Finer grain error tracking of individual
-			// chunks (in the case of archives).
-			scanErr = s.scanFile(ctx, cleanPath, ch)
-		}
-	}()
-
-	for chunk := range ch {
-		if chunk == nil {
-			continue
-		}
-		if err := reporter.ChunkOk(ctx, *chunk); err != nil {
-			return err
-		}
-	}
-
-	if scanErr != nil && scanErr != io.EOF {
-		logger.Info("error scanning filesystem", "error", scanErr)
-		return reporter.ChunkErr(ctx, scanErr)
-	}
-	return nil
-}

pkg/sources/git/git.go

@@ -755,7 +755,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
 		}
 		path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username(), "--shallow-since", timestamp)
 		if err != nil {
-			return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
+			return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
 		}
 	default:
 		ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
@@ -793,7 +793,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
 			}
 			path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username())
 			if err != nil {
-				return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
+				return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
 			}
 		default:
 			ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)

pkg/sources/source_manager.go

@@ -1,164 +0,0 @@
-package sources
-
-import (
-	"fmt"
-	"sync"
-	"sync/atomic"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"golang.org/x/sync/errgroup"
-)
-
-// handle uniquely identifies a Source given to the manager to manage. If the
-// SourceManager is connected to the API, it will be equivalent to the unique
-// source ID, otherwise it behaves as a counter.
-type handle int64
-
-// SourceInitFunc is a function that takes a source and job ID and returns an
-// initialized Source.
-type SourceInitFunc func(sourceID int64, jobID int64) (Source, error)
-
-type SourceManager struct {
-	api apiClient
-	// Map of handle to source initializer.
-	handles     map[handle]SourceInitFunc
-	handlesLock sync.Mutex
-	// Pool limiting the amount of concurrent sources running.
-	pool errgroup.Group
-	// Downstream chunks channel to be scanned.
-	outputChunks chan *Chunk
-}
-
-// apiClient is an interface for optionally communicating with an external API.
-type apiClient interface {
-	// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
-	RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
-	// GetJobID queries the API for an existing job or new job ID.
-	GetJobID(ctx context.Context, id int64) (int64, error)
-}
-
-// WithAPI adds an API client to the manager for tracking jobs and progress.
-func WithAPI(api apiClient) func(*SourceManager) {
-	return func(man *SourceManager) { man.api = api }
-}
-
-// WithConcurrency limits the concurrent number of sources a manager can run.
-func WithConcurrency(concurrency int) func(*SourceManager) {
-	return func(man *SourceManager) { man.pool.SetLimit(concurrency) }
-}
-
-// NewManager creates a new manager with the provided options.
-func NewManager(outputChunks chan *Chunk, opts ...func(*SourceManager)) *SourceManager {
-	man := SourceManager{
-		// Default to the headless API. Can be overwritten by the WithAPI option.
-		api:          &headlessAPI{},
-		handles:      make(map[handle]SourceInitFunc),
-		outputChunks: outputChunks,
-	}
-	for _, opt := range opts {
-		opt(&man)
-	}
-	return &man
-}
-
-// Enroll informs the SourceManager to track and manage a Source.
-func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
-	id, err := s.api.RegisterSource(ctx, name, kind)
-	if err != nil {
-		return 0, err
-	}
-	handleID := handle(id)
-	s.handlesLock.Lock()
-	defer s.handlesLock.Unlock()
-	if _, ok := s.handles[handleID]; ok {
-		// TODO: smartly handle this?
-		return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
-	}
-	s.handles[handleID] = f
-	return 0, nil
-}
-
-// Run blocks until a resource is available to run the source, then synchronously runs it.
-func (s *SourceManager) Run(ctx context.Context, handle handle) error {
-	// Check the handle is valid before waiting on the pool.
-	if _, ok := s.getInitFunc(handle); !ok {
-		return fmt.Errorf("unrecognized handle")
-	}
-	if err := ctx.Err(); err != nil {
-		return err
-	}
-	ch := make(chan error)
-	s.pool.Go(func() error {
-		defer common.Recover(ctx)
-		// TODO: The manager should record these errors.
-		ch <- s.run(ctx, handle)
-		return nil
-	})
-	return <-ch
-}
-
-// ScheduleRun blocks until a resource is available to run the source, then
-// asynchronously runs it. Error information is lost in this case.
-func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
-	// Check the handle is valid before waiting on the pool.
-	if _, ok := s.getInitFunc(handle); !ok {
-		return fmt.Errorf("unrecognized handle")
-	}
-	if err := ctx.Err(); err != nil {
-		return err
-	}
-	s.pool.Go(func() error {
-		defer common.Recover(ctx)
-		// TODO: The manager should record these errors.
-		_ = s.run(ctx, handle)
-		return nil
-	})
-	// TODO: Maybe wait for a signal here that initialization was successful?
-	return nil
-}
-
-// run is a helper method to sychronously run the source. It does not check for
-// acquired resources.
-func (s *SourceManager) run(ctx context.Context, handle handle) error {
-	jobID, err := s.api.GetJobID(ctx, int64(handle))
-	if err != nil {
-		return err
-	}
-	initFunc, ok := s.getInitFunc(handle)
-	if !ok {
-		return fmt.Errorf("unrecognized handle")
-	}
-	source, err := initFunc(jobID, int64(handle))
-	if err != nil {
-		return err
-	}
-	// TODO: Support UnitChunker and SourceUnitEnumerator.
-	// TODO: This is where we can introspect on the chunks collected.
-	return source.Chunks(ctx, s.outputChunks)
-}
-
-// getInitFunc is a helper method for safe concurrent access to the
-// map[handle]SourceInitFunc map.
-func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
-	s.handlesLock.Lock()
-	defer s.handlesLock.Unlock()
-	f, ok := s.handles[handle]
-	return f, ok
-}
-
-// headlessAPI implements the apiClient interface locally.
-type headlessAPI struct {
-	// Counters for assigning handle and job IDs.
-	sourceIDCounter int64
-	jobIDCounter    int64
-}
-
-func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
-	return atomic.AddInt64(&api.sourceIDCounter, 1), nil
-}
-
-func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
-	return atomic.AddInt64(&api.jobIDCounter, 1), nil
-}

pkg/sources/sources.go

@@ -53,41 +53,21 @@ type SourceUnitUnmarshaller interface {
 // SourceUnitEnumerator defines an optional interface a Source can implement to
 // support enumerating an initialized Source into SourceUnits.
 type SourceUnitEnumerator interface {
-	// Enumerate creates 0 or more units from an initialized source,
+	// Enumerate enumerates the initialized Source, outputting units. This
-	// reporting them or any errors to the UnitReporter. This method is
+	// method is synchronous but can be called in a goroutine to support
-	// synchronous but can be called in a goroutine to support concurrent
+	// concurrent enumeration and chunking. An error should only be
-	// enumeration and chunking. An error should only be returned from this
+	// returned from this method in the case of context cancellation. All
-	// method in the case of context cancellation, fatal source errors, or
+	// other errors related to unit enumeration are tracked in the
-	// errors returned by the reporter All other errors related to unit
+	// EnumerationResult.
-	// enumeration are tracked by the UnitReporter.
+	Enumerate(ctx context.Context, units chan<- EnumerationResult) error
-	Enumerate(ctx context.Context, reporter UnitReporter) error
 }
 
-// UnitReporter defines the interface a source will use to report whether a
+// EnumerationResult is the result of an enumeration, containing the unit and
-// unit was found during enumeration. Either method may be called any number of
+// error if any. Unit and Error are mutually exclusive (only one will be
-// times. Implementors of this interface should allow for concurrent calls.
+// non-nil).
-type UnitReporter interface {
+type EnumerationResult struct {
-	UnitOk(ctx context.Context, unit SourceUnit) error
+	Unit  SourceUnit
-	UnitErr(ctx context.Context, err error) error
+	Error error
-}
-
-// SourceUnitChunker defines an optional interface a Source can implement to
-// support chunking a single SourceUnit.
-type SourceUnitChunker interface {
-	// ChunkUnit creates 0 or more chunks from a unit, reporting them or
-	// any errors to the ChunkReporter. An error should only be returned
-	// from this method in the case of context cancellation, fatal source
-	// errors, or errors returned by the reporter. All other errors related
-	// to unit chunking are tracked by the ChunkReporter.
-	ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
-}
-
-// ChunkReporter defines the interface a source will use to report whether a
-// chunk was found during unit chunking. Either method may be called any number
-// of times. Implementors of this interface should allow for concurrent calls.
-type ChunkReporter interface {
-	ChunkOk(ctx context.Context, chunk Chunk) error
-	ChunkErr(ctx context.Context, err error) error
 }
 
 // SourceUnit is an object that represents a Source's unit of work. This is
@@ -263,3 +243,16 @@ func (p *Progress) GetProgress() *Progress {
 	defer p.mut.Unlock()
 	return p
 }
+
+// CommonEnumerationOk is a helper function to construct an EnumerationResult
+// using a CommonSourceUnit.
+func CommonEnumerationOk(id string) EnumerationResult {
+	unit := CommonSourceUnit{ID: id}
+	return EnumerationResult{Unit: unit}
+}
+
+// EnumerationErr is a helper function to construct an EnumerationResult from
+// an error.
+func EnumerationErr(err error) EnumerationResult {
+	return EnumerationResult{Error: err}
+}

proto/detectors.proto

@@ -929,10 +929,7 @@ enum DetectorType {
   Prefect = 918;
   Docusign = 919;
   Couchbase = 920;
-  Dockerhub = 921;
+  IBMCloud = 923; // 921 and 922 are reserved for Trufflehog Web and Scanner Detectors
-  TrufflehogEnterprise = 922;
-  EnvoyApiKey = 923;
-  Postgres = 925;
 }
 
 message Result {