Compare commits

..

2 Commits

Author SHA1 Message Date
Zubair Khan a09c0f265f save ibm progress 2023-07-17 16:59:38 -04:00
Zubair Khan f0db55c548 init 2023-07-12 15:11:20 -04:00
54 changed files with 313 additions and 1595 deletions
+3 -2
View File
@@ -8,8 +8,9 @@ updates:
- package-ecosystem: "gomod" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "monthly"
interval: "weekly"
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
# Check for updates to GitHub Actions every weekday
interval: "weekly"
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.20'
go-version: '1.18'
- name: Checkout code
uses: actions/checkout@v3
+1 -1
View File
@@ -11,7 +11,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.20'
go-version: '1.18'
- name: Checkout code
uses: actions/checkout@v3
- name: Smoke
+9 -9
View File
@@ -22,7 +22,7 @@ require (
github.com/felixge/fgprof v0.9.3
github.com/getsentry/sentry-go v0.22.0
github.com/go-errors/errors v1.4.2
github.com/go-git/go-git/v5 v5.8.0
github.com/go-git/go-git/v5 v5.7.0
github.com/go-ldap/ldap/v3 v3.4.5
github.com/go-logr/logr v1.2.4
github.com/go-logr/zapr v1.2.4
@@ -33,7 +33,7 @@ require (
github.com/google/go-cmp v0.5.9
github.com/google/go-containerregistry v0.14.0
github.com/google/go-github/v42 v42.0.0
github.com/googleapis/gax-go/v2 v2.12.0
github.com/googleapis/gax-go/v2 v2.11.0
github.com/hashicorp/go-retryablehttp v0.7.4
github.com/jlaffaye/ftp v0.2.0
github.com/joho/godotenv v1.5.1
@@ -56,10 +56,10 @@ require (
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.11.0
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
golang.org/x/oauth2 v0.10.0
golang.org/x/oauth2 v0.9.0
golang.org/x/sync v0.3.0
golang.org/x/text v0.11.0
google.golang.org/api v0.131.0
google.golang.org/api v0.130.0
google.golang.org/protobuf v1.31.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
@@ -68,7 +68,7 @@ require (
require (
cloud.google.com/go v0.110.2 // indirect
cloud.google.com/go/compute v1.20.1 // indirect
cloud.google.com/go/compute v1.19.3 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -170,10 +170,10 @@ require (
golang.org/x/tools v0.10.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
google.golang.org/grpc v1.56.2 // indirect
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
google.golang.org/grpc v1.56.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
+18 -18
View File
@@ -11,8 +11,8 @@ cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
@@ -178,8 +178,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
@@ -281,8 +281,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki
github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
@@ -574,8 +574,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -698,8 +698,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ=
google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -722,12 +722,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -740,8 +740,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+3 -19
View File
@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
1. Add the test secret to GCP Secrets. See [managing test secrets](#managing-test-secrets)
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
4. Update the tests with these test cases at minimum:
1. Found and verified (using a credential loaded from GCP Secrets)
2. Found and unverified (determinately, i.e. the secret is invalid)
3. Found and unverified (indeterminately due to timeout)
4. Found and unverified (indeterminately due to an unexpected API response)
5. Not found
6. Any false positive cases that you come across
2. Found and unverified
3. Not found
4. Any false positive cases that you come across
5. Create a merge request for review. CI tests must be passing.
## Addendum
### Verification indeterminacy
There are two types of reasons that secret verification can fail:
* The candidate secret is not actually a valid secret.
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
### Managing Test Secrets
Do not embed test credentials in the test code. Instead, use GCP Secrets Manager.
+3 -19
View File
@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
4. Update the tests with these test cases at minimum:
1. Found and verified (using a credential loaded from GCP Secrets)
2. Found and unverified (determinately, i.e. the secret is invalid)
3. Found and unverified (indeterminately due to timeout)
4. Found and unverified (indeterminately due to an unexpected API response)
5. Not found
6. Any false positive cases that you come across
2. Found and unverified
3. Not found
4. Any false positive cases that you come across
5. Create a pull request for review.
## Addendum
### Verification indeterminacy
There are two types of reasons that secret verification can fail:
* The candidate secret is not actually a valid secret.
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
### Using a test secret file
1. Create a file called `.env` with this env file format:
+4 -4
View File
@@ -34,14 +34,14 @@ func main() {
case "detector":
mustWriteTemplates([]templateJob{
{
TemplatePath: "pkg/detectors/alchemy/alchemy.go",
TemplatePath: "pkg/detectors/heroku/heroku.go",
WritePath: filepath.Join(folderPath(), nameLower+".go"),
ReplaceString: []string{"alchemy"},
ReplaceString: []string{"heroku"},
},
{
TemplatePath: "pkg/detectors/alchemy/alchemy_test.go",
TemplatePath: "pkg/detectors/heroku/heroku_test.go",
WritePath: filepath.Join(folderPath(), nameLower+"_test.go"),
ReplaceString: []string{"alchemy"},
ReplaceString: []string{"heroku"},
},
})
// case "source":
+1 -1
View File
@@ -207,7 +207,7 @@ func run(state overseer.State) {
if *profile {
go func() {
router := http.NewServeMux()
router.Handle("/debug/pprof/", http.DefaultServeMux)
router.Handle("/debug/pprof", http.DefaultServeMux)
router.Handle("/debug/fgprof", fgprof.Handler())
logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
if err := http.ListenAndServe(":18066", router); err != nil {
+2 -26
View File
@@ -3,7 +3,6 @@ package common
import (
"crypto/tls"
"crypto/x509"
"io"
"net"
"net/http"
"strings"
@@ -76,14 +75,6 @@ func PinnedCertPool() *x509.CertPool {
return trustedCerts
}
type FakeTransport struct {
CreateResponse func(req *http.Request) (*http.Response, error)
}
func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return t.CreateResponse(req)
}
type CustomTransport struct {
T http.RoundTripper
}
@@ -100,21 +91,6 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
return &CustomTransport{T}
}
func ConstantStatusHttpClient(statusCode int) *http.Client {
return &http.Client{
Timeout: DefaultResponseTimeout,
Transport: FakeTransport{
CreateResponse: func(req *http.Request) (*http.Response, error) {
return &http.Response{
Request: req,
Body: io.NopCloser(strings.NewReader("")),
StatusCode: statusCode,
}, nil
},
},
}
}
func PinnedRetryableHttpClient() *http.Client {
httpClient := retryablehttp.NewClient()
httpClient.Logger = nil
@@ -176,9 +152,9 @@ func SaneHttpClient() *http.Client {
}
// SaneHttpClientTimeOut adds a custom timeout for some scanners
func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
httpClient := &http.Client{}
httpClient.Timeout = timeout
httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
httpClient.Transport = NewCustomTransport(nil)
return httpClient
}
+14 -3
View File
@@ -18,11 +18,22 @@ var (
type Context interface {
context.Context
Logger() logr.Logger
Parent() context.Context
SetParent(ctx context.Context) Context
}
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
// the same types.
type CancelFunc = context.CancelFunc
// Parent returns the parent context.
func (l logCtx) Parent() context.Context {
return l.Context
}
// SetParent sets the parent context on the context.
func (l logCtx) SetParent(ctx context.Context) Context {
l.Context = ctx
return l
}
type CancelFunc context.CancelFunc
// logCtx implements Context.
type logCtx struct {
+3 -15
View File
@@ -2,7 +2,6 @@ package alchemy
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -12,15 +11,14 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alchemy"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
)
@@ -49,10 +47,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.alchemy.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
if err != nil {
continue
@@ -67,13 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
if res.StatusCode != 401 {
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
}
} else {
s1.VerificationError = err
}
}
+11 -52
View File
@@ -32,12 +32,11 @@ func TestAlchemy_FromChunk(t *testing.T) {
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
@@ -53,8 +52,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "found, unverified",
@@ -70,8 +68,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
wantErr: false,
},
{
name: "not found",
@@ -81,48 +78,14 @@ func TestAlchemy_FromChunk(t *testing.T) {
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Alchemy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantStatusHttpClient(404)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Alchemy,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Alchemy.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -132,10 +95,6 @@ func TestAlchemy_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
}
got[i].VerificationError = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
-2
View File
@@ -184,8 +184,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
}
} else {
s1.VerificationError = err
}
res.Body.Close()
} else {
-95
View File
@@ -1,95 +0,0 @@
package dockerhub
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Can use email or username for login.
usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
emailPat = regexp.MustCompile(common.EmailPattern)
// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
accessTokenPat = regexp.MustCompile(`\bdckr_pat_([a-zA-Z0-9_-]){27}\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"dckr_pat_"}
}
// FromData will find and optionally verify Dockerhub secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
emailMatches := emailPat.FindAllString(dataStr, -1)
dataStr = emailPat.ReplaceAllString(dataStr, "")
usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
accessTokenMatches := accessTokenPat.FindAllString(dataStr, -1)
userMatches := emailMatches
for _, usernameMatch := range usernameMatches {
if len(usernameMatch) > 1 {
userMatches = append(userMatches, usernameMatch[1])
}
}
for _, resUserMatch := range userMatches {
for _, resAccessTokenMatch := range accessTokenMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Dockerhub,
Raw: []byte(fmt.Sprintf("%s: %s", resUserMatch, resAccessTokenMatch)),
}
if verify {
payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, resUserMatch, resAccessTokenMatch))
req, err := http.NewRequestWithContext(ctx, "GET", "https://hub.docker.com/v2/users/login", payload)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
body, err := io.ReadAll(res.Body)
if err != nil {
continue
}
// Valid credentials can still return a 401 status code if 2FA is enabled
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 401 && strings.Contains(string(body), "login_2fa_token")) {
s1.Verified = true
}
}
}
results = append(results, s1)
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Dockerhub
}
-138
View File
@@ -1,138 +0,0 @@
//go:build detectors
// +build detectors
package dockerhub
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestDockerhub_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
username := testSecrets.MustGetField("DOCKERHUB_USERNAME")
email := testSecrets.MustGetField("DOCKERHUB_EMAIL")
pat := testSecrets.MustGetField("DOCKERHUB_PAT")
inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, verified (email)",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Dockerhub,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -2
View File
@@ -7,7 +7,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(5 * time.Second)
client = common.SaneHttpClientTimeOut(5)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getemail"}) + `\b([a-zA-Z0-9-]{20})\b`)
@@ -1,8 +1,8 @@
package envoyapikey
package ibmcloud
import (
"context"
"io"
"fmt"
"net/http"
"regexp"
"strings"
@@ -21,16 +21,16 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"envoy"}) + `\b([a-zA-Z0-9]{220})\b`)
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_-]{44})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"envoy"}
return []string{"ibm"}
}
// FromData will find and optionally verify Envoy secrets in a given set of bytes.
// FromData will find and optionally verify IBM Cloud secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -43,27 +43,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
DetectorType: detectorspb.DetectorType_Ibmcloud,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.envoy.com/v1/locations", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ibmcloud.com/apps", nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/vnd.envoy+json; version=3")
req.Header.Add("X-Api-Key", resMatch)
req.Header.Add("Accept", "application/vnd.ibmcloud+json; version=3")
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
// Invalid API keys can also return status code 200, so check for presence of 'status 401' in response body.
if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == 403 {
if !strings.Contains(string(body), `"status":401`) {
s1.Verified = true
}
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -80,5 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_EnvoyApiKey
return detectorspb.DetectorType_Ibmcloud
}
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
package envoyapikey
package ibmcloud
import (
"context"
@@ -10,22 +10,21 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestEnvoyapikey_FromChunk(t *testing.T) {
func TestIbmcloud_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("ENVOYAPIKEY")
inactiveSecret := testSecrets.MustGetField("ENVOYAPIKEY_INACTIVE")
secret := testSecrets.MustGetField("IBM_CLOUD_KEY")
inactiveSecret := testSecrets.MustGetField("IBM_CLOUD_KEY_INACTIVE")
type args struct {
ctx context.Context
@@ -44,12 +43,12 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a envoyapikey secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a ibmcloud secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
DetectorType: detectorspb.DetectorType_IbmClou,
Verified: true,
},
},
@@ -60,12 +59,12 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a envoyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a ibmcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
DetectorType: detectorspb.DetectorType_Ibmcloud,
Verified: false,
},
},
@@ -88,7 +87,7 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Envoyapikey.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Ibmcloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
@@ -98,7 +97,7 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Envoyapikey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("Ibmcloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
+11 -28
View File
@@ -86,14 +86,7 @@ matchLoop:
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
pingRes := j.ping(ctx)
s.Verified = pingRes.err == nil
// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
// detectors to use tri-state verification.
if pingRes.err != nil && !pingRes.determinate {
s.VerificationError = pingRes.err
}
s.Verified = j.ping(ctx)
// TODO: specialized redaction
}
@@ -108,15 +101,15 @@ matchLoop:
}
func tryRedactAnonymousJDBC(conn string) string {
if s, ok := tryRedactBasicAuth(conn); ok {
return s
}
if s, ok := tryRedactURLParams(conn); ok {
return s
}
if s, ok := tryRedactODBC(conn); ok {
return s
}
if s, ok := tryRedactBasicAuth(conn); ok {
return s
}
if s, ok := tryRedactRegex(conn); ok {
return s
}
@@ -165,8 +158,8 @@ func tryRedactODBC(conn string) (string, bool) {
var found bool
var newParams []string
for _, param := range strings.Split(conn, ";") {
key, val, isKvp := strings.Cut(param, "=")
if isKvp && strings.Contains(strings.ToLower(key), "pass") {
key, val, _ := strings.Cut(param, "=")
if strings.Contains(strings.ToLower(key), "pass") {
newParams = append(newParams, key+"="+strings.Repeat("*", len(val)))
found = true
continue
@@ -205,13 +198,8 @@ var supportedSubprotocols = map[string]func(string) (jdbc, error){
"sqlserver": parseSqlServer,
}
type pingResult struct {
err error
determinate bool
}
type jdbc interface {
ping(context.Context) pingResult
ping(context.Context) bool
}
func newJDBC(conn string) (jdbc, error) {
@@ -232,16 +220,11 @@ func newJDBC(conn string) (jdbc, error) {
return parser(subname)
}
func ping(ctx context.Context, driverName string, isDeterminate func(error) bool, candidateConns ...string) pingResult {
var indeterminateErrors []error
for _, c := range candidateConns {
err := pingErr(ctx, driverName, c)
if err == nil || isDeterminate(err) {
return pingResult{err, true}
}
indeterminateErrors = append(indeterminateErrors, err)
func ping(ctx context.Context, driverName, conn string) bool {
if err := pingErr(ctx, driverName, conn); err != nil {
return false
}
return pingResult{errors.Join(indeterminateErrors...), false}
return true
}
func pingErr(ctx context.Context, driverName, conn string) error {
@@ -105,22 +105,6 @@ func TestJdbcVerified(t *testing.T) {
},
wantErr: false,
},
{
name: "sql server verified",
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=localhost;database=%s;password=%s", sqlServerDatabase, sqlServerPass)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_JDBC,
Verified: true,
Redacted: "jdbc:sqlserver://odbc:server=localhost;database=master;password=**************",
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
-46
View File
@@ -5,7 +5,6 @@ package jdbc
import (
"context"
"github.com/stretchr/testify/assert"
"os"
"testing"
@@ -208,51 +207,6 @@ func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
}
}
func TestJdbc_Redact(t *testing.T) {
tests := []struct {
name string
conn string
want string
}{
{
name: "basic auth'",
conn: "//user:secret@tcp(127.0.0.1:3306)/",
want: "//user:******@tcp(127.0.0.1:3306)/",
},
{
name: "basic auth including raw string 'pass'",
conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/",
want: "//wrongUser:*********@tcp(127.0.0.1:3306)/",
},
{
name: "basic auth including raw string 'pass' with unfortunate db name",
conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/passwords",
want: "//wrongUser:*********@tcp(127.0.0.1:3306)/passwords",
},
{
name: "url param-style",
conn: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=p@ssw04d",
want: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=********",
},
{
name: "odbc-style without server",
conn: "//odbc:server=localhost;user id=sa;database=master;password=/p?s=sw&rd",
want: "//odbc:server=localhost;user id=sa;database=master;password=**********",
},
{
name: "odbc-style with server",
conn: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=super-secret-password",
want: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=*********************",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tryRedactAnonymousJDBC(tt.conn)
assert.Equal(t, tt.want, got)
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
+19 -28
View File
@@ -3,8 +3,9 @@ package jdbc
import (
"context"
"errors"
"github.com/go-sql-driver/mysql"
"strings"
_ "github.com/go-sql-driver/mysql"
)
type mysqlJDBC struct {
@@ -15,40 +16,30 @@ type mysqlJDBC struct {
params string
}
func (s *mysqlJDBC) ping(ctx context.Context) pingResult {
return ping(ctx, "mysql", isMySQLErrorDeterminate,
s.conn,
buildMySQLConnectionString(s.host, s.database, s.userPass, s.params),
buildMySQLConnectionString(s.host, "", s.userPass, s.params))
func (s *mysqlJDBC) ping(ctx context.Context) bool {
if ping(ctx, "mysql", s.conn) {
return true
}
// try building connection string (should be same as s.conn though)
if ping(ctx, "mysql", s.build()) {
return true
}
// try removing database
s.database = ""
return ping(ctx, "mysql", s.build())
}
func buildMySQLConnectionString(host, database, userPass, params string) string {
conn := host + "/" + database
if userPass != "" {
conn = userPass + "@" + conn
func (s *mysqlJDBC) build() string {
conn := s.host + "/" + s.database
if s.userPass != "" {
conn = s.userPass + "@" + conn
}
if params != "" {
conn = conn + "?" + params
if s.params != "" {
conn = conn + "?" + s.params
}
return conn
}
func isMySQLErrorDeterminate(err error) bool {
// MySQL error numbers from https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
if mySQLErr, isMySQLErr := err.(*mysql.MySQLError); isMySQLErr {
switch mySQLErr.Number {
case 1044:
// User access denied to a particular database
return false // "Indeterminate" so that other connection variations will be tried
case 1045:
// User access denied
return true
}
}
return false
}
func parseMySQL(subname string) (jdbc, error) {
// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
hostAndDB, params, _ := strings.Cut(subname, "?")
+19 -28
View File
@@ -21,54 +21,45 @@ const (
)
func TestMySQL(t *testing.T) {
type result struct {
parseErr bool
pingOk bool
pingDeterminate bool
}
tests := []struct {
input string
want result
input string
wantErr bool
wantPing bool
}{
{
input: "",
want: result{parseErr: true},
input: "",
wantErr: true,
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: true, pingDeterminate: true},
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
wantPing: true,
},
{
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: false, pingDeterminate: true},
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
wantPing: false,
},
{
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
want: result{pingOk: false, pingDeterminate: true},
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
wantPing: false,
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
want: result{pingOk: true, pingDeterminate: true},
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
wantPing: true,
},
{
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
want: result{pingOk: true, pingDeterminate: true},
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
wantPing: true,
},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
j, err := parseMySQL(tt.input)
if err != nil {
got := result{parseErr: true}
assert.Equal(t, tt.want, got)
if tt.wantErr {
assert.Error(t, err)
return
}
pr := j.ping(context.Background())
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
assert.Equal(t, tt.want, got)
assert.NoError(t, err)
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
})
}
}
+32 -51
View File
@@ -4,8 +4,9 @@ import (
"context"
"errors"
"fmt"
"github.com/lib/pq"
"strings"
_ "github.com/lib/pq"
)
type postgresJDBC struct {
@@ -13,35 +14,37 @@ type postgresJDBC struct {
params map[string]string
}
func (s *postgresJDBC) ping(ctx context.Context) pingResult {
// It is crucial that we try to build a connection string ourselves before using the one we found. This is because
// if the found connection string doesn't include a username, the driver will attempt to connect using the current
// user's name, which will fail in a way that looks like a determinate failure, thus terminating the waterfall. In
// contrast, when we build a connection string ourselves, if there's no username, we try 'postgres' instead, which
// actually has a chance of working.
return ping(ctx, "postgres", isPostgresErrorDeterminate,
buildPostgresConnectionString(s.params, true),
buildPostgresConnectionString(s.params, false),
s.conn,
"postgres://"+s.conn)
}
func isPostgresErrorDeterminate(err error) bool {
// Postgres codes from https://www.postgresql.org/docs/current/errcodes-appendix.html
if pqErr, isPostgresError := err.(*pq.Error); isPostgresError {
switch pqErr.Code {
case "28P01":
// Invalid username/password
return true
case "3D000":
// Unknown database
return false // "Indeterminate" so that other connection variations will be tried
case "3F000":
// Unknown schema
return false // "Indeterminate" so that other connection variations will be tried
}
func (s *postgresJDBC) ping(ctx context.Context) bool {
// try the provided connection string directly
if ping(ctx, "postgres", s.conn) {
return true
}
// try as a URL
if ping(ctx, "postgres", "postgres://"+s.conn) {
return true
}
// build a connection string
data := map[string]string{
// default user
"user": "postgres",
}
for key, val := range s.params {
if key == "host" {
if h, p, found := strings.Cut(val, ":"); found {
data["host"] = h
data["port"] = p
continue
}
}
data[key] = val
}
if ping(ctx, "postgres", joinKeyValues(data, " ")) {
return true
}
if s.params["dbname"] != "" {
delete(s.params, "dbname")
return s.ping(ctx)
}
return false
}
@@ -76,25 +79,3 @@ func parsePostgres(subname string) (jdbc, error) {
return &postgresJDBC{subname[2:], params}, nil
}
func buildPostgresConnectionString(params map[string]string, includeDbName bool) string {
data := map[string]string{
// default user
"user": "postgres",
}
for key, val := range params {
if key == "host" {
if h, p, found := strings.Cut(val, ":"); found {
data["host"] = h
data["port"] = p
continue
}
}
if key == "dbname" && !includeDbName {
continue
}
data[key] = val
}
return joinKeyValues(data, " ")
}
+21 -34
View File
@@ -20,62 +20,49 @@ const (
)
func TestPostgres(t *testing.T) {
type result struct {
parseErr bool
pingOk bool
pingDeterminate bool
}
tests := []struct {
input string
want result
input string
wantErr bool
wantPing bool
}{
{
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
wantPing: true,
},
{
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
wantPing: true,
},
{
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
want: result{pingOk: true, pingDeterminate: true},
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
wantPing: true,
},
{
input: "//localhost:5432/foo?password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: false},
input: "//localhost:5432/foo?password=" + postgresPass,
wantPing: false,
},
{
input: "//localhost:5432/foo?sslmode=disable&password=foo",
want: result{pingOk: false, pingDeterminate: true},
input: "//localhost:5432/foo?sslmode=disable&password=foo",
wantPing: false,
},
{
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: true},
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
wantPing: false,
},
{
input: "//badhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
want: result{pingOk: false, pingDeterminate: false},
},
{
input: "invalid",
want: result{parseErr: true},
input: "invalid",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
j, err := parsePostgres(tt.input)
if err != nil {
got := result{parseErr: true}
assert.Equal(t, tt.want, got)
if tt.wantErr {
assert.Error(t, err)
return
}
pr := j.ping(context.Background())
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
assert.Equal(t, tt.want, got)
assert.NoError(t, err)
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
})
}
}
+3 -9
View File
@@ -14,18 +14,12 @@ type sqliteJDBC struct {
testing bool
}
var cannotVerifySqliteError error = errors.New("sqlite credentials cannot be verified")
func (s *sqliteJDBC) ping(ctx context.Context) pingResult {
func (s *sqliteJDBC) ping(ctx context.Context) bool {
if !s.testing {
// sqlite is not a networked database, so we cannot verify
return pingResult{cannotVerifySqliteError, true}
return false
}
return ping(ctx, "sqlite3", isSqliteErrorDeterminate, s.filename)
}
func isSqliteErrorDeterminate(err error) bool {
return true
return ping(ctx, "sqlite3", s.filename)
}
func parseSqlite(subname string) (jdbc, error) {
+1 -1
View File
@@ -39,7 +39,7 @@ func TestParseSqlite(t *testing.T) {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.True(t, j.ping(context.Background()).err == nil)
assert.True(t, j.ping(context.Background()))
}
})
}
+10 -19
View File
@@ -3,8 +3,9 @@ package jdbc
import (
"context"
"errors"
mssql "github.com/denisenkom/go-mssqldb"
"strings"
_ "github.com/denisenkom/go-mssqldb"
)
type sqlServerJDBC struct {
@@ -12,25 +13,15 @@ type sqlServerJDBC struct {
params map[string]string
}
func (s *sqlServerJDBC) ping(ctx context.Context) pingResult {
return ping(ctx, "mssql", isSqlServerErrorDeterminate,
joinKeyValues(s.params, ";"),
s.conn,
"sqlserver://"+s.conn)
}
func isSqlServerErrorDeterminate(err error) bool {
// Error numbers from https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors?view=sql-server-ver16
if mssqlError, isMssqlError := err.(mssql.Error); isMssqlError {
switch mssqlError.Number {
case 18456:
// Login failed
// This is a determinate failure iff we tried to use a real user
return mssqlError.Message != "login error: Login failed for user ''."
}
func (s *sqlServerJDBC) ping(ctx context.Context) bool {
if ping(ctx, "mssql", s.conn) {
return true
}
return false
if ping(ctx, "mssql", joinKeyValues(s.params, ";")) {
return true
}
// try URL format
return ping(ctx, "mssql", "sqlserver://"+s.conn)
}
func parseSqlServer(subname string) (jdbc, error) {
@@ -21,50 +21,33 @@ const (
)
func TestSqlServer(t *testing.T) {
type result struct {
parseErr bool
pingOk bool
pingDeterminate bool
}
tests := []struct {
input string
want result
input string
wantErr bool
wantPing bool
}{
{
input: "",
want: result{parseErr: true},
input: "",
wantErr: true,
},
{
input: "//server=localhost;user id=sa;database=master;password=" + sqlServerPass,
want: result{pingOk: true, pingDeterminate: true},
input: "//odbc:server=localhost;user id=sa;database=master;password=" + sqlServerPass,
wantPing: true,
},
{
input: "//server=badhost;user id=sa;database=master;password=" + sqlServerPass,
want: result{pingOk: false, pingDeterminate: false},
},
{
input: "//localhost;database=master;spring.datasource.password=" + sqlServerPass,
want: result{pingOk: true, pingDeterminate: true},
},
{
input: "//localhost;database=master;spring.datasource.password=badpassword",
want: result{pingOk: false, pingDeterminate: true},
input: "//localhost;database= master;spring.datasource.password=" + sqlServerPass,
wantPing: true,
},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
j, err := parseSqlServer(tt.input)
if err != nil {
got := result{parseErr: true}
assert.Equal(t, tt.want, got)
if tt.wantErr {
assert.Error(t, err)
return
}
pr := j.ping(context.Background())
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
assert.Equal(t, tt.want, got)
assert.NoError(t, err)
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
})
}
}
+2 -2
View File
@@ -20,13 +20,13 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly"}) + `\b([a-z0-9-]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"launchdarkly", "launch_darkly"}
return []string{"launchdarkly"}
}
// FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.
@@ -6,7 +6,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +18,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(5 * time.Second)
client = common.SaneHttpClientTimeOut(5)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)
+2 -2
View File
@@ -19,8 +19,8 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
tokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
tokenPat = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
// TODO: Oauth client secrets
)
-105
View File
@@ -1,105 +0,0 @@
package postgres
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"postgres"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
)
// find various connection strings
// postgresql://user:secret@localhost:5432/mydb
// "host=localhost dbname=publishing user=www password=foo"
// detectors.PrefixRegex([]string{"nytimes"})
//var (
// connectionStringPats = map[string]*regexp.Regexp{
// "KeyValue": regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
// "": regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
// "Slack Workspace Access Token": regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
// "Slack Workspace Refresh Token": regexp.MustCompile(`xoxr\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
// }
// verifyURL = "https://slack.com/api/auth.test"
//)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"postgres://", ""}
}
// FromData will find and optionally verify Postgres secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 2 {
continue
}
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Postgres,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.postgres.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
if err != nil {
continue
}
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
if res.StatusCode != 401 {
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
}
} else {
s1.VerificationError = err
}
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Postgres
}
-160
View File
@@ -1,160 +0,0 @@
//go:build detectors
// +build detectors
package postgres
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestPostgres_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("POSTGRES")
inactiveSecret := testSecrets.MustGetField("POSTGRES_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a postgres secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantStatusHttpClient(404)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Postgres,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
}
got[i].VerificationError = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -2
View File
@@ -7,7 +7,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(10 * time.Second)
client = common.SaneHttpClientTimeOut(10)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)
@@ -7,7 +7,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -20,7 +19,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(10 * time.Second)
client = common.SaneHttpClientTimeOut(10)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"screenshotlayer"}) + `\b([a-zA-Z0-9_]{32})\b`)
@@ -1,108 +0,0 @@
package trufflehogenterprise
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\bthog-key-[0-9a-f]{16}\b`)
secretPat = regexp.MustCompile(`\bthog-secret-[0-9a-f]{32}\b`)
hostnamePat = regexp.MustCompile(`\b[a-z]+-[a-z]+-[a-z]+\.[a-z][0-9]\.[a-z]+\.trufflehog\.org\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"thog"}
}
// FromData will find and optionally verify Trufflehog Enterprise secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
hostnameMatches := hostnamePat.FindAllStringSubmatch(dataStr, -1)
for _, keyMatch := range keyMatches {
if len(keyMatch) != 1 {
continue
}
resKeyMatch := strings.TrimSpace(keyMatch[0])
for _, secretMatch := range secretMatches {
if len(secretMatch) != 1 {
continue
}
resSecretMatch := strings.TrimSpace(secretMatch[0])
for _, hostnameMatch := range hostnameMatches {
if len(hostnameMatch) != 1 {
continue
}
resHostnameMatch := strings.TrimSpace(hostnameMatch[0])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
Raw: []byte(resKeyMatch),
}
if verify {
endpoint := fmt.Sprintf("https://%s/api/v1/sources", resHostnameMatch)
req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
if err != nil {
continue
}
req.Header.Add("Accept", "application/vnd.trufflehogenterprise+json; version=3")
req.Header.Add("X-Thog-Secret", resSecretMatch)
req.Header.Add("X-Thog-Key", resKeyMatch)
res, err := client.Do(req)
if err == nil {
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "data")
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
results = append(results, s1)
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_TrufflehogEnterprise
}
@@ -1,121 +0,0 @@
//go:build detectors
// +build detectors
package trufflehogenterprise
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestTrufflehogenterprise_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
host := testSecrets.MustGetField("THOG_HOSTNAME")
key := testSecrets.MustGetField("THOG_WEB_KEY")
secret := testSecrets.MustGetField("THOG_WEB_SECRET")
inactiveSecret := testSecrets.MustGetField("THOG_WEB_INACTIVE_SECRET")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a thog secret %s for %s with key %s within", secret, host, key)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a trufflehogenterprise secret %s for %s with key %s within but not valid", inactiveSecret, host, key)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Trufflehogenterprise.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Trufflehogenterprise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+9 -1
View File
@@ -3,7 +3,9 @@ package twilio
import (
"context"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -51,8 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
form := url.Values{}
form.Add("FriendlyName", "MyServiceName")
req, err := http.NewRequestWithContext(
ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
ctx, "POST", "https://verify.twilio.com/v2/Services",
strings.NewReader(form.Encode()),
)
if err != nil {
continue
}
+1 -2
View File
@@ -53,8 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
password := match[1]
// Skip findings where the password only has "*" characters, this is a redacted password
// Also include the url encoded "*" characters: "%2A"
if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
if strings.Trim(password, "*") == "" {
continue
}
+2 -13
View File
@@ -31,14 +31,14 @@ func TestURI_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_URI,
Verified: false,
Redacted: "https://user:********@httpwatch.com",
Redacted: "https://user:********@www.httpwatch.com",
},
},
wantErr: false,
@@ -87,17 +87,6 @@ func TestURI_FromChunk(t *testing.T) {
},
wantErr: false,
},
{
name: "nothing found, password was redacted two different ways",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("Both %s and %s have been redacted within", "https://httpwatch::********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar", "https://httpwatch::%2A%2A%2A%2A%2A%2A@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")),
verify: true,
},
want: []detectors.Result{},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
+1 -2
View File
@@ -6,7 +6,6 @@ import (
"net/http"
"regexp"
"strings"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +18,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
client = common.SaneHttpClientTimeOut(5 * time.Second)
client = common.SaneHttpClientTimeOut(5)
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"zenserp"}) + `\b([0-9a-z-]{36})\b`)
-7
View File
@@ -1,8 +1,6 @@
package engine
import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
@@ -161,7 +159,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convertkit"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convier"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/copper"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/couchbase"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/countrylayer"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/courier"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coveralls"
@@ -201,7 +198,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/disqus"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ditto"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dnscheck"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dockerhub"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docparser"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/documo"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docusign"
@@ -1527,9 +1523,6 @@ func DefaultDetectors() []detectors.Detector {
prefect.Scanner{},
buildkitev2.Scanner{},
opsgenie.Scanner{},
dockerhub.Scanner{},
couchbase.Scanner{},
envoyapikey.Scanner{},
}
}
+21 -12
View File
@@ -113,7 +113,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
chunks: make(chan *sources.Chunk),
results: make(chan detectors.ResultWithMetadata),
detectorAvgTime: sync.Map{},
sourcesWg: &errgroup.Group{},
}
for _, option := range options {
@@ -128,8 +127,10 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
}
ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
// Limit number of concurrent goroutines dedicated to chunking a source.
e.sourcesWg.SetLimit(e.concurrency)
sourcesWg, egCtx := errgroup.WithContext(ctx)
sourcesWg.SetLimit(e.concurrency)
e.sourcesWg = sourcesWg
ctx.SetParent(egCtx)
if len(e.decoders) == 0 {
e.decoders = decoders.DefaultDecoders()
@@ -184,7 +185,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
for i := 0; i < e.concurrency; i++ {
e.workersWg.Add(1)
go func() {
defer common.Recover(ctx)
defer common.RecoverWithExit(ctx)
defer e.workersWg.Done()
e.detectorWorker(ctx)
}()
@@ -377,23 +378,31 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
}
// SupportsLineNumbers determines if a line number can be found for a source type.
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
switch sourceType {
case sourcespb.SourceType_SOURCE_TYPE_GIT,
// lineNumberSupportedSources is a list of sources that support line numbers.
// It is stored this way because slice consts are not supported.
func lineNumberSupportedSources() []sourcespb.SourceType {
return []sourcespb.SourceType{
sourcespb.SourceType_SOURCE_TYPE_GIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB,
sourcespb.SourceType_SOURCE_TYPE_GITLAB,
sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
sourcespb.SourceType_SOURCE_TYPE_GERRIT,
sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
return true
default:
return false
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
}
}
// SupportsLineNumbers determines if a line number can be found for a source type.
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
for _, i := range lineNumberSupportedSources() {
if i == sourceType {
return true
}
}
return false
}
// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
lines := bytes.Split(chunk.Data, []byte("\n"))
-28
View File
@@ -5,7 +5,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -73,30 +72,3 @@ func TestDefaultDecoders(t *testing.T) {
t.Errorf("DefaultDecoders() = %v, expected UTF8 decoder to be first", ds)
}
}
func TestSupportsLineNumbers(t *testing.T) {
testCases := []struct {
name string
input sourcespb.SourceType
expected bool
}{
{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
if result := SupportsLineNumbers(tc.input); result != tc.expected {
t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
}
})
}
}
func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
sourceType := sourcespb.SourceType_SOURCE_TYPE_GITHUB
for i := 0; i < b.N; i++ {
_ = SupportsLineNumbers(sourceType)
}
}
+1 -16
View File
@@ -28,22 +28,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
}
fileSystemSource := filesystem.Source{}
fileSystemSource.WithFilter(c.Filter)
// TODO: This is how it would be used; everything after these comments would be removed.
// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
// func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
// fileSystemSource := filesystem.Source{}
// fileSystemSource.WithFilter(c.Filter)
// if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
// return nil, err
// }
// return &fileSystemSource, nil
// })
// if err != nil {
// return err
// }
// return e.sourceManager.ScheduleRun(ctx, handle)
ctx = context.WithValues(ctx,
"source_type", fileSystemSource.Type().String(),
@@ -53,6 +37,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
if err != nil {
return errors.WrapPrefix(err, "could not init filesystem source", 0)
}
fileSystemSource.WithFilter(c.Filter)
e.sourcesWg.Go(func() error {
defer common.RecoverWithExit(ctx)
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
+7 -17
View File
@@ -7,7 +7,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -16,7 +15,6 @@ import (
type expResult struct {
B string
LineNumber int64
Verified bool
}
func TestGitEngine(t *testing.T) {
@@ -40,16 +38,16 @@ func TestGitEngine(t *testing.T) {
for tName, tTest := range map[string]testProfile{
"all_secrets": {
expected: map[string]expResult{
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4, true},
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3, false},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7, false},
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4},
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
},
filter: common.FilterEmpty(),
},
"base_commit": {
expected: map[string]expResult{
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
},
filter: common.FilterEmpty(),
base: "2f251b8c1e72135a375b659951097ec7749d4af9",
@@ -59,12 +57,8 @@ func TestGitEngine(t *testing.T) {
e := Start(ctx,
WithConcurrency(1),
WithDecoders(decoders.DefaultDecoders()...),
WithDetectors(true, DefaultDetectors()...),
WithDetectors(false, DefaultDetectors()...),
)
// Make the channels buffered so Finish returns.
e.chunks = make(chan *sources.Chunk, 10)
e.results = make(chan detectors.ResultWithMetadata, 10)
cfg := sources.GitConfig{
RepoPath: path,
HeadRef: tTest.branch,
@@ -79,8 +73,7 @@ func TestGitEngine(t *testing.T) {
logFatalFunc := func(_ error, _ string, _ ...any) {
t.Fatalf("error logging function should not have been called")
}
// Wait for all the chunks to be processed.
e.Finish(ctx, logFatalFunc)
go e.Finish(ctx, logFatalFunc)
resultCount := 0
for result := range e.ResultsChan() {
switch meta := result.SourceMetadata.GetData().(type) {
@@ -91,9 +84,6 @@ func TestGitEngine(t *testing.T) {
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
}
if tTest.expected[meta.Git.Commit].Verified != result.Verified {
t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
}
}
resultCount++
+6 -19
View File
@@ -992,9 +992,6 @@ const (
DetectorType_Prefect DetectorType = 918
DetectorType_Docusign DetectorType = 919
DetectorType_Couchbase DetectorType = 920
DetectorType_Dockerhub DetectorType = 921
DetectorType_TrufflehogEnterprise DetectorType = 922
DetectorType_EnvoyApiKey DetectorType = 923
)
// Enum value maps for DetectorType.
@@ -1917,9 +1914,6 @@ var (
918: "Prefect",
919: "Docusign",
920: "Couchbase",
921: "Dockerhub",
922: "TrufflehogEnterprise",
923: "EnvoyApiKey",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2839,9 +2833,6 @@ var (
"Prefect": 918,
"Docusign": 919,
"Couchbase": 920,
"Dockerhub": 921,
"TrufflehogEnterprise": 922,
"EnvoyApiKey": 923,
}
)
@@ -3220,7 +3211,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc6, 0x73, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4141,15 +4132,11 @@ var file_detectors_proto_rawDesc = []byte{
0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x72, 0x65,
0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63, 0x75, 0x73,
0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63, 0x68, 0x62,
0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72,
0x68, 0x75, 0x62, 0x10, 0x99, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x75, 0x66, 0x66, 0x6c,
0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
0x10, 0x9b, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75,
0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f,
0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
+3 -45
View File
@@ -46,7 +46,6 @@ type Source struct {
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var _ sources.SourceUnitEnumerator = (*Source)(nil)
var _ sources.SourceUnitChunker = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
@@ -221,53 +220,12 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
// Enumerate implements SourceUnitEnumerator interface. This implementation simply
// passes the configured paths as the source unit, whether it be a single
// filepath or a directory.
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
func (s *Source) Enumerate(ctx context.Context, units chan<- sources.EnumerationResult) error {
for _, path := range s.paths {
item := sources.CommonSourceUnit{ID: path}
if err := reporter.UnitOk(ctx, item); err != nil {
item := sources.CommonEnumerationOk(path)
if err := common.CancellableWrite(ctx, units, item); err != nil {
return err
}
}
return nil
}
// ChunkUnit implements SourceUnitChunker interface.
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
path := unit.SourceUnitID()
logger := ctx.Logger().WithValues("path", path)
cleanPath := filepath.Clean(path)
fileInfo, err := os.Stat(cleanPath)
if err != nil {
return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
}
ch := make(chan *sources.Chunk)
var scanErr error
go func() {
defer close(ch)
if fileInfo.IsDir() {
// TODO: Finer grain error tracking of individual chunks.
scanErr = s.scanDir(ctx, cleanPath, ch)
} else {
// TODO: Finer grain error tracking of individual
// chunks (in the case of archives).
scanErr = s.scanFile(ctx, cleanPath, ch)
}
}()
for chunk := range ch {
if chunk == nil {
continue
}
if err := reporter.ChunkOk(ctx, *chunk); err != nil {
return err
}
}
if scanErr != nil && scanErr != io.EOF {
logger.Info("error scanning filesystem", "error", scanErr)
return reporter.ChunkErr(ctx, scanErr)
}
return nil
}
+2 -2
View File
@@ -755,7 +755,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
}
path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username(), "--shallow-since", timestamp)
if err != nil {
return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
}
default:
ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
@@ -793,7 +793,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
}
path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username())
if err != nil {
return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
}
default:
ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
-164
View File
@@ -1,164 +0,0 @@
package sources
import (
"fmt"
"sync"
"sync/atomic"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"golang.org/x/sync/errgroup"
)
// handle uniquely identifies a Source given to the manager to manage. If the
// SourceManager is connected to the API, it will be equivalent to the unique
// source ID, otherwise it behaves as a counter.
type handle int64
// SourceInitFunc is a function that takes a source and job ID and returns an
// initialized Source.
type SourceInitFunc func(sourceID int64, jobID int64) (Source, error)
type SourceManager struct {
api apiClient
// Map of handle to source initializer.
handles map[handle]SourceInitFunc
handlesLock sync.Mutex
// Pool limiting the amount of concurrent sources running.
pool errgroup.Group
// Downstream chunks channel to be scanned.
outputChunks chan *Chunk
}
// apiClient is an interface for optionally communicating with an external API.
type apiClient interface {
// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
// GetJobID queries the API for an existing job or new job ID.
GetJobID(ctx context.Context, id int64) (int64, error)
}
// WithAPI adds an API client to the manager for tracking jobs and progress.
func WithAPI(api apiClient) func(*SourceManager) {
return func(man *SourceManager) { man.api = api }
}
// WithConcurrency limits the concurrent number of sources a manager can run.
func WithConcurrency(concurrency int) func(*SourceManager) {
return func(man *SourceManager) { man.pool.SetLimit(concurrency) }
}
// NewManager creates a new manager with the provided options.
func NewManager(outputChunks chan *Chunk, opts ...func(*SourceManager)) *SourceManager {
man := SourceManager{
// Default to the headless API. Can be overwritten by the WithAPI option.
api: &headlessAPI{},
handles: make(map[handle]SourceInitFunc),
outputChunks: outputChunks,
}
for _, opt := range opts {
opt(&man)
}
return &man
}
// Enroll informs the SourceManager to track and manage a Source.
func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
id, err := s.api.RegisterSource(ctx, name, kind)
if err != nil {
return 0, err
}
handleID := handle(id)
s.handlesLock.Lock()
defer s.handlesLock.Unlock()
if _, ok := s.handles[handleID]; ok {
// TODO: smartly handle this?
return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
}
s.handles[handleID] = f
return 0, nil
}
// Run blocks until a resource is available to run the source, then synchronously runs it.
func (s *SourceManager) Run(ctx context.Context, handle handle) error {
// Check the handle is valid before waiting on the pool.
if _, ok := s.getInitFunc(handle); !ok {
return fmt.Errorf("unrecognized handle")
}
if err := ctx.Err(); err != nil {
return err
}
ch := make(chan error)
s.pool.Go(func() error {
defer common.Recover(ctx)
// TODO: The manager should record these errors.
ch <- s.run(ctx, handle)
return nil
})
return <-ch
}
// ScheduleRun blocks until a resource is available to run the source, then
// asynchronously runs it. Error information is lost in this case.
func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
// Check the handle is valid before waiting on the pool.
if _, ok := s.getInitFunc(handle); !ok {
return fmt.Errorf("unrecognized handle")
}
if err := ctx.Err(); err != nil {
return err
}
s.pool.Go(func() error {
defer common.Recover(ctx)
// TODO: The manager should record these errors.
_ = s.run(ctx, handle)
return nil
})
// TODO: Maybe wait for a signal here that initialization was successful?
return nil
}
// run is a helper method to sychronously run the source. It does not check for
// acquired resources.
func (s *SourceManager) run(ctx context.Context, handle handle) error {
jobID, err := s.api.GetJobID(ctx, int64(handle))
if err != nil {
return err
}
initFunc, ok := s.getInitFunc(handle)
if !ok {
return fmt.Errorf("unrecognized handle")
}
source, err := initFunc(jobID, int64(handle))
if err != nil {
return err
}
// TODO: Support UnitChunker and SourceUnitEnumerator.
// TODO: This is where we can introspect on the chunks collected.
return source.Chunks(ctx, s.outputChunks)
}
// getInitFunc is a helper method for safe concurrent access to the
// map[handle]SourceInitFunc map.
func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
s.handlesLock.Lock()
defer s.handlesLock.Unlock()
f, ok := s.handles[handle]
return f, ok
}
// headlessAPI implements the apiClient interface locally.
type headlessAPI struct {
// Counters for assigning handle and job IDs.
sourceIDCounter int64
jobIDCounter int64
}
func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
return atomic.AddInt64(&api.sourceIDCounter, 1), nil
}
func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
return atomic.AddInt64(&api.jobIDCounter, 1), nil
}
+26 -33
View File
@@ -53,41 +53,21 @@ type SourceUnitUnmarshaller interface {
// SourceUnitEnumerator defines an optional interface a Source can implement to
// support enumerating an initialized Source into SourceUnits.
type SourceUnitEnumerator interface {
// Enumerate creates 0 or more units from an initialized source,
// reporting them or any errors to the UnitReporter. This method is
// synchronous but can be called in a goroutine to support concurrent
// enumeration and chunking. An error should only be returned from this
// method in the case of context cancellation, fatal source errors, or
// errors returned by the reporter All other errors related to unit
// enumeration are tracked by the UnitReporter.
Enumerate(ctx context.Context, reporter UnitReporter) error
// Enumerate enumerates the initialized Source, outputting units. This
// method is synchronous but can be called in a goroutine to support
// concurrent enumeration and chunking. An error should only be
// returned from this method in the case of context cancellation. All
// other errors related to unit enumeration are tracked in the
// EnumerationResult.
Enumerate(ctx context.Context, units chan<- EnumerationResult) error
}
// UnitReporter defines the interface a source will use to report whether a
// unit was found during enumeration. Either method may be called any number of
// times. Implementors of this interface should allow for concurrent calls.
type UnitReporter interface {
UnitOk(ctx context.Context, unit SourceUnit) error
UnitErr(ctx context.Context, err error) error
}
// SourceUnitChunker defines an optional interface a Source can implement to
// support chunking a single SourceUnit.
type SourceUnitChunker interface {
// ChunkUnit creates 0 or more chunks from a unit, reporting them or
// any errors to the ChunkReporter. An error should only be returned
// from this method in the case of context cancellation, fatal source
// errors, or errors returned by the reporter. All other errors related
// to unit chunking are tracked by the ChunkReporter.
ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
}
// ChunkReporter defines the interface a source will use to report whether a
// chunk was found during unit chunking. Either method may be called any number
// of times. Implementors of this interface should allow for concurrent calls.
type ChunkReporter interface {
ChunkOk(ctx context.Context, chunk Chunk) error
ChunkErr(ctx context.Context, err error) error
// EnumerationResult is the result of an enumeration, containing the unit and
// error if any. Unit and Error are mutually exclusive (only one will be
// non-nil).
type EnumerationResult struct {
Unit SourceUnit
Error error
}
// SourceUnit is an object that represents a Source's unit of work. This is
@@ -263,3 +243,16 @@ func (p *Progress) GetProgress() *Progress {
defer p.mut.Unlock()
return p
}
// CommonEnumerationOk is a helper function to construct an EnumerationResult
// using a CommonSourceUnit.
func CommonEnumerationOk(id string) EnumerationResult {
unit := CommonSourceUnit{ID: id}
return EnumerationResult{Unit: unit}
}
// EnumerationErr is a helper function to construct an EnumerationResult from
// an error.
func EnumerationErr(err error) EnumerationResult {
return EnumerationResult{Error: err}
}
+1 -4
View File
@@ -929,10 +929,7 @@ enum DetectorType {
Prefect = 918;
Docusign = 919;
Couchbase = 920;
Dockerhub = 921;
TrufflehogEnterprise = 922;
EnvoyApiKey = 923;
Postgres = 925;
IBMCloud = 923; // 921 and 922 are reserved for Trufflehog Web and Scanner Detectors
}
message Result {