Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| a09c0f265f | |||
| f0db55c548 |
@@ -8,8 +8,9 @@ updates:
|
||||
- package-ecosystem: "gomod" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
interval: "weekly"
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
# Check for updates to GitHub Actions every weekday
|
||||
interval: "weekly"
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.18'
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
@@ -11,7 +11,7 @@ jobs:
|
||||
- name: Install Go
|
||||
uses: actions/setup-go@v4
|
||||
with:
|
||||
go-version: '1.20'
|
||||
go-version: '1.18'
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
- name: Smoke
|
||||
|
||||
@@ -22,7 +22,7 @@ require (
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.22.0
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.8.0
|
||||
github.com/go-git/go-git/v5 v5.7.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.5
|
||||
github.com/go-logr/logr v1.2.4
|
||||
github.com/go-logr/zapr v1.2.4
|
||||
@@ -33,7 +33,7 @@ require (
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/go-containerregistry v0.14.0
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/googleapis/gax-go/v2 v2.12.0
|
||||
github.com/googleapis/gax-go/v2 v2.11.0
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
@@ -56,10 +56,10 @@ require (
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.11.0
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
golang.org/x/oauth2 v0.10.0
|
||||
golang.org/x/oauth2 v0.9.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/text v0.11.0
|
||||
google.golang.org/api v0.131.0
|
||||
google.golang.org/api v0.130.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
@@ -68,7 +68,7 @@ require (
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.2 // indirect
|
||||
cloud.google.com/go/compute v1.20.1 // indirect
|
||||
cloud.google.com/go/compute v1.19.3 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.0 // indirect
|
||||
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
|
||||
@@ -170,10 +170,10 @@ require (
|
||||
golang.org/x/tools v0.10.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
|
||||
google.golang.org/grpc v1.56.2 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect
|
||||
google.golang.org/grpc v1.56.1 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
@@ -11,8 +11,8 @@ cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
|
||||
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
|
||||
cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
|
||||
cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
|
||||
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
|
||||
cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
|
||||
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
|
||||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
|
||||
@@ -178,8 +178,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
|
||||
github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ=
|
||||
github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
|
||||
github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2aE=
|
||||
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
|
||||
@@ -281,8 +281,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
|
||||
github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
|
||||
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
|
||||
github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
|
||||
@@ -574,8 +574,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
|
||||
golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
|
||||
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
|
||||
golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs=
|
||||
golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -698,8 +698,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
|
||||
google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
|
||||
google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
|
||||
google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ=
|
||||
google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -722,12 +722,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
|
||||
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
|
||||
google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
|
||||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
|
||||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -740,8 +740,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
|
||||
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
|
||||
google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI=
|
||||
google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
|
||||
google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
|
||||
google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
|
||||
@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
|
||||
1. Add the test secret to GCP Secrets. See [managing test secrets](#managing-test-secrets)
|
||||
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
|
||||
4. Update the tests with these test cases at minimum:
|
||||
1. Found and verified (using a credential loaded from GCP Secrets)
|
||||
2. Found and unverified (determinately, i.e. the secret is invalid)
|
||||
3. Found and unverified (indeterminately due to timeout)
|
||||
4. Found and unverified (indeterminately due to an unexpected API response)
|
||||
5. Not found
|
||||
6. Any false positive cases that you come across
|
||||
2. Found and unverified
|
||||
3. Not found
|
||||
4. Any false positive cases that you come across
|
||||
5. Create a merge request for review. CI tests must be passing.
|
||||
|
||||
## Addendum
|
||||
|
||||
### Verification indeterminacy
|
||||
|
||||
There are two types of reasons that secret verification can fail:
|
||||
* The candidate secret is not actually a valid secret.
|
||||
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
|
||||
|
||||
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
|
||||
|
||||
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
|
||||
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
|
||||
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
|
||||
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
|
||||
|
||||
### Managing Test Secrets
|
||||
|
||||
Do not embed test credentials in the test code. Instead, use GCP Secrets Manager.
|
||||
|
||||
@@ -62,31 +62,15 @@ If you think that something should be included outside of these guidelines, plea
|
||||
1. Create a [test secrets file, and export the variable](#using-a-test-secret-file)
|
||||
2. Update the pattern regex and keywords. Try iterating with [regex101.com](http://regex101.com/).
|
||||
3. Update the verifier code to use a non-destructive API call that can determine whether the secret is valid or not.
|
||||
* Make sure you understand [verification indeterminacy](#verification-indeterminacy).
|
||||
4. Update the tests with these test cases at minimum:
|
||||
1. Found and verified (using a credential loaded from GCP Secrets)
|
||||
2. Found and unverified (determinately, i.e. the secret is invalid)
|
||||
3. Found and unverified (indeterminately due to timeout)
|
||||
4. Found and unverified (indeterminately due to an unexpected API response)
|
||||
5. Not found
|
||||
6. Any false positive cases that you come across
|
||||
2. Found and unverified
|
||||
3. Not found
|
||||
4. Any false positive cases that you come across
|
||||
5. Create a pull request for review.
|
||||
|
||||
## Addendum
|
||||
|
||||
### Verification indeterminacy
|
||||
|
||||
There are two types of reasons that secret verification can fail:
|
||||
* The candidate secret is not actually a valid secret.
|
||||
* Something went wrong in the process unrelated to the candidate secret, such as a transient network error or an unexpected API response.
|
||||
|
||||
In Trufflehog parlance, the first type of verification response is called _determinate_ and the second type is called _indeterminate_. Verification code should distinguish between the two by returning an error object in the result struct **only** for indeterminate failures. In general, a verifier should return an error (indicating an indeterminate failure) in all cases that haven't been explicitly identified as determinate failure states.
|
||||
|
||||
For example, consider a hypothetical authentication endpoint that returns `200 OK` for valid credentials and `403 Forbidden` for invalid credentials. The verifier for this endpoint could make an HTTP request and use the response status code to decide what to return:
|
||||
* A `200` response would indicate that verification succeeded. (Or maybe any `2xx` response.)
|
||||
* A `403` response would indicate that verification failed **determinately** and no error object should be returned.
|
||||
* Any other response would indicate that verification failed **indeterminately** and an error object should be returned.
|
||||
|
||||
### Using a test secret file
|
||||
|
||||
1. Create a file called `.env` with this env file format:
|
||||
|
||||
@@ -34,14 +34,14 @@ func main() {
|
||||
case "detector":
|
||||
mustWriteTemplates([]templateJob{
|
||||
{
|
||||
TemplatePath: "pkg/detectors/alchemy/alchemy.go",
|
||||
TemplatePath: "pkg/detectors/heroku/heroku.go",
|
||||
WritePath: filepath.Join(folderPath(), nameLower+".go"),
|
||||
ReplaceString: []string{"alchemy"},
|
||||
ReplaceString: []string{"heroku"},
|
||||
},
|
||||
{
|
||||
TemplatePath: "pkg/detectors/alchemy/alchemy_test.go",
|
||||
TemplatePath: "pkg/detectors/heroku/heroku_test.go",
|
||||
WritePath: filepath.Join(folderPath(), nameLower+"_test.go"),
|
||||
ReplaceString: []string{"alchemy"},
|
||||
ReplaceString: []string{"heroku"},
|
||||
},
|
||||
})
|
||||
// case "source":
|
||||
|
||||
@@ -207,7 +207,7 @@ func run(state overseer.State) {
|
||||
if *profile {
|
||||
go func() {
|
||||
router := http.NewServeMux()
|
||||
router.Handle("/debug/pprof/", http.DefaultServeMux)
|
||||
router.Handle("/debug/pprof", http.DefaultServeMux)
|
||||
router.Handle("/debug/fgprof", fgprof.Handler())
|
||||
logger.Info("starting pprof and fgprof server on :18066 /debug/pprof and /debug/fgprof")
|
||||
if err := http.ListenAndServe(":18066", router); err != nil {
|
||||
|
||||
+2
-26
@@ -3,7 +3,6 @@ package common
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -76,14 +75,6 @@ func PinnedCertPool() *x509.CertPool {
|
||||
return trustedCerts
|
||||
}
|
||||
|
||||
type FakeTransport struct {
|
||||
CreateResponse func(req *http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
func (t FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return t.CreateResponse(req)
|
||||
}
|
||||
|
||||
type CustomTransport struct {
|
||||
T http.RoundTripper
|
||||
}
|
||||
@@ -100,21 +91,6 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
|
||||
return &CustomTransport{T}
|
||||
}
|
||||
|
||||
func ConstantStatusHttpClient(statusCode int) *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: DefaultResponseTimeout,
|
||||
Transport: FakeTransport{
|
||||
CreateResponse: func(req *http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
Request: req,
|
||||
Body: io.NopCloser(strings.NewReader("")),
|
||||
StatusCode: statusCode,
|
||||
}, nil
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func PinnedRetryableHttpClient() *http.Client {
|
||||
httpClient := retryablehttp.NewClient()
|
||||
httpClient.Logger = nil
|
||||
@@ -176,9 +152,9 @@ func SaneHttpClient() *http.Client {
|
||||
}
|
||||
|
||||
// SaneHttpClientTimeOut adds a custom timeout for some scanners
|
||||
func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {
|
||||
func SaneHttpClientTimeOut(timeOutSeconds int64) *http.Client {
|
||||
httpClient := &http.Client{}
|
||||
httpClient.Timeout = timeout
|
||||
httpClient.Timeout = time.Second * time.Duration(timeOutSeconds)
|
||||
httpClient.Transport = NewCustomTransport(nil)
|
||||
return httpClient
|
||||
}
|
||||
|
||||
+14
-3
@@ -18,11 +18,22 @@ var (
|
||||
type Context interface {
|
||||
context.Context
|
||||
Logger() logr.Logger
|
||||
Parent() context.Context
|
||||
SetParent(ctx context.Context) Context
|
||||
}
|
||||
|
||||
// CancelFunc is a type alias to context.CancelFunc to allow use as if they are
|
||||
// the same types.
|
||||
type CancelFunc = context.CancelFunc
|
||||
// Parent returns the parent context.
|
||||
func (l logCtx) Parent() context.Context {
|
||||
return l.Context
|
||||
}
|
||||
|
||||
// SetParent sets the parent context on the context.
|
||||
func (l logCtx) SetParent(ctx context.Context) Context {
|
||||
l.Context = ctx
|
||||
return l
|
||||
}
|
||||
|
||||
type CancelFunc context.CancelFunc
|
||||
|
||||
// logCtx implements Context.
|
||||
type logCtx struct {
|
||||
|
||||
@@ -2,7 +2,6 @@ package alchemy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,15 +11,14 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"alchemy"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
|
||||
)
|
||||
@@ -49,10 +47,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.alchemy.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
@@ -67,13 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode != 401 {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,12 +32,11 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -53,8 +52,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -70,8 +68,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -81,48 +78,14 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Alchemy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantStatusHttpClient(404)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Alchemy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Alchemy.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -132,10 +95,6 @@ func TestAlchemy_FromChunk(t *testing.T) {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
|
||||
}
|
||||
got[i].VerificationError = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
|
||||
@@ -184,8 +184,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
res.Body.Close()
|
||||
} else {
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
package dockerhub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Can use email or username for login.
|
||||
usernamePat = regexp.MustCompile(`(?im)(?:user|usr|-u)\S{0,40}?[:=\s]{1,3}[ '"=]?([a-zA-Z0-9]{4,40})\b`)
|
||||
emailPat = regexp.MustCompile(common.EmailPattern)
|
||||
|
||||
// Can use password or personal access token (PAT) for login, but this scanner will only check for PATs.
|
||||
accessTokenPat = regexp.MustCompile(`\bdckr_pat_([a-zA-Z0-9_-]){27}\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"dckr_pat_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Dockerhub secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
emailMatches := emailPat.FindAllString(dataStr, -1)
|
||||
dataStr = emailPat.ReplaceAllString(dataStr, "")
|
||||
usernameMatches := usernamePat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
accessTokenMatches := accessTokenPat.FindAllString(dataStr, -1)
|
||||
|
||||
userMatches := emailMatches
|
||||
for _, usernameMatch := range usernameMatches {
|
||||
if len(usernameMatch) > 1 {
|
||||
userMatches = append(userMatches, usernameMatch[1])
|
||||
}
|
||||
}
|
||||
|
||||
for _, resUserMatch := range userMatches {
|
||||
for _, resAccessTokenMatch := range accessTokenMatches {
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Dockerhub,
|
||||
Raw: []byte(fmt.Sprintf("%s: %s", resUserMatch, resAccessTokenMatch)),
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(fmt.Sprintf(`{"username": "%s", "password": "%s"}`, resUserMatch, resAccessTokenMatch))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://hub.docker.com/v2/users/login", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
body, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Valid credentials can still return a 401 status code if 2FA is enabled
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 401 && strings.Contains(string(body), "login_2fa_token")) {
|
||||
s1.Verified = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Dockerhub
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package dockerhub
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestDockerhub_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
username := testSecrets.MustGetField("DOCKERHUB_USERNAME")
|
||||
email := testSecrets.MustGetField("DOCKERHUB_EMAIL")
|
||||
pat := testSecrets.MustGetField("DOCKERHUB_PAT")
|
||||
inactivePat := testSecrets.MustGetField("DOCKERHUB_INACTIVE_PAT")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, pat)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Dockerhub,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, verified (email)",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", email, pat)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Dockerhub,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("docker login -u %s -p %s", username, inactivePat)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Dockerhub,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Dockerhub.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Dockerhub.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -20,7 +19,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(5 * time.Second)
|
||||
client = common.SaneHttpClientTimeOut(5)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"getemail"}) + `\b([a-zA-Z0-9-]{20})\b`)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
package envoyapikey
|
||||
package ibmcloud
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -21,16 +21,16 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"envoy"}) + `\b([a-zA-Z0-9]{220})\b`)
|
||||
keyPat = regexp.MustCompile(`\b([A-Za-z0-9_-]{44})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"envoy"}
|
||||
return []string{"ibm"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Envoy secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify IBM Cloud secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -43,27 +43,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
|
||||
DetectorType: detectorspb.DetectorType_Ibmcloud,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.envoy.com/v1/locations", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ibmcloud.com/apps", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/vnd.envoy+json; version=3")
|
||||
req.Header.Add("X-Api-Key", resMatch)
|
||||
req.Header.Add("Accept", "application/vnd.ibmcloud+json; version=3")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
|
||||
// Invalid API keys can also return status code 200, so check for presence of 'status 401' in response body.
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == 403 {
|
||||
if !strings.Contains(string(body), `"status":401`) {
|
||||
s1.Verified = true
|
||||
}
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
@@ -80,5 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_EnvoyApiKey
|
||||
return detectorspb.DetectorType_Ibmcloud
|
||||
}
|
||||
+11
-12
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package envoyapikey
|
||||
package ibmcloud
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -10,22 +10,21 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestEnvoyapikey_FromChunk(t *testing.T) {
|
||||
func TestIbmcloud_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("ENVOYAPIKEY")
|
||||
inactiveSecret := testSecrets.MustGetField("ENVOYAPIKEY_INACTIVE")
|
||||
secret := testSecrets.MustGetField("IBM_CLOUD_KEY")
|
||||
inactiveSecret := testSecrets.MustGetField("IBM_CLOUD_KEY_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -44,12 +43,12 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a envoyapikey secret %s within", secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a ibmcloud secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
|
||||
DetectorType: detectorspb.DetectorType_IbmClou,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
@@ -60,12 +59,12 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a envoyapikey secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a ibmcloud secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_EnvoyApiKey,
|
||||
DetectorType: detectorspb.DetectorType_Ibmcloud,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
@@ -88,7 +87,7 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Envoyapikey.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Ibmcloud.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
@@ -98,7 +97,7 @@ func TestEnvoyapikey_FromChunk(t *testing.T) {
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Envoyapikey.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
t.Errorf("Ibmcloud.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
+11
-28
@@ -86,14 +86,7 @@ matchLoop:
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
pingRes := j.ping(ctx)
|
||||
s.Verified = pingRes.err == nil
|
||||
// If there's a ping error that is marked as "determinate" we throw it away. We do this because this was the
|
||||
// behavior before tri-state verification was introduced and preserving it allows us to gradually migrate
|
||||
// detectors to use tri-state verification.
|
||||
if pingRes.err != nil && !pingRes.determinate {
|
||||
s.VerificationError = pingRes.err
|
||||
}
|
||||
s.Verified = j.ping(ctx)
|
||||
// TODO: specialized redaction
|
||||
}
|
||||
|
||||
@@ -108,15 +101,15 @@ matchLoop:
|
||||
}
|
||||
|
||||
func tryRedactAnonymousJDBC(conn string) string {
|
||||
if s, ok := tryRedactBasicAuth(conn); ok {
|
||||
return s
|
||||
}
|
||||
if s, ok := tryRedactURLParams(conn); ok {
|
||||
return s
|
||||
}
|
||||
if s, ok := tryRedactODBC(conn); ok {
|
||||
return s
|
||||
}
|
||||
if s, ok := tryRedactBasicAuth(conn); ok {
|
||||
return s
|
||||
}
|
||||
if s, ok := tryRedactRegex(conn); ok {
|
||||
return s
|
||||
}
|
||||
@@ -165,8 +158,8 @@ func tryRedactODBC(conn string) (string, bool) {
|
||||
var found bool
|
||||
var newParams []string
|
||||
for _, param := range strings.Split(conn, ";") {
|
||||
key, val, isKvp := strings.Cut(param, "=")
|
||||
if isKvp && strings.Contains(strings.ToLower(key), "pass") {
|
||||
key, val, _ := strings.Cut(param, "=")
|
||||
if strings.Contains(strings.ToLower(key), "pass") {
|
||||
newParams = append(newParams, key+"="+strings.Repeat("*", len(val)))
|
||||
found = true
|
||||
continue
|
||||
@@ -205,13 +198,8 @@ var supportedSubprotocols = map[string]func(string) (jdbc, error){
|
||||
"sqlserver": parseSqlServer,
|
||||
}
|
||||
|
||||
type pingResult struct {
|
||||
err error
|
||||
determinate bool
|
||||
}
|
||||
|
||||
type jdbc interface {
|
||||
ping(context.Context) pingResult
|
||||
ping(context.Context) bool
|
||||
}
|
||||
|
||||
func newJDBC(conn string) (jdbc, error) {
|
||||
@@ -232,16 +220,11 @@ func newJDBC(conn string) (jdbc, error) {
|
||||
return parser(subname)
|
||||
}
|
||||
|
||||
func ping(ctx context.Context, driverName string, isDeterminate func(error) bool, candidateConns ...string) pingResult {
|
||||
var indeterminateErrors []error
|
||||
for _, c := range candidateConns {
|
||||
err := pingErr(ctx, driverName, c)
|
||||
if err == nil || isDeterminate(err) {
|
||||
return pingResult{err, true}
|
||||
}
|
||||
indeterminateErrors = append(indeterminateErrors, err)
|
||||
func ping(ctx context.Context, driverName, conn string) bool {
|
||||
if err := pingErr(ctx, driverName, conn); err != nil {
|
||||
return false
|
||||
}
|
||||
return pingResult{errors.Join(indeterminateErrors...), false}
|
||||
return true
|
||||
}
|
||||
|
||||
func pingErr(ctx context.Context, driverName, conn string) error {
|
||||
|
||||
@@ -105,22 +105,6 @@ func TestJdbcVerified(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "sql server verified",
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("jdbc:sqlserver://odbc:server=localhost;database=%s;password=%s", sqlServerDatabase, sqlServerPass)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JDBC,
|
||||
Verified: true,
|
||||
Redacted: "jdbc:sqlserver://odbc:server=localhost;database=master;password=**************",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -5,7 +5,6 @@ package jdbc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@@ -208,51 +207,6 @@ func TestJdbc_FromDataWithIgnorePattern(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestJdbc_Redact(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
conn string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "basic auth'",
|
||||
conn: "//user:secret@tcp(127.0.0.1:3306)/",
|
||||
want: "//user:******@tcp(127.0.0.1:3306)/",
|
||||
},
|
||||
{
|
||||
name: "basic auth including raw string 'pass'",
|
||||
conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/",
|
||||
want: "//wrongUser:*********@tcp(127.0.0.1:3306)/",
|
||||
},
|
||||
{
|
||||
name: "basic auth including raw string 'pass' with unfortunate db name",
|
||||
conn: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/passwords",
|
||||
want: "//wrongUser:*********@tcp(127.0.0.1:3306)/passwords",
|
||||
},
|
||||
{
|
||||
name: "url param-style",
|
||||
conn: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=p@ssw04d",
|
||||
want: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=********",
|
||||
},
|
||||
{
|
||||
name: "odbc-style without server",
|
||||
conn: "//odbc:server=localhost;user id=sa;database=master;password=/p?s=sw&rd",
|
||||
want: "//odbc:server=localhost;user id=sa;database=master;password=**********",
|
||||
},
|
||||
{
|
||||
name: "odbc-style with server",
|
||||
conn: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=super-secret-password",
|
||||
want: "jdbc:sqlserver://a.b.c.net;database=database-name;spring.datasource.password=*********************",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tryRedactAnonymousJDBC(tt.conn)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
|
||||
+19
-28
@@ -3,8 +3,9 @@ package jdbc
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"strings"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
type mysqlJDBC struct {
|
||||
@@ -15,40 +16,30 @@ type mysqlJDBC struct {
|
||||
params string
|
||||
}
|
||||
|
||||
func (s *mysqlJDBC) ping(ctx context.Context) pingResult {
|
||||
return ping(ctx, "mysql", isMySQLErrorDeterminate,
|
||||
s.conn,
|
||||
buildMySQLConnectionString(s.host, s.database, s.userPass, s.params),
|
||||
buildMySQLConnectionString(s.host, "", s.userPass, s.params))
|
||||
func (s *mysqlJDBC) ping(ctx context.Context) bool {
|
||||
if ping(ctx, "mysql", s.conn) {
|
||||
return true
|
||||
}
|
||||
// try building connection string (should be same as s.conn though)
|
||||
if ping(ctx, "mysql", s.build()) {
|
||||
return true
|
||||
}
|
||||
// try removing database
|
||||
s.database = ""
|
||||
return ping(ctx, "mysql", s.build())
|
||||
}
|
||||
|
||||
func buildMySQLConnectionString(host, database, userPass, params string) string {
|
||||
conn := host + "/" + database
|
||||
if userPass != "" {
|
||||
conn = userPass + "@" + conn
|
||||
func (s *mysqlJDBC) build() string {
|
||||
conn := s.host + "/" + s.database
|
||||
if s.userPass != "" {
|
||||
conn = s.userPass + "@" + conn
|
||||
}
|
||||
if params != "" {
|
||||
conn = conn + "?" + params
|
||||
if s.params != "" {
|
||||
conn = conn + "?" + s.params
|
||||
}
|
||||
return conn
|
||||
}
|
||||
|
||||
func isMySQLErrorDeterminate(err error) bool {
|
||||
// MySQL error numbers from https://dev.mysql.com/doc/mysql-errors/8.0/en/server-error-reference.html
|
||||
if mySQLErr, isMySQLErr := err.(*mysql.MySQLError); isMySQLErr {
|
||||
switch mySQLErr.Number {
|
||||
case 1044:
|
||||
// User access denied to a particular database
|
||||
return false // "Indeterminate" so that other connection variations will be tried
|
||||
case 1045:
|
||||
// User access denied
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func parseMySQL(subname string) (jdbc, error) {
|
||||
// expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
|
||||
hostAndDB, params, _ := strings.Cut(subname, "?")
|
||||
|
||||
@@ -21,54 +21,45 @@ const (
|
||||
)
|
||||
|
||||
func TestMySQL(t *testing.T) {
|
||||
type result struct {
|
||||
parseErr bool
|
||||
pingOk bool
|
||||
pingDeterminate bool
|
||||
}
|
||||
tests := []struct {
|
||||
input string
|
||||
want result
|
||||
input string
|
||||
wantErr bool
|
||||
wantPing bool
|
||||
}{
|
||||
{
|
||||
input: "",
|
||||
want: result{parseErr: true},
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
wantPing: true,
|
||||
},
|
||||
{
|
||||
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
wantPing: false,
|
||||
},
|
||||
{
|
||||
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
|
||||
wantPing: false,
|
||||
},
|
||||
{
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
|
||||
wantPing: true,
|
||||
},
|
||||
{
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
|
||||
wantPing: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
j, err := parseMySQL(tt.input)
|
||||
|
||||
if err != nil {
|
||||
got := result{parseErr: true}
|
||||
assert.Equal(t, tt.want, got)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
pr := j.ping(context.Background())
|
||||
|
||||
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
|
||||
assert.Equal(t, tt.want, got)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/lib/pq"
|
||||
"strings"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type postgresJDBC struct {
|
||||
@@ -13,35 +14,37 @@ type postgresJDBC struct {
|
||||
params map[string]string
|
||||
}
|
||||
|
||||
func (s *postgresJDBC) ping(ctx context.Context) pingResult {
|
||||
// It is crucial that we try to build a connection string ourselves before using the one we found. This is because
|
||||
// if the found connection string doesn't include a username, the driver will attempt to connect using the current
|
||||
// user's name, which will fail in a way that looks like a determinate failure, thus terminating the waterfall. In
|
||||
// contrast, when we build a connection string ourselves, if there's no username, we try 'postgres' instead, which
|
||||
// actually has a chance of working.
|
||||
return ping(ctx, "postgres", isPostgresErrorDeterminate,
|
||||
buildPostgresConnectionString(s.params, true),
|
||||
buildPostgresConnectionString(s.params, false),
|
||||
s.conn,
|
||||
"postgres://"+s.conn)
|
||||
}
|
||||
|
||||
func isPostgresErrorDeterminate(err error) bool {
|
||||
// Postgres codes from https://www.postgresql.org/docs/current/errcodes-appendix.html
|
||||
if pqErr, isPostgresError := err.(*pq.Error); isPostgresError {
|
||||
switch pqErr.Code {
|
||||
case "28P01":
|
||||
// Invalid username/password
|
||||
return true
|
||||
case "3D000":
|
||||
// Unknown database
|
||||
return false // "Indeterminate" so that other connection variations will be tried
|
||||
case "3F000":
|
||||
// Unknown schema
|
||||
return false // "Indeterminate" so that other connection variations will be tried
|
||||
}
|
||||
func (s *postgresJDBC) ping(ctx context.Context) bool {
|
||||
// try the provided connection string directly
|
||||
if ping(ctx, "postgres", s.conn) {
|
||||
return true
|
||||
}
|
||||
// try as a URL
|
||||
if ping(ctx, "postgres", "postgres://"+s.conn) {
|
||||
return true
|
||||
}
|
||||
// build a connection string
|
||||
data := map[string]string{
|
||||
// default user
|
||||
"user": "postgres",
|
||||
}
|
||||
for key, val := range s.params {
|
||||
if key == "host" {
|
||||
if h, p, found := strings.Cut(val, ":"); found {
|
||||
data["host"] = h
|
||||
data["port"] = p
|
||||
continue
|
||||
}
|
||||
}
|
||||
data[key] = val
|
||||
}
|
||||
if ping(ctx, "postgres", joinKeyValues(data, " ")) {
|
||||
return true
|
||||
}
|
||||
if s.params["dbname"] != "" {
|
||||
delete(s.params, "dbname")
|
||||
return s.ping(ctx)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -76,25 +79,3 @@ func parsePostgres(subname string) (jdbc, error) {
|
||||
|
||||
return &postgresJDBC{subname[2:], params}, nil
|
||||
}
|
||||
|
||||
func buildPostgresConnectionString(params map[string]string, includeDbName bool) string {
|
||||
data := map[string]string{
|
||||
// default user
|
||||
"user": "postgres",
|
||||
}
|
||||
for key, val := range params {
|
||||
if key == "host" {
|
||||
if h, p, found := strings.Cut(val, ":"); found {
|
||||
data["host"] = h
|
||||
data["port"] = p
|
||||
continue
|
||||
}
|
||||
}
|
||||
if key == "dbname" && !includeDbName {
|
||||
continue
|
||||
}
|
||||
data[key] = val
|
||||
}
|
||||
|
||||
return joinKeyValues(data, " ")
|
||||
}
|
||||
|
||||
@@ -20,62 +20,49 @@ const (
|
||||
)
|
||||
|
||||
func TestPostgres(t *testing.T) {
|
||||
type result struct {
|
||||
parseErr bool
|
||||
pingOk bool
|
||||
pingDeterminate bool
|
||||
}
|
||||
tests := []struct {
|
||||
input string
|
||||
want result
|
||||
input string
|
||||
wantErr bool
|
||||
wantPing bool
|
||||
}{
|
||||
{
|
||||
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
|
||||
wantPing: true,
|
||||
},
|
||||
{
|
||||
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
|
||||
wantPing: true,
|
||||
},
|
||||
{
|
||||
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
|
||||
wantPing: true,
|
||||
},
|
||||
{
|
||||
input: "//localhost:5432/foo?password=" + postgresPass,
|
||||
want: result{pingOk: false, pingDeterminate: false},
|
||||
input: "//localhost:5432/foo?password=" + postgresPass,
|
||||
wantPing: false,
|
||||
},
|
||||
{
|
||||
input: "//localhost:5432/foo?sslmode=disable&password=foo",
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
input: "//localhost:5432/foo?sslmode=disable&password=foo",
|
||||
wantPing: false,
|
||||
},
|
||||
{
|
||||
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
|
||||
wantPing: false,
|
||||
},
|
||||
{
|
||||
input: "//badhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
|
||||
want: result{pingOk: false, pingDeterminate: false},
|
||||
},
|
||||
{
|
||||
input: "invalid",
|
||||
want: result{parseErr: true},
|
||||
input: "invalid",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
j, err := parsePostgres(tt.input)
|
||||
|
||||
if err != nil {
|
||||
got := result{parseErr: true}
|
||||
assert.Equal(t, tt.want, got)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
pr := j.ping(context.Background())
|
||||
|
||||
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
|
||||
assert.Equal(t, tt.want, got)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,18 +14,12 @@ type sqliteJDBC struct {
|
||||
testing bool
|
||||
}
|
||||
|
||||
var cannotVerifySqliteError error = errors.New("sqlite credentials cannot be verified")
|
||||
|
||||
func (s *sqliteJDBC) ping(ctx context.Context) pingResult {
|
||||
func (s *sqliteJDBC) ping(ctx context.Context) bool {
|
||||
if !s.testing {
|
||||
// sqlite is not a networked database, so we cannot verify
|
||||
return pingResult{cannotVerifySqliteError, true}
|
||||
return false
|
||||
}
|
||||
return ping(ctx, "sqlite3", isSqliteErrorDeterminate, s.filename)
|
||||
}
|
||||
|
||||
func isSqliteErrorDeterminate(err error) bool {
|
||||
return true
|
||||
return ping(ctx, "sqlite3", s.filename)
|
||||
}
|
||||
|
||||
func parseSqlite(subname string) (jdbc, error) {
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestParseSqlite(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, j.ping(context.Background()).err == nil)
|
||||
assert.True(t, j.ping(context.Background()))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3,8 +3,9 @@ package jdbc
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
mssql "github.com/denisenkom/go-mssqldb"
|
||||
"strings"
|
||||
|
||||
_ "github.com/denisenkom/go-mssqldb"
|
||||
)
|
||||
|
||||
type sqlServerJDBC struct {
|
||||
@@ -12,25 +13,15 @@ type sqlServerJDBC struct {
|
||||
params map[string]string
|
||||
}
|
||||
|
||||
func (s *sqlServerJDBC) ping(ctx context.Context) pingResult {
|
||||
return ping(ctx, "mssql", isSqlServerErrorDeterminate,
|
||||
joinKeyValues(s.params, ";"),
|
||||
s.conn,
|
||||
"sqlserver://"+s.conn)
|
||||
}
|
||||
|
||||
func isSqlServerErrorDeterminate(err error) bool {
|
||||
// Error numbers from https://learn.microsoft.com/en-us/sql/relational-databases/errors-events/database-engine-events-and-errors?view=sql-server-ver16
|
||||
if mssqlError, isMssqlError := err.(mssql.Error); isMssqlError {
|
||||
switch mssqlError.Number {
|
||||
case 18456:
|
||||
// Login failed
|
||||
// This is a determinate failure iff we tried to use a real user
|
||||
return mssqlError.Message != "login error: Login failed for user ''."
|
||||
}
|
||||
func (s *sqlServerJDBC) ping(ctx context.Context) bool {
|
||||
if ping(ctx, "mssql", s.conn) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
if ping(ctx, "mssql", joinKeyValues(s.params, ";")) {
|
||||
return true
|
||||
}
|
||||
// try URL format
|
||||
return ping(ctx, "mssql", "sqlserver://"+s.conn)
|
||||
}
|
||||
|
||||
func parseSqlServer(subname string) (jdbc, error) {
|
||||
|
||||
@@ -21,50 +21,33 @@ const (
|
||||
)
|
||||
|
||||
func TestSqlServer(t *testing.T) {
|
||||
type result struct {
|
||||
parseErr bool
|
||||
pingOk bool
|
||||
pingDeterminate bool
|
||||
}
|
||||
tests := []struct {
|
||||
input string
|
||||
want result
|
||||
input string
|
||||
wantErr bool
|
||||
wantPing bool
|
||||
}{
|
||||
{
|
||||
input: "",
|
||||
want: result{parseErr: true},
|
||||
input: "",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
input: "//server=localhost;user id=sa;database=master;password=" + sqlServerPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
input: "//odbc:server=localhost;user id=sa;database=master;password=" + sqlServerPass,
|
||||
wantPing: true,
|
||||
},
|
||||
{
|
||||
input: "//server=badhost;user id=sa;database=master;password=" + sqlServerPass,
|
||||
want: result{pingOk: false, pingDeterminate: false},
|
||||
},
|
||||
{
|
||||
input: "//localhost;database=master;spring.datasource.password=" + sqlServerPass,
|
||||
want: result{pingOk: true, pingDeterminate: true},
|
||||
},
|
||||
{
|
||||
input: "//localhost;database=master;spring.datasource.password=badpassword",
|
||||
want: result{pingOk: false, pingDeterminate: true},
|
||||
input: "//localhost;database= master;spring.datasource.password=" + sqlServerPass,
|
||||
wantPing: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
j, err := parseSqlServer(tt.input)
|
||||
|
||||
if err != nil {
|
||||
got := result{parseErr: true}
|
||||
assert.Equal(t, tt.want, got)
|
||||
if tt.wantErr {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
pr := j.ping(context.Background())
|
||||
|
||||
got := result{pingOk: pr.err == nil, pingDeterminate: pr.determinate}
|
||||
assert.Equal(t, tt.want, got)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantPing, j.ping(context.Background()))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,13 +20,13 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly", "launch_darkly"}) + `\b([a-z0-9-]{40})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"launchdarkly"}) + `\b([a-z0-9-]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"launchdarkly", "launch_darkly"}
|
||||
return []string{"launchdarkly"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify LaunchDarkly secrets in a given set of bytes.
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,7 +18,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(5 * time.Second)
|
||||
client = common.SaneHttpClientTimeOut(5)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)
|
||||
|
||||
@@ -19,8 +19,8 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
domainPat = regexp.MustCompile(`\b[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com\b`)
|
||||
tokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9_-]{40}\b`)
|
||||
domainPat = regexp.MustCompile(`[a-z0-9-]{1,40}\.okta(?:preview|-emea){0,1}\.com`)
|
||||
tokenPat = regexp.MustCompile(`00[a-zA-Z0-9_-]{40}`)
|
||||
// TODO: Oauth client secrets
|
||||
)
|
||||
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"postgres"}) + `\b([0-9a-zA-Z]{23}_[0-9a-zA-Z]{8})\b`)
|
||||
)
|
||||
|
||||
// find various connection strings
|
||||
|
||||
// postgresql://user:secret@localhost:5432/mydb
|
||||
// "host=localhost dbname=publishing user=www password=foo"
|
||||
|
||||
// detectors.PrefixRegex([]string{"nytimes"})
|
||||
|
||||
//var (
|
||||
// connectionStringPats = map[string]*regexp.Regexp{
|
||||
// "KeyValue": regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
// "": regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
// "Slack Workspace Access Token": regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
// "Slack Workspace Refresh Token": regexp.MustCompile(`xoxr\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
// }
|
||||
// verifyURL = "https://slack.com/api/auth.test"
|
||||
//)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"postgres://", ""}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Postgres secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://eth-mainnet.g.postgres.com/v2/"+resMatch+"/getNFTs/?owner=vitalik.eth", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode != 401 {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Postgres
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestPostgres_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("POSTGRES")
|
||||
inactiveSecret := testSecrets.MustGetField("POSTGRES_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a postgres secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantStatusHttpClient(404)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a postgres secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Postgres,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Postgres.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr)
|
||||
}
|
||||
got[i].VerificationError = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Postgres.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -20,7 +19,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(10 * time.Second)
|
||||
client = common.SaneHttpClientTimeOut(10)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"scrapersite"}) + `\b([a-zA-Z0-9]{45})\b`)
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -20,7 +19,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(10 * time.Second)
|
||||
client = common.SaneHttpClientTimeOut(10)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"screenshotlayer"}) + `\b([a-zA-Z0-9_]{32})\b`)
|
||||
|
||||
@@ -1,108 +0,0 @@
|
||||
package trufflehogenterprise
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\bthog-key-[0-9a-f]{16}\b`)
|
||||
secretPat = regexp.MustCompile(`\bthog-secret-[0-9a-f]{32}\b`)
|
||||
hostnamePat = regexp.MustCompile(`\b[a-z]+-[a-z]+-[a-z]+\.[a-z][0-9]\.[a-z]+\.trufflehog\.org\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"thog"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Trufflehog Enterprise secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
hostnameMatches := hostnamePat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, keyMatch := range keyMatches {
|
||||
if len(keyMatch) != 1 {
|
||||
continue
|
||||
}
|
||||
resKeyMatch := strings.TrimSpace(keyMatch[0])
|
||||
for _, secretMatch := range secretMatches {
|
||||
|
||||
if len(secretMatch) != 1 {
|
||||
continue
|
||||
}
|
||||
resSecretMatch := strings.TrimSpace(secretMatch[0])
|
||||
|
||||
for _, hostnameMatch := range hostnameMatches {
|
||||
if len(hostnameMatch) != 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
resHostnameMatch := strings.TrimSpace(hostnameMatch[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
|
||||
Raw: []byte(resKeyMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
endpoint := fmt.Sprintf("https://%s/api/v1/sources", resHostnameMatch)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", endpoint, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Accept", "application/vnd.trufflehogenterprise+json; version=3")
|
||||
req.Header.Add("X-Thog-Secret", resSecretMatch)
|
||||
req.Header.Add("X-Thog-Key", resKeyMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "data")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_TrufflehogEnterprise
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package trufflehogenterprise
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestTrufflehogenterprise_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
host := testSecrets.MustGetField("THOG_HOSTNAME")
|
||||
key := testSecrets.MustGetField("THOG_WEB_KEY")
|
||||
secret := testSecrets.MustGetField("THOG_WEB_SECRET")
|
||||
inactiveSecret := testSecrets.MustGetField("THOG_WEB_INACTIVE_SECRET")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a thog secret %s for %s with key %s within", secret, host, key)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a trufflehogenterprise secret %s for %s with key %s within but not valid", inactiveSecret, host, key)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_TrufflehogEnterprise,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Trufflehogenterprise.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Trufflehogenterprise.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,9 @@ package twilio
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -51,8 +53,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
|
||||
form := url.Values{}
|
||||
form.Add("FriendlyName", "MyServiceName")
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
|
||||
ctx, "POST", "https://verify.twilio.com/v2/Services",
|
||||
|
||||
strings.NewReader(form.Encode()),
|
||||
)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -53,8 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
password := match[1]
|
||||
|
||||
// Skip findings where the password only has "*" characters, this is a redacted password
|
||||
// Also include the url encoded "*" characters: "%2A"
|
||||
if strings.Trim(password, "*") == "" || strings.Trim(password, "%2A") == "" {
|
||||
if strings.Trim(password, "*") == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -31,14 +31,14 @@ func TestURI_FromChunk(t *testing.T) {
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
|
||||
data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_URI,
|
||||
Verified: false,
|
||||
Redacted: "https://user:********@httpwatch.com",
|
||||
Redacted: "https://user:********@www.httpwatch.com",
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -87,17 +87,6 @@ func TestURI_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "nothing found, password was redacted two different ways",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("Both %s and %s have been redacted within", "https://httpwatch::********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar", "https://httpwatch::%2A%2A%2A%2A%2A%2A@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -19,7 +18,7 @@ type Scanner struct{}
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClientTimeOut(5 * time.Second)
|
||||
client = common.SaneHttpClientTimeOut(5)
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"zenserp"}) + `\b([0-9a-z-]{36})\b`)
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
|
||||
@@ -161,7 +159,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convertkit"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/convier"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/copper"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/couchbase"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/countrylayer"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/courier"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coveralls"
|
||||
@@ -201,7 +198,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/disqus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ditto"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dnscheck"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/dockerhub"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docparser"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/documo"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/docusign"
|
||||
@@ -1527,9 +1523,6 @@ func DefaultDetectors() []detectors.Detector {
|
||||
prefect.Scanner{},
|
||||
buildkitev2.Scanner{},
|
||||
opsgenie.Scanner{},
|
||||
dockerhub.Scanner{},
|
||||
couchbase.Scanner{},
|
||||
envoyapikey.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+21
-12
@@ -113,7 +113,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
chunks: make(chan *sources.Chunk),
|
||||
results: make(chan detectors.ResultWithMetadata),
|
||||
detectorAvgTime: sync.Map{},
|
||||
sourcesWg: &errgroup.Group{},
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
@@ -128,8 +127,10 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
}
|
||||
ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
|
||||
|
||||
// Limit number of concurrent goroutines dedicated to chunking a source.
|
||||
e.sourcesWg.SetLimit(e.concurrency)
|
||||
sourcesWg, egCtx := errgroup.WithContext(ctx)
|
||||
sourcesWg.SetLimit(e.concurrency)
|
||||
e.sourcesWg = sourcesWg
|
||||
ctx.SetParent(egCtx)
|
||||
|
||||
if len(e.decoders) == 0 {
|
||||
e.decoders = decoders.DefaultDecoders()
|
||||
@@ -184,7 +185,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
for i := 0; i < e.concurrency; i++ {
|
||||
e.workersWg.Add(1)
|
||||
go func() {
|
||||
defer common.Recover(ctx)
|
||||
defer common.RecoverWithExit(ctx)
|
||||
defer e.workersWg.Done()
|
||||
e.detectorWorker(ctx)
|
||||
}()
|
||||
@@ -377,23 +378,31 @@ func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// SupportsLineNumbers determines if a line number can be found for a source type.
|
||||
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
|
||||
switch sourceType {
|
||||
case sourcespb.SourceType_SOURCE_TYPE_GIT,
|
||||
// lineNumberSupportedSources is a list of sources that support line numbers.
|
||||
// It is stored this way because slice consts are not supported.
|
||||
func lineNumberSupportedSources() []sourcespb.SourceType {
|
||||
return []sourcespb.SourceType{
|
||||
sourcespb.SourceType_SOURCE_TYPE_GIT,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GITHUB,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GITLAB,
|
||||
sourcespb.SourceType_SOURCE_TYPE_BITBUCKET,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GERRIT,
|
||||
sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
|
||||
sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
|
||||
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
|
||||
}
|
||||
}
|
||||
|
||||
// SupportsLineNumbers determines if a line number can be found for a source type.
|
||||
func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
|
||||
for _, i := range lineNumberSupportedSources() {
|
||||
if i == sourceType {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
|
||||
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
|
||||
lines := bytes.Split(chunk.Data, []byte("\n"))
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -73,30 +72,3 @@ func TestDefaultDecoders(t *testing.T) {
|
||||
t.Errorf("DefaultDecoders() = %v, expected UTF8 decoder to be first", ds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupportsLineNumbers(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
input sourcespb.SourceType
|
||||
expected bool
|
||||
}{
|
||||
{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
|
||||
{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
|
||||
{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if result := SupportsLineNumbers(tc.input); result != tc.expected {
|
||||
t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
|
||||
sourceType := sourcespb.SourceType_SOURCE_TYPE_GITHUB
|
||||
for i := 0; i < b.N; i++ {
|
||||
_ = SupportsLineNumbers(sourceType)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,22 +28,6 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
}
|
||||
|
||||
fileSystemSource := filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
|
||||
// TODO: This is how it would be used; everything after these comments would be removed.
|
||||
// handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(),
|
||||
// func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
// fileSystemSource := filesystem.Source{}
|
||||
// fileSystemSource.WithFilter(c.Filter)
|
||||
// if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// return &fileSystemSource, nil
|
||||
// })
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// return e.sourceManager.ScheduleRun(ctx, handle)
|
||||
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", fileSystemSource.Type().String(),
|
||||
@@ -53,6 +37,7 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not init filesystem source", 0)
|
||||
}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
e.sourcesWg.Go(func() error {
|
||||
defer common.RecoverWithExit(ctx)
|
||||
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
|
||||
|
||||
+7
-17
@@ -7,7 +7,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
@@ -16,7 +15,6 @@ import (
|
||||
type expResult struct {
|
||||
B string
|
||||
LineNumber int64
|
||||
Verified bool
|
||||
}
|
||||
|
||||
func TestGitEngine(t *testing.T) {
|
||||
@@ -40,16 +38,16 @@ func TestGitEngine(t *testing.T) {
|
||||
for tName, tTest := range map[string]testProfile{
|
||||
"all_secrets": {
|
||||
expected: map[string]expResult{
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
|
||||
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4, true},
|
||||
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3, false},
|
||||
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7, false},
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
|
||||
"84e9c75e388ae3e866e121087ea2dd45a71068f2": {"AKIAILE3JG6KMS3HZGCA", 4},
|
||||
"8afb0ecd4998b1179e428db5ebbcdc8221214432": {"369963c1434c377428ca8531fbc46c0c43d037a0", 3},
|
||||
"27fbead3bf883cdb7de9d7825ed401f28f9398f1": {"ffc7e0f9400fb6300167009e42d2f842cd7956e2", 7},
|
||||
},
|
||||
filter: common.FilterEmpty(),
|
||||
},
|
||||
"base_commit": {
|
||||
expected: map[string]expResult{
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2, true},
|
||||
"70001020fab32b1fcf2f1f0e5c66424eae649826": {"AKIAXYZDQCEN4B6JSJQI", 2},
|
||||
},
|
||||
filter: common.FilterEmpty(),
|
||||
base: "2f251b8c1e72135a375b659951097ec7749d4af9",
|
||||
@@ -59,12 +57,8 @@ func TestGitEngine(t *testing.T) {
|
||||
e := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(true, DefaultDetectors()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
)
|
||||
// Make the channels buffered so Finish returns.
|
||||
e.chunks = make(chan *sources.Chunk, 10)
|
||||
e.results = make(chan detectors.ResultWithMetadata, 10)
|
||||
|
||||
cfg := sources.GitConfig{
|
||||
RepoPath: path,
|
||||
HeadRef: tTest.branch,
|
||||
@@ -79,8 +73,7 @@ func TestGitEngine(t *testing.T) {
|
||||
logFatalFunc := func(_ error, _ string, _ ...any) {
|
||||
t.Fatalf("error logging function should not have been called")
|
||||
}
|
||||
// Wait for all the chunks to be processed.
|
||||
e.Finish(ctx, logFatalFunc)
|
||||
go e.Finish(ctx, logFatalFunc)
|
||||
resultCount := 0
|
||||
for result := range e.ResultsChan() {
|
||||
switch meta := result.SourceMetadata.GetData().(type) {
|
||||
@@ -91,9 +84,6 @@ func TestGitEngine(t *testing.T) {
|
||||
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
|
||||
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
|
||||
}
|
||||
if tTest.expected[meta.Git.Commit].Verified != result.Verified {
|
||||
t.Errorf("%s: unexpected verification. Got: %v, Expected: %v", tName, result.Verified, tTest.expected[meta.Git.Commit].Verified)
|
||||
}
|
||||
}
|
||||
resultCount++
|
||||
|
||||
|
||||
@@ -992,9 +992,6 @@ const (
|
||||
DetectorType_Prefect DetectorType = 918
|
||||
DetectorType_Docusign DetectorType = 919
|
||||
DetectorType_Couchbase DetectorType = 920
|
||||
DetectorType_Dockerhub DetectorType = 921
|
||||
DetectorType_TrufflehogEnterprise DetectorType = 922
|
||||
DetectorType_EnvoyApiKey DetectorType = 923
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1917,9 +1914,6 @@ var (
|
||||
918: "Prefect",
|
||||
919: "Docusign",
|
||||
920: "Couchbase",
|
||||
921: "Dockerhub",
|
||||
922: "TrufflehogEnterprise",
|
||||
923: "EnvoyApiKey",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2839,9 +2833,6 @@ var (
|
||||
"Prefect": 918,
|
||||
"Docusign": 919,
|
||||
"Couchbase": 920,
|
||||
"Dockerhub": 921,
|
||||
"TrufflehogEnterprise": 922,
|
||||
"EnvoyApiKey": 923,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3220,7 +3211,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc6, 0x73, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4141,15 +4132,11 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50, 0x72, 0x65,
|
||||
0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63, 0x75, 0x73,
|
||||
0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63, 0x68, 0x62,
|
||||
0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72,
|
||||
0x68, 0x75, 0x62, 0x10, 0x99, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
|
||||
0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
|
||||
0x10, 0x9b, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
|
||||
0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
|
||||
0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
|
||||
0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
|
||||
0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
|
||||
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75,
|
||||
0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f,
|
||||
0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
|
||||
0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -46,7 +46,6 @@ type Source struct {
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
var _ sources.SourceUnitEnumerator = (*Source)(nil)
|
||||
var _ sources.SourceUnitChunker = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
@@ -221,53 +220,12 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou
|
||||
// Enumerate implements SourceUnitEnumerator interface. This implementation simply
|
||||
// passes the configured paths as the source unit, whether it be a single
|
||||
// filepath or a directory.
|
||||
func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) error {
|
||||
func (s *Source) Enumerate(ctx context.Context, units chan<- sources.EnumerationResult) error {
|
||||
for _, path := range s.paths {
|
||||
item := sources.CommonSourceUnit{ID: path}
|
||||
if err := reporter.UnitOk(ctx, item); err != nil {
|
||||
item := sources.CommonEnumerationOk(path)
|
||||
if err := common.CancellableWrite(ctx, units, item); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ChunkUnit implements SourceUnitChunker interface.
|
||||
func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
|
||||
path := unit.SourceUnitID()
|
||||
logger := ctx.Logger().WithValues("path", path)
|
||||
|
||||
cleanPath := filepath.Clean(path)
|
||||
fileInfo, err := os.Stat(cleanPath)
|
||||
if err != nil {
|
||||
return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
|
||||
}
|
||||
|
||||
ch := make(chan *sources.Chunk)
|
||||
var scanErr error
|
||||
go func() {
|
||||
defer close(ch)
|
||||
if fileInfo.IsDir() {
|
||||
// TODO: Finer grain error tracking of individual chunks.
|
||||
scanErr = s.scanDir(ctx, cleanPath, ch)
|
||||
} else {
|
||||
// TODO: Finer grain error tracking of individual
|
||||
// chunks (in the case of archives).
|
||||
scanErr = s.scanFile(ctx, cleanPath, ch)
|
||||
}
|
||||
}()
|
||||
|
||||
for chunk := range ch {
|
||||
if chunk == nil {
|
||||
continue
|
||||
}
|
||||
if err := reporter.ChunkOk(ctx, *chunk); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if scanErr != nil && scanErr != io.EOF {
|
||||
logger.Info("error scanning filesystem", "error", scanErr)
|
||||
return reporter.ChunkErr(ctx, scanErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -755,7 +755,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
|
||||
}
|
||||
path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username(), "--shallow-since", timestamp)
|
||||
if err != nil {
|
||||
return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
|
||||
return path, true, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
|
||||
}
|
||||
default:
|
||||
ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
|
||||
@@ -793,7 +793,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
|
||||
}
|
||||
path, _, err = CloneRepoUsingToken(ctx, password, remotePath, uri.User.Username())
|
||||
if err != nil {
|
||||
return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", uri.Redacted(), err)
|
||||
return path, remote, fmt.Errorf("failed to clone authenticated Git repo (%s): %s", remotePath, err)
|
||||
}
|
||||
default:
|
||||
ctx.Logger().V(1).Info("cloning repo without authentication", "uri", uri)
|
||||
|
||||
@@ -1,164 +0,0 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// handle uniquely identifies a Source given to the manager to manage. If the
|
||||
// SourceManager is connected to the API, it will be equivalent to the unique
|
||||
// source ID, otherwise it behaves as a counter.
|
||||
type handle int64
|
||||
|
||||
// SourceInitFunc is a function that takes a source and job ID and returns an
|
||||
// initialized Source.
|
||||
type SourceInitFunc func(sourceID int64, jobID int64) (Source, error)
|
||||
|
||||
type SourceManager struct {
|
||||
api apiClient
|
||||
// Map of handle to source initializer.
|
||||
handles map[handle]SourceInitFunc
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
// Downstream chunks channel to be scanned.
|
||||
outputChunks chan *Chunk
|
||||
}
|
||||
|
||||
// apiClient is an interface for optionally communicating with an external API.
|
||||
type apiClient interface {
|
||||
// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
|
||||
RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
|
||||
// GetJobID queries the API for an existing job or new job ID.
|
||||
GetJobID(ctx context.Context, id int64) (int64, error)
|
||||
}
|
||||
|
||||
// WithAPI adds an API client to the manager for tracking jobs and progress.
|
||||
func WithAPI(api apiClient) func(*SourceManager) {
|
||||
return func(man *SourceManager) { man.api = api }
|
||||
}
|
||||
|
||||
// WithConcurrency limits the concurrent number of sources a manager can run.
|
||||
func WithConcurrency(concurrency int) func(*SourceManager) {
|
||||
return func(man *SourceManager) { man.pool.SetLimit(concurrency) }
|
||||
}
|
||||
|
||||
// NewManager creates a new manager with the provided options.
|
||||
func NewManager(outputChunks chan *Chunk, opts ...func(*SourceManager)) *SourceManager {
|
||||
man := SourceManager{
|
||||
// Default to the headless API. Can be overwritten by the WithAPI option.
|
||||
api: &headlessAPI{},
|
||||
handles: make(map[handle]SourceInitFunc),
|
||||
outputChunks: outputChunks,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(&man)
|
||||
}
|
||||
return &man
|
||||
}
|
||||
|
||||
// Enroll informs the SourceManager to track and manage a Source.
|
||||
func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
|
||||
id, err := s.api.RegisterSource(ctx, name, kind)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
handleID := handle(id)
|
||||
s.handlesLock.Lock()
|
||||
defer s.handlesLock.Unlock()
|
||||
if _, ok := s.handles[handleID]; ok {
|
||||
// TODO: smartly handle this?
|
||||
return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
|
||||
}
|
||||
s.handles[handleID] = f
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// Run blocks until a resource is available to run the source, then synchronously runs it.
|
||||
func (s *SourceManager) Run(ctx context.Context, handle handle) error {
|
||||
// Check the handle is valid before waiting on the pool.
|
||||
if _, ok := s.getInitFunc(handle); !ok {
|
||||
return fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ch := make(chan error)
|
||||
s.pool.Go(func() error {
|
||||
defer common.Recover(ctx)
|
||||
// TODO: The manager should record these errors.
|
||||
ch <- s.run(ctx, handle)
|
||||
return nil
|
||||
})
|
||||
return <-ch
|
||||
}
|
||||
|
||||
// ScheduleRun blocks until a resource is available to run the source, then
|
||||
// asynchronously runs it. Error information is lost in this case.
|
||||
func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) error {
|
||||
// Check the handle is valid before waiting on the pool.
|
||||
if _, ok := s.getInitFunc(handle); !ok {
|
||||
return fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.pool.Go(func() error {
|
||||
defer common.Recover(ctx)
|
||||
// TODO: The manager should record these errors.
|
||||
_ = s.run(ctx, handle)
|
||||
return nil
|
||||
})
|
||||
// TODO: Maybe wait for a signal here that initialization was successful?
|
||||
return nil
|
||||
}
|
||||
|
||||
// run is a helper method to sychronously run the source. It does not check for
|
||||
// acquired resources.
|
||||
func (s *SourceManager) run(ctx context.Context, handle handle) error {
|
||||
jobID, err := s.api.GetJobID(ctx, int64(handle))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
initFunc, ok := s.getInitFunc(handle)
|
||||
if !ok {
|
||||
return fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
source, err := initFunc(jobID, int64(handle))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// TODO: Support UnitChunker and SourceUnitEnumerator.
|
||||
// TODO: This is where we can introspect on the chunks collected.
|
||||
return source.Chunks(ctx, s.outputChunks)
|
||||
}
|
||||
|
||||
// getInitFunc is a helper method for safe concurrent access to the
|
||||
// map[handle]SourceInitFunc map.
|
||||
func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) {
|
||||
s.handlesLock.Lock()
|
||||
defer s.handlesLock.Unlock()
|
||||
f, ok := s.handles[handle]
|
||||
return f, ok
|
||||
}
|
||||
|
||||
// headlessAPI implements the apiClient interface locally.
|
||||
type headlessAPI struct {
|
||||
// Counters for assigning handle and job IDs.
|
||||
sourceIDCounter int64
|
||||
jobIDCounter int64
|
||||
}
|
||||
|
||||
func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
|
||||
return atomic.AddInt64(&api.sourceIDCounter, 1), nil
|
||||
}
|
||||
|
||||
func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
|
||||
return atomic.AddInt64(&api.jobIDCounter, 1), nil
|
||||
}
|
||||
+26
-33
@@ -53,41 +53,21 @@ type SourceUnitUnmarshaller interface {
|
||||
// SourceUnitEnumerator defines an optional interface a Source can implement to
|
||||
// support enumerating an initialized Source into SourceUnits.
|
||||
type SourceUnitEnumerator interface {
|
||||
// Enumerate creates 0 or more units from an initialized source,
|
||||
// reporting them or any errors to the UnitReporter. This method is
|
||||
// synchronous but can be called in a goroutine to support concurrent
|
||||
// enumeration and chunking. An error should only be returned from this
|
||||
// method in the case of context cancellation, fatal source errors, or
|
||||
// errors returned by the reporter All other errors related to unit
|
||||
// enumeration are tracked by the UnitReporter.
|
||||
Enumerate(ctx context.Context, reporter UnitReporter) error
|
||||
// Enumerate enumerates the initialized Source, outputting units. This
|
||||
// method is synchronous but can be called in a goroutine to support
|
||||
// concurrent enumeration and chunking. An error should only be
|
||||
// returned from this method in the case of context cancellation. All
|
||||
// other errors related to unit enumeration are tracked in the
|
||||
// EnumerationResult.
|
||||
Enumerate(ctx context.Context, units chan<- EnumerationResult) error
|
||||
}
|
||||
|
||||
// UnitReporter defines the interface a source will use to report whether a
|
||||
// unit was found during enumeration. Either method may be called any number of
|
||||
// times. Implementors of this interface should allow for concurrent calls.
|
||||
type UnitReporter interface {
|
||||
UnitOk(ctx context.Context, unit SourceUnit) error
|
||||
UnitErr(ctx context.Context, err error) error
|
||||
}
|
||||
|
||||
// SourceUnitChunker defines an optional interface a Source can implement to
|
||||
// support chunking a single SourceUnit.
|
||||
type SourceUnitChunker interface {
|
||||
// ChunkUnit creates 0 or more chunks from a unit, reporting them or
|
||||
// any errors to the ChunkReporter. An error should only be returned
|
||||
// from this method in the case of context cancellation, fatal source
|
||||
// errors, or errors returned by the reporter. All other errors related
|
||||
// to unit chunking are tracked by the ChunkReporter.
|
||||
ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
|
||||
}
|
||||
|
||||
// ChunkReporter defines the interface a source will use to report whether a
|
||||
// chunk was found during unit chunking. Either method may be called any number
|
||||
// of times. Implementors of this interface should allow for concurrent calls.
|
||||
type ChunkReporter interface {
|
||||
ChunkOk(ctx context.Context, chunk Chunk) error
|
||||
ChunkErr(ctx context.Context, err error) error
|
||||
// EnumerationResult is the result of an enumeration, containing the unit and
|
||||
// error if any. Unit and Error are mutually exclusive (only one will be
|
||||
// non-nil).
|
||||
type EnumerationResult struct {
|
||||
Unit SourceUnit
|
||||
Error error
|
||||
}
|
||||
|
||||
// SourceUnit is an object that represents a Source's unit of work. This is
|
||||
@@ -263,3 +243,16 @@ func (p *Progress) GetProgress() *Progress {
|
||||
defer p.mut.Unlock()
|
||||
return p
|
||||
}
|
||||
|
||||
// CommonEnumerationOk is a helper function to construct an EnumerationResult
|
||||
// using a CommonSourceUnit.
|
||||
func CommonEnumerationOk(id string) EnumerationResult {
|
||||
unit := CommonSourceUnit{ID: id}
|
||||
return EnumerationResult{Unit: unit}
|
||||
}
|
||||
|
||||
// EnumerationErr is a helper function to construct an EnumerationResult from
|
||||
// an error.
|
||||
func EnumerationErr(err error) EnumerationResult {
|
||||
return EnumerationResult{Error: err}
|
||||
}
|
||||
|
||||
@@ -929,10 +929,7 @@ enum DetectorType {
|
||||
Prefect = 918;
|
||||
Docusign = 919;
|
||||
Couchbase = 920;
|
||||
Dockerhub = 921;
|
||||
TrufflehogEnterprise = 922;
|
||||
EnvoyApiKey = 923;
|
||||
Postgres = 925;
|
||||
IBMCloud = 923; // 921 and 922 are reserved for Trufflehog Web and Scanner Detectors
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
Reference in New Issue
Block a user