Compare commits

..

3 Commits

Author SHA1 Message Date
Zubair Khan 0cffc3b499 save 2023-06-26 17:33:08 -04:00
Zubair Khan 71cb6dd850 chunking comments 2023-06-26 17:23:13 -04:00
Zubair Khan f800a99dcc issue comment scanning 2023-06-22 16:35:34 -07:00
51 changed files with 354 additions and 14709 deletions
+2 -1
View File
@@ -10,7 +10,8 @@ RUN --mount=type=cache,target=/go/pkg/mod \
FROM alpine:3.15
RUN apk add --no-cache bash git openssh-client ca-certificates \
&& update-ca-certificates
&& rm -rf /var/cache/apk/* && \
update-ca-certificates
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
COPY entrypoint.sh /etc/entrypoint.sh
RUN chmod +x /etc/entrypoint.sh
-2
View File
@@ -151,8 +151,6 @@ trufflehog docker --image trufflesecurity/secrets --only-verified
+ Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
+ It says a private key was verified, what does that mean?
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
+ Is there an easy way to ignore specific secrets?
+ If the scanned source [supports line numbers](https://github.com/trufflesecurity/trufflehog/blob/d6375ba92172fd830abb4247cca15e3176448c5d/pkg/engine/engine.go#L358-L365), then you can add a `trufflehog:ignore` comment on the line containing the secret to ignore that secrets.
# :newspaper: What's new in v3?
+19 -23
View File
@@ -5,8 +5,8 @@ go 1.20
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
require (
cloud.google.com/go/secretmanager v1.11.1
cloud.google.com/go/storage v1.31.0
cloud.google.com/go/secretmanager v1.11.0
cloud.google.com/go/storage v1.29.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/TheZeroSlave/zapsentry v1.15.0
github.com/aws/aws-sdk-go v1.44.83
@@ -14,16 +14,15 @@ require (
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.4.0
github.com/couchbase/gocb/v2 v2.6.3
github.com/crewjam/rfc5424 v0.1.0
github.com/denisenkom/go-mssqldb v0.12.3
github.com/envoyproxy/protoc-gen-validate v1.0.2
github.com/envoyproxy/protoc-gen-validate v1.0.1
github.com/fatih/color v1.15.0
github.com/felixge/fgprof v0.9.3
github.com/getsentry/sentry-go v0.22.0
github.com/go-errors/errors v1.4.2
github.com/go-git/go-git/v5 v5.7.0
github.com/go-ldap/ldap/v3 v3.4.5
github.com/go-ldap/ldap/v3 v3.4.4
github.com/go-logr/logr v1.2.4
github.com/go-logr/zapr v1.2.4
github.com/go-redis/redis v6.15.9+incompatible
@@ -31,7 +30,7 @@ require (
github.com/gobwas/glob v0.2.3
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-cmp v0.5.9
github.com/google/go-containerregistry v0.14.0
github.com/google/go-containerregistry v0.15.2
github.com/google/go-github/v42 v42.0.0
github.com/googleapis/gax-go/v2 v2.11.0
github.com/hashicorp/go-retryablehttp v0.7.4
@@ -51,26 +50,26 @@ require (
github.com/sergi/go-diff v1.3.1
github.com/stretchr/testify v1.8.4
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
github.com/xanzy/go-gitlab v0.86.0
go.mongodb.org/mongo-driver v1.12.0
github.com/xanzy/go-gitlab v0.85.0
go.mongodb.org/mongo-driver v1.11.6
go.uber.org/zap v1.24.0
golang.org/x/crypto v0.10.0
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
golang.org/x/oauth2 v0.9.0
golang.org/x/sync v0.3.0
golang.org/x/sync v0.2.0
golang.org/x/text v0.10.0
google.golang.org/api v0.129.0
google.golang.org/protobuf v1.31.0
google.golang.org/api v0.128.0
google.golang.org/protobuf v1.30.0
gopkg.in/alecthomas/kingpin.v2 v2.2.6
gopkg.in/h2non/gock.v1 v1.1.2
sigs.k8s.io/yaml v1.3.0
)
require (
cloud.google.com/go v0.110.2 // indirect
cloud.google.com/go v0.110.0 // indirect
cloud.google.com/go/compute v1.19.3 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.1.0 // indirect
cloud.google.com/go/iam v1.0.1 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
@@ -78,11 +77,10 @@ require (
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
@@ -95,10 +93,8 @@ require (
github.com/cloudflare/circl v1.3.3 // indirect
github.com/connesc/cipherio v0.2.1 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/couchbase/gocbcore/v10 v10.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/dlclark/regexp2 v1.4.0 // indirect
github.com/docker/cli v23.0.5+incompatible // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v23.0.5+incompatible // indirect
@@ -120,7 +116,7 @@ require (
github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -157,25 +153,25 @@ require (
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xdg-go/scram v1.1.1 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.11.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.10.0 // indirect
golang.org/x/tools v0.8.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/grpc v1.56.1 // indirect
google.golang.org/grpc v1.55.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
+41 -62
View File
@@ -7,8 +7,8 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
@@ -17,16 +17,16 @@ cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1h
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94=
cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk=
cloud.google.com/go/iam v1.0.1 h1:lyeCAU6jpnVNrE9zGQkTl3WgNgK/X+uWwaw0kynZJMU=
cloud.google.com/go/iam v1.0.1/go.mod h1:yR3tmSL8BcZB4bxByRv2jkSIahVmCtfKZwLYGBalRE8=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE4+zkuEQs=
cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw=
cloud.google.com/go/secretmanager v1.11.0 h1:tNzxvs3fP57iTrhaS1XdEPwg4LSPKiDNgp+ZLbj0hks=
cloud.google.com/go/secretmanager v1.11.0/go.mod h1:qeQq0/jyJqrGeULu0GkRsVSPKTvf98AEqJnuEIQiJwA=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI=
cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0=
cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
@@ -49,8 +49,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e h1:NeAW1fUYUEWhft7pkxDf6WoUvEZJ/uOKsvtpjLnn8MU=
github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -64,14 +64,10 @@ github.com/TheZeroSlave/zapsentry v1.15.0 h1:w/YglG4Hc2L2VoEH6JKC+3YzhUL18/OZRjJ
github.com/TheZeroSlave/zapsentry v1.15.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64=
github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek=
github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
@@ -119,13 +115,6 @@ github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEyc
github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA=
github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
github.com/couchbase/gocbcore/v10 v10.2.3 h1:PEkRSNSkKjUBXx82Ucr094+anoiCG5GleOOQZOHo6D4=
github.com/couchbase/gocbcore/v10 v10.2.3/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/crewjam/rfc5424 v0.1.0 h1:MSeXJm22oKovLzWj44AHwaItjIMUMugYGkEzfa831H8=
@@ -137,8 +126,6 @@ github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+
github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E=
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE=
github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
@@ -160,8 +147,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE=
github.com/envoyproxy/protoc-gen-validate v1.0.1 h1:kt9FtLiooDc0vbwTLhdg3dyNX1K9Qwa1EK9LcD4jVUQ=
github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs=
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
@@ -186,9 +173,8 @@ github.com/go-git/go-git/v5 v5.7.0 h1:t9AudWVLmqzlo+4bqdf7GY+46SUuRsx59SboFxkq2a
github.com/go-git/go-git/v5 v5.7.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
@@ -260,9 +246,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.14.0 h1:z58vMqHxuwvAsVwvKEkmVBz2TlgBgH5k6koEXBtlYkw=
github.com/google/go-containerregistry v0.14.0/go.mod h1:aiJ2fp/SXvkWgmYHioXnbMdlgB8eXiiYOY55gfN91Wk=
github.com/google/go-github/v39 v39.0.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
github.com/google/go-github/v52 v52.0.0 h1:uyGWOY+jMQ8GVGSX8dkSwCzlehU3WfdxQ7GweO/JP7M=
@@ -283,8 +268,8 @@ github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM=
github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4=
github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
@@ -320,13 +305,10 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -363,12 +345,9 @@ github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrS
github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@@ -440,7 +419,6 @@ github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt
github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -449,15 +427,17 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
@@ -466,16 +446,16 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
github.com/xanzy/go-gitlab v0.86.0 h1:jR8V9cK9jXRQDb46KOB20NCF3ksY09luaG0IfXE6p7w=
github.com/xanzy/go-gitlab v0.86.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
github.com/xanzy/go-gitlab v0.85.0 h1:E/wjnsd/mM5kV6O9y5+i6zxjx+wfAwa97sgcT1ETNwk=
github.com/xanzy/go-gitlab v0.85.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xdg-go/scram v1.1.1 h1:VOMT+81stJgXW3CpHyqHN3AXDYIMsx56mEFrB37Mb/E=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -483,8 +463,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
go.mongodb.org/mongo-driver v1.11.6 h1:XM7G6PjiGAO5betLF13BIa5TlLUUE3uJ/2Ox3Lz1K+o=
go.mongodb.org/mongo-driver v1.11.6/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -537,7 +517,6 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
@@ -549,8 +528,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -599,8 +578,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -695,8 +674,8 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg=
golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -711,8 +690,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.129.0 h1:2XbdjjNfFPXQyufzQVwPf1RRnHH8Den2pfNE2jw7L8w=
google.golang.org/api v0.129.0/go.mod h1:dFjiXlanKwWE3612X97llhsoI36FAoIiRj3aTl5b/zE=
google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg=
google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -753,8 +732,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -767,8 +746,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+1 -1
View File
@@ -476,7 +476,7 @@ func run(state overseer.State) {
}
}
// asynchronously wait for scanning to finish and cleanup
go e.Finish(ctx, logFatal)
go e.Finish(ctx)
if !*jsonLegacy && !*jsonOut {
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
+1 -33
View File
@@ -2,7 +2,6 @@ package common
import (
"fmt"
"regexp"
"strconv"
"strings"
)
@@ -16,11 +15,7 @@ const RegexPattern = "0-9a-z"
const AlphaNumPattern = "0-9a-zA-Z"
const HexPattern = "0-9a-f"
type RegexState struct {
compiledRegex *regexp.Regexp
}
// Custom Regex functions
//Custom Regex functions
func BuildRegex(pattern string, specialChar string, length int) string {
return fmt.Sprintf(`\b([%s%s]{%s})\b`, pattern, specialChar, strconv.Itoa(length))
}
@@ -42,30 +37,3 @@ func RangeValidation(rangeInput string) bool {
func ToUpperCase(input string) string {
return strings.ToUpper(input)
}
func (r RegexState) Matches(data []byte) []string {
matches := r.compiledRegex.FindAllStringSubmatch(string(data), -1)
res := make([]string, 0, len(matches))
// trim off spaces and different quote types ('").
for i := range matches {
res = append(res, strings.Trim(matches[i][1], `"' )`))
}
return res
}
// UsernameRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
func UsernameRegexCheck(pattern string) RegexState {
raw := fmt.Sprintf(`(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})\b`, pattern)
return RegexState{regexp.MustCompile(raw)}
}
// PasswordRegexCheck constructs an username usernameRegex pattern from a given pattern of excluded characters.
func PasswordRegexCheck(pattern string) RegexState {
raw := fmt.Sprintf(`(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:%+v]{4,40})`, pattern)
return RegexState{regexp.MustCompile(raw)}
}
-59
View File
@@ -1,59 +0,0 @@
package common
import (
"github.com/stretchr/testify/assert"
"regexp"
"testing"
)
const (
usernamePattern = `?()/\+=\s\n`
passwordPattern = `^<>;.*&|£\n\s`
usernameRegex = `(?im)(?:user|usr)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`
passwordRegex = `(?im)(?:pass|password)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:^<>;.*&|£\n\s]{4,40})`
)
func TestUsernameRegexCheck(t *testing.T) {
usernameRegexPat := UsernameRegexCheck(usernamePattern)
expectedRegexPattern := regexp.MustCompile(usernameRegex)
if usernameRegexPat.compiledRegex.String() != expectedRegexPattern.String() {
t.Errorf("\n got %v \n want %v", usernameRegexPat.compiledRegex, expectedRegexPattern)
}
testString := `username = "johnsmith123"
username='johnsmith123'
username:="johnsmith123"
username = johnsmith123
username=johnsmith123`
expectedStr := []string{"johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123", "johnsmith123"}
usernameRegexMatches := usernameRegexPat.Matches([]byte(testString))
assert.Exactly(t, usernameRegexMatches, expectedStr)
}
func TestPasswordRegexCheck(t *testing.T) {
passwordRegexPat := PasswordRegexCheck(passwordPattern)
expectedRegexPattern := regexp.MustCompile(passwordRegex)
assert.Equal(t, passwordRegexPat.compiledRegex, expectedRegexPattern)
testString := `password = "johnsmith123$!"
password='johnsmith123$!'
password:="johnsmith123$!"
password = johnsmith123$!
password=johnsmith123$!
PasswordAuthenticator(username, "johnsmith123$!")`
expectedStr := []string{"johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!", "johnsmith123$!",
"johnsmith123$!"}
passwordRegexMatches := passwordRegexPat.Matches([]byte(testString))
assert.Exactly(t, passwordRegexMatches, expectedStr)
}
-19
View File
@@ -1,11 +1,5 @@
package common
import (
"bufio"
"io"
"strings"
)
func AddStringSliceItem(item string, slice *[]string) {
for _, i := range *slice {
if i == item {
@@ -23,16 +17,3 @@ func RemoveStringSliceItem(item string, slice *[]string) {
}
}
}
func ResponseContainsSubstring(reader io.ReadCloser, target string) (bool, error) {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
if strings.Contains(scanner.Text(), target) {
return true, nil
}
}
if err := scanner.Err(); err != nil {
return false, err
}
return false, nil
}
-59
View File
@@ -1,9 +1,7 @@
package common
import (
"io"
"reflect"
"strings"
"testing"
)
@@ -76,60 +74,3 @@ func TestRemoveItem(t *testing.T) {
}
}
}
// Test ParseResponseForKeywords with a reader that contains the keyword and a reader that doesn't.
func TestParseResponseForKeywords(t *testing.T) {
testCases := []struct {
name string
input string
keyword string
expected bool
}{
{
name: "Should find keyword",
input: "ey: abc",
keyword: "ey",
expected: true,
},
{
name: "Should not find keyword",
input: "fake response",
keyword: "ey",
expected: false,
},
{
name: "Empty string",
input: "",
keyword: "ey",
expected: false,
},
{
name: "Keyword at end",
input: "abc ey",
keyword: "ey",
expected: true,
},
{
name: "Keyword at start",
input: "ey abc",
keyword: "ey",
expected: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
testReader := strings.NewReader(tc.input)
testReadCloser := io.NopCloser(testReader)
found, err := ResponseContainsSubstring(testReadCloser, tc.keyword)
if err != nil {
t.Errorf("Error: %v", err)
}
if found != tc.expected {
t.Errorf("Expected %v, got %v", tc.expected, found)
}
})
}
}
-152
View File
@@ -1,152 +0,0 @@
package couchbase
import (
"context"
"fmt"
"log"
"regexp"
"strings"
"time"
"unicode"
"github.com/couchbase/gocb/v2"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
connectionStringPat = regexp.MustCompile(`\bcb\.[a-z0-9]+\.cloud\.couchbase\.com\b`)
usernamePat = `?()/\+=\s\n`
passwordPat = `^<>;.*&|£\n\s`
//passwordPat = regexp.MustCompile(`(?i)(?:pass|pwd)(?:.|[\n\r]){0,15}(\b[^<>;.*&|£\n\s]{8,100}$)`)
// passwordPat = regexp.MustCompile(`(?im)(?:pass|pwd)\S{0,40}?[:=\s]{1,3}[ '"=]{0,1}([^:?()/\+=\s\n]{4,40})\b`)
)
func meetsCouchbasePasswordRequirements(password string) (string, bool) {
var hasLower, hasUpper, hasNumber, hasSpecialChar bool
for _, char := range password {
switch {
case unicode.IsLower(char):
hasLower = true
case unicode.IsUpper(char):
hasUpper = true
case unicode.IsNumber(char):
hasNumber = true
case unicode.IsPunct(char) || unicode.IsSymbol(char):
hasSpecialChar = true
}
if hasLower && hasUpper && hasNumber && hasSpecialChar {
return password, true
}
}
return "", false
}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"couchbase://", "couchbases://"}
}
// FromData will find and optionally verify Couchbase secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
connectionStringMatches := connectionStringPat.FindAllStringSubmatch(dataStr, -1)
// prepend 'couchbases://' to the connection string as the connection
// string format is couchbases://cb.stuff.cloud.couchbase.com but the
// cb.stuff.cloud.couchbase.com may be separated from the couchbases:// in codebases.
for i, connectionStringMatch := range connectionStringMatches {
connectionStringMatches[i][0] = "couchbases://" + connectionStringMatch[0]
}
usernameRegexState := common.UsernameRegexCheck(usernamePat)
usernameMatches := usernameRegexState.Matches(data)
passwordRegexState := common.PasswordRegexCheck(passwordPat)
passwordMatches := passwordRegexState.Matches(data)
for _, connectionStringMatch := range connectionStringMatches {
resConnectionStringMatch := strings.TrimSpace(connectionStringMatch[0])
for _, resUsernameMatch := range usernameMatches {
for _, resPasswordMatch := range passwordMatches {
_, metPasswordRequirements := meetsCouchbasePasswordRequirements(resPasswordMatch)
if !metPasswordRequirements {
continue
}
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Couchbase,
Raw: []byte(fmt.Sprintf("%s:%s@%s", resUsernameMatch, resPasswordMatch, resConnectionStringMatch)),
}
if verify {
options := gocb.ClusterOptions{
Authenticator: gocb.PasswordAuthenticator{
Username: resUsernameMatch,
Password: resPasswordMatch,
},
}
// Sets a pre-configured profile called "wan-development" to help avoid latency issues
// when accessing Capella from a different Wide Area Network
// or Availability Zone (e.g. your laptop).
if err := options.ApplyProfile(gocb.ClusterConfigProfileWanDevelopment); err != nil {
log.Fatal("apply profile err", err)
}
// Initialize the Connection
cluster, err := gocb.Connect(resConnectionStringMatch, options)
if err != nil {
continue
}
// We'll ping the KV nodes in our cluster.
pings, err := cluster.Ping(&gocb.PingOptions{
Timeout: time.Second * 5,
})
if err != nil {
continue
}
for _, ping := range pings.Services {
for _, pingEndpoint := range ping {
if pingEndpoint.State == gocb.PingStateOk {
s1.Verified = true
break
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resPasswordMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
}
results = append(results, s1)
}
}
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Couchbase
}
-122
View File
@@ -1,122 +0,0 @@
//go:build detectors
// +build detectors
package couchbase
import (
"context"
"fmt"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestCouchbase_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
endpoint := testSecrets.MustGetField("COUCHBASE_ENDPOINT")
username := testSecrets.MustGetField("COUCHBASE_USERNAME")
password := testSecrets.MustGetField("COUCHBASE_PASSWORD")
inactiveSecret := testSecrets.MustGetField("COUCHBASE_INACTIVE_PASSWORD")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, password)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Couchbase,
Verified: true,
},
},
wantErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("db uri: %s \n username = %s \n password = %s", endpoint, username, inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Couchbase,
Verified: false,
},
},
wantErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Couchbase.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Couchbase.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+16 -4
View File
@@ -3,8 +3,10 @@ package docusign
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"github.com/go-errors/errors"
"io"
"net/http"
"regexp"
"strings"
@@ -81,15 +83,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return nil, errors.WrapPrefix(err, "Error making request", 0)
}
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "ey")
res.Body.Close()
// Read the response body
body, err := io.ReadAll(res.Body)
if err != nil {
return nil, err
return nil, errors.WrapPrefix(err, "Error reading response body", 0)
}
// Close the response body
res.Body.Close()
// Parse the response body into a Response struct
var parsedResponse Response
err = json.Unmarshal(body, &parsedResponse)
if err != nil {
return nil, errors.WrapPrefix(err, "Error parsing response", 0)
}
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
if res.StatusCode >= 200 && res.StatusCode < 300 && strings.HasPrefix(parsedResponse.AccessToken, "ey") {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+4 -4
View File
@@ -6,7 +6,7 @@ import (
"unicode"
)
var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}
var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample"}
type FalsePositive string
@@ -31,9 +31,9 @@ var FalsePositiveWordlists = Wordlists{
programmingBookWords: bytesToCleanWordList(programmingBookWords),
}
// IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
// Currently that includes: No number, english word in key, or matches common example pattens.
// Only the secret key material should be passed into this function
//IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met
//Currently that includes: No number, english word in key, or matches common example pattens.
//Only the secret key material should be passed into this function
func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {
for _, fp := range falsePositives {
@@ -1,8 +1,7 @@
package opsgenie
package image4
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
@@ -21,20 +20,16 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"opsgenie"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"image4"}) + `\b([0-9a-zA-Z]{22}[0-9a-zA-Z\=]{2})`)
)
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Opsgenie
}
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"opsgenie"}
return []string{"image4"}
}
// FromData will find and optionally verify Opsgenie secrets in a given set of bytes.
// FromData will find and optionally verify Image4 secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -47,16 +42,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Opsgenie,
DetectorType: detectorspb.DetectorType_Image4,
Raw: []byte(resMatch),
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.image4.io/v1.0/subscription", nil)
if err != nil {
continue
}
req.Header.Add("Authorization", fmt.Sprintf("GenieKey %s", resMatch))
req.SetBasicAuth(resMatch, "")
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
@@ -74,5 +69,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
results = append(results, s1)
}
return detectors.CleanResults(results), nil
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Image4
}
@@ -1,4 +1,7 @@
package opsgenie
//go:build detectors
// +build detectors
package image4
import (
"context"
@@ -13,15 +16,15 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestOpsgenie_FromChunk(t *testing.T) {
func TestImage4_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("OPSGENIE")
inactiveSecret := testSecrets.MustGetField("OPSGENIE_INACTIVE")
secret := testSecrets.MustGetField("IMAGE4")
inactiveSecret := testSecrets.MustGetField("IMAGE4_INACTIVE")
type args struct {
ctx context.Context
@@ -40,12 +43,12 @@ func TestOpsgenie_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a opsgenie secret %s within", secret)),
data: []byte(fmt.Sprintf("You can find a image4 secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Opsgenie,
DetectorType: detectorspb.DetectorType_Image4,
Verified: true,
},
},
@@ -56,12 +59,12 @@ func TestOpsgenie_FromChunk(t *testing.T) {
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a opsgenie secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
data: []byte(fmt.Sprintf("You can find a image4 secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_Opsgenie,
DetectorType: detectorspb.DetectorType_Image4,
Verified: false,
},
},
@@ -84,7 +87,7 @@ func TestOpsgenie_FromChunk(t *testing.T) {
s := Scanner{}
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Opsgenie.FromData() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("Image4.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
@@ -94,7 +97,7 @@ func TestOpsgenie_FromChunk(t *testing.T) {
got[i].Raw = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Opsgenie.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
t.Errorf("Image4.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
+1 -1
View File
@@ -63,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
resp, err := common.SaneHttpClient().Do(req)
if err != nil {
continue
return results, err
}
defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+20 -15
View File
@@ -38,18 +38,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
keyMatches := keyPat.FindAllString(dataStr, -1)
for _, key := range keyMatches {
secMatches := secretPat.FindAllString(dataStr, -1)
for _, secret := range secMatches {
if verify {
secMatches := secretPat.FindAllString(dataStr, -1)
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_RazorPay,
Raw: []byte(key),
RawV2: []byte(key + secret),
Redacted: key,
}
for _, secret := range secMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_RazorPay,
Raw: []byte(key),
RawV2: []byte(key + secret),
Redacted: key,
}
if verify {
req, err := http.NewRequest("GET", "https://api.razorpay.com/v1/items?count=1", nil)
if err != nil {
continue
@@ -65,21 +66,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if res.StatusCode >= 200 && res.StatusCode < 300 {
if json.Valid(bodyBytes) {
s1.Verified = true
} else {
s1.Verified = false
}
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
continue
results = append(results, s1)
}
results = append(results, s1)
}
}
return detectors.CleanResults(results), nil
results = detectors.CleanResults(results)
return
}
func (s Scanner) Type() detectorspb.DetectorType {
+11 -21
View File
@@ -2,7 +2,7 @@ package shortcut
import (
"context"
"github.com/go-errors/errors"
"io"
"net/http"
"regexp"
"strings"
@@ -21,7 +21,7 @@ var (
client = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"shortcut"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"shortcut"}) + `\b([0-9a-f-]{36})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -48,33 +48,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.app.shortcut.com/api/v3/member", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.app.shortcut.com/api/v3/projects", nil)
if err != nil {
continue
}
req.Header.Add("Content-Type", "application/json")
req.Header.Add("Shortcut-Token", resMatch)
res, err := client.Do(req)
if err != nil {
return nil, errors.WrapPrefix(err, "Error making request", 0)
}
verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "name")
res.Body.Close()
if err != nil {
return nil, err
}
if err == nil {
if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
defer res.Body.Close()
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
continue
}
body := string(bodyBytes)
if strings.Contains(body, "app_url") {
s1.Verified = true
} else {
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
}
}
}
+1 -1
View File
@@ -19,7 +19,7 @@ import (
func TestShortcut_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
+1 -1
View File
@@ -62,7 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
body := string(bodyBytes)
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
if (res.StatusCode >= 200 && res.StatusCode < 300) || (res.StatusCode == 400 && strings.Contains(body, "no_text")) {
s1.Verified = true
}
}
+1 -1
View File
@@ -24,7 +24,7 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"sql", "database", "Data Source", "Server=", "Network address="}
return []string{"sql", "database", "Data Source"}
}
// FromData will find and optionally verify SpotifyKey secrets in a given set of bytes.
+16 -17
View File
@@ -21,7 +21,6 @@ var (
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
client = common.SaneHttpClient()
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -44,15 +43,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
sidMatches := sidPat.FindAllString(dataStr, -1)
for _, sid := range sidMatches {
for _, key := range keyMatches {
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Twilio,
Raw: []byte(sid),
RawV2: []byte(sid + key),
Redacted: sid,
}
s := detectors.Result{
DetectorType: detectorspb.DetectorType_Twilio,
Raw: []byte(sid),
Redacted: sid,
}
if verify {
if verify {
client := common.SaneHttpClient()
for _, key := range keyMatches {
form := url.Values{}
form.Add("FriendlyName", "MyServiceName")
@@ -72,22 +71,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
res.Body.Close() // The request body is unused.
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
s.Verified = true
}
}
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
continue
}
if len(keyMatches) > 0 {
results = append(results, s1)
}
if len(keyMatches) > 0 {
results = append(results, s)
}
}
return detectors.CleanResults(results), nil
return
}
func (s Scanner) Type() detectorspb.DetectorType {
-1
View File
@@ -98,7 +98,6 @@ func TestTwilio_FromChunk(t *testing.T) {
t.Fatal("no raw secret present")
}
got[i].Raw = nil
got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+5 -5
View File
@@ -1,7 +1,6 @@
package engine
import (
"fmt"
"runtime"
"github.com/go-errors/errors"
@@ -39,13 +38,14 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
return errors.WrapPrefix(err, "failed to init Circle CI source", 0)
}
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
err := circleSource.Chunks(ctx, e.ChunksChan())
if err != nil {
return fmt.Errorf("error scanning CircleCI: %w", err)
ctx.Logger().Error(err, "error scanning Circle CI")
}
return nil
})
}()
return nil
}
+2 -2
View File
@@ -323,6 +323,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/iconfinder"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/iexapis"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/iexcloud"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/image4"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/imagekit"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/imagga"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/impala"
@@ -458,7 +459,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/opengraphr"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openuv"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openweather"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/opsgenie"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/optimizely"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/owlbot"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/packagecloud"
@@ -1417,6 +1417,7 @@ func DefaultDetectors() []detectors.Detector {
exportsdk.Scanner{},
rechargepayments.Scanner{},
browserstack.Scanner{},
image4.Scanner{},
crossbrowsertesting.Scanner{},
flowdash.Scanner{},
lunchmoney.Scanner{},
@@ -1524,7 +1525,6 @@ func DefaultDetectors() []detectors.Detector {
aiven.Scanner{},
prefect.Scanner{},
buildkitev2.Scanner{},
opsgenie.Scanner{},
}
}
+5 -5
View File
@@ -1,7 +1,6 @@
package engine
import (
"fmt"
"runtime"
"github.com/go-errors/errors"
@@ -26,13 +25,14 @@ func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
return errors.WrapPrefix(err, "could not init docker source", 0)
}
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
err := dockerSource.Chunks(ctx, e.ChunksChan())
if err != nil {
return fmt.Errorf("error scanning docker image: %w", err)
ctx.Logger().Error(err, "error scanning docker image")
}
return nil
})
}()
return nil
}
+9 -61
View File
@@ -2,7 +2,6 @@ package engine
import (
"bytes"
"fmt"
"reflect"
"runtime"
"strings"
@@ -10,10 +9,7 @@ import (
"sync/atomic"
"time"
"github.com/alecthomas/chroma"
"github.com/alecthomas/chroma/lexers"
ahocorasick "github.com/petar-dambovaliev/aho-corasick"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -25,7 +21,6 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
DetectorsV2 "github.com/trufflesecurity/trufflehog/v3/pkg/v2Detectors"
)
type Engine struct {
@@ -37,7 +32,7 @@ type Engine struct {
chunksScanned uint64
bytesScanned uint64
detectorAvgTime sync.Map
sourcesWg *errgroup.Group
sourcesWg sync.WaitGroup
workersWg sync.WaitGroup
// filterUnverified is used to reduce the number of unverified results.
// If there are multiple unverified results for the same chunk for the same detector,
@@ -47,9 +42,6 @@ type Engine struct {
// prefilter is a ahocorasick struct used for doing efficient string
// matching given a set of words (keywords from the rules in the config)
prefilter ahocorasick.AhoCorasick
// v2 detection engine map, key is the length of characters in the secret capture group
v2Detector DetectorsV2.DetectorBuckets
}
type EngineOption func(*Engine)
@@ -60,8 +52,6 @@ func WithConcurrency(concurrency int) EngineOption {
}
}
const ignoreTag = "trufflehog:ignore"
func WithDetectors(verify bool, d ...detectors.Detector) EngineOption {
return func(e *Engine) {
if e.detectors == nil {
@@ -119,7 +109,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
chunks: make(chan *sources.Chunk),
results: make(chan detectors.ResultWithMetadata),
detectorAvgTime: sync.Map{},
v2Detector: *DetectorsV2.NewDetectorBuckets(),
}
for _, option := range options {
@@ -134,11 +123,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
}
ctx.Logger().V(2).Info("engine started", "workers", e.concurrency)
sourcesWg, egCtx := errgroup.WithContext(ctx)
sourcesWg.SetLimit(e.concurrency)
e.sourcesWg = sourcesWg
ctx.SetParent(egCtx)
if len(e.decoders) == 0 {
e.decoders = decoders.DefaultDecoders()
}
@@ -204,14 +188,10 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
// Finish waits for running sources to complete and workers to finish scanning
// chunks before closing their respective channels. Once Finish is called, no
// more sources may be scanned by the engine.
func (e *Engine) Finish(ctx context.Context, logFunc func(error, string, ...any)) {
func (e *Engine) Finish(ctx context.Context) {
defer common.RecoverWithExit(ctx)
// wait for the sources to finish putting chunks onto the chunks channel
sourceErr := e.sourcesWg.Wait()
if sourceErr != nil {
logFunc(sourceErr, "error occurred while collecting chunks")
}
e.sourcesWg.Wait()
close(e.chunks)
// wait for the workers to finish processing all of the chunks and putting
// results onto the results channel
@@ -285,29 +265,6 @@ func (e *Engine) detectorWorker(ctx context.Context) {
continue
}
// use chroma to find all strings and comments
l := lexers.Analyse(string(chunk.Data))
if l != nil {
iterator, err := l.Tokenise(nil, string(chunk.Data))
if err != nil {
fmt.Println("error: ", err)
}
for _, token := range iterator.Tokens() {
if token.Type == chroma.LiteralString || token.Type == chroma.Comment {
for _, v2D := range e.v2Detector.GetDetectors(len(token.Value)) {
for _, v2 := range v2D.Credentials {
if len(token.Value) >= v2.CharacterRange.Min {
if v2.Regex.MatchString(token.Value) {
fmt.Println("cred: ", v2D.Name, "regex:", v2.Regex.String(), "string: ", token.Value)
}
}
}
}
}
}
}
continue
// build a map of all keywords that were matched in the chunk
for _, m := range e.prefilter.FindAll(string(decoded.Data)) {
matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}
@@ -348,7 +305,6 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
for _, result := range results {
resultChunk := chunk
ignoreLinePresent := false
if SupportsLineNumbers(chunk.SourceType) {
copyChunk := *chunk
copyMetaDataClone := proto.Clone(chunk.SourceMetadata)
@@ -356,12 +312,9 @@ func (e *Engine) detectorWorker(ctx context.Context) {
copyChunk.SourceMetadata = copyMetaData
}
fragStart, mdLine := FragmentFirstLine(&copyChunk)
ignoreLinePresent = SetResultLineNumber(&copyChunk, &result, fragStart, mdLine)
SetResultLineNumber(&copyChunk, &result, fragStart, mdLine)
resultChunk = &copyChunk
}
if ignoreLinePresent {
continue
}
result.DecoderType = decoderType
e.results <- detectors.CopyMetadata(resultChunk, result)
@@ -414,18 +367,14 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
}
// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) int64 {
lines := bytes.Split(chunk.Data, []byte("\n"))
for i, line := range lines {
if bytes.Contains(line, result.Raw) {
// if the line contains the ignore tag, we should ignore the result
if bytes.Contains(line, []byte(ignoreTag)) {
return int64(i), true
}
return int64(i), false
return int64(i)
}
}
return 0, false
return 0
}
// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
@@ -452,8 +401,7 @@ func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
}
// SetResultLineNumber sets the line number in the provided result.
func SetResultLineNumber(chunk *sources.Chunk, result *detectors.Result, fragStart int64, mdLine *int64) bool {
offset, skip := FragmentLineOffset(chunk, result)
func SetResultLineNumber(chunk *sources.Chunk, result *detectors.Result, fragStart int64, mdLine *int64) {
offset := FragmentLineOffset(chunk, result)
*mdLine = fragStart + offset
return skip
}
-64
View File
@@ -1,64 +0,0 @@
package engine
import (
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
func TestFragmentLineOffset(t *testing.T) {
tests := []struct {
name string
chunk *sources.Chunk
result *detectors.Result
expectedLine int64
ignore bool
}{
{
name: "ignore found on same line",
chunk: &sources.Chunk{
Data: []byte("line1\nline2\nsecret here trufflehog:ignore\nline4"),
},
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 2,
ignore: true,
},
{
name: "no ignore",
chunk: &sources.Chunk{
Data: []byte("line1\nline2\nsecret here\nline4"),
},
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 2,
ignore: false,
},
{
name: "ignore on different line",
chunk: &sources.Chunk{
Data: []byte("line1\nline2\ntrufflehog:ignore\nline4\nsecret here\nline6"),
},
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 4,
ignore: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
lineOffset, isIgnored := FragmentLineOffset(tt.chunk, tt.result)
if lineOffset != tt.expectedLine {
t.Errorf("Expected line offset to be %d, got %d", tt.expectedLine, lineOffset)
}
if isIgnored != tt.ignore {
t.Errorf("Expected isIgnored to be %v, got %v", tt.ignore, isIgnored)
}
})
}
}
+5 -5
View File
@@ -1,7 +1,6 @@
package engine
import (
"fmt"
"runtime"
"github.com/go-errors/errors"
@@ -38,13 +37,14 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
return errors.WrapPrefix(err, "could not init filesystem source", 0)
}
fileSystemSource.WithFilter(c.Filter)
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
err := fileSystemSource.Chunks(ctx, e.ChunksChan())
if err != nil {
return fmt.Errorf("error scanning filesystem: %w", err)
ctx.Logger().Error(err, "error scanning filesystem")
}
return nil
})
}()
return nil
}
+5 -4
View File
@@ -54,13 +54,14 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
return fmt.Errorf("failed to initialize GCS source: %w", err)
}
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
if err := source.Chunks(ctx, e.ChunksChan()); err != nil {
return fmt.Errorf("could not scan GCS: %w", err)
ctx.Logger().Error(err, "could not scan GCS")
}
return nil
})
}()
return nil
}
+2 -21
View File
@@ -5,7 +5,6 @@ import (
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -56,30 +55,12 @@ func TestScanGCS(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
e := Start(ctx,
WithConcurrency(1),
WithDecoders(decoders.DefaultDecoders()...),
WithDetectors(false, DefaultDetectors()...),
)
go func() {
resultCount := 0
for range e.ResultsChan() {
resultCount++
}
}()
err := e.ScanGCS(ctx, test.gcsConfig)
e := &Engine{}
err := e.ScanGCS(context.Background(), test.gcsConfig)
if err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") {
t.Errorf("ScanGCS() got: %v, want: %v", err, nil)
return
}
logFatalFunc := func(_ error, _ string, _ ...any) {
t.Fatalf("error logging function should not have been called")
}
e.Finish(ctx, logFatalFunc)
if err == nil && test.wantErr {
t.Errorf("ScanGCS() got: %v, want: %v", err, "error")
+5 -4
View File
@@ -66,13 +66,14 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
"source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(),
"source_name", "git",
)
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
if err != nil {
return fmt.Errorf("could not scan repo: %w", err)
ctx.Logger().Error(err, "could not scan repo")
}
return nil
})
}()
return nil
}
+32 -41
View File
@@ -53,45 +53,39 @@ func TestGitEngine(t *testing.T) {
base: "2f251b8c1e72135a375b659951097ec7749d4af9",
},
} {
t.Run(tName, func(t *testing.T) {
e := Start(ctx,
WithConcurrency(1),
WithDecoders(decoders.DefaultDecoders()...),
WithDetectors(false, DefaultDetectors()...),
)
cfg := sources.GitConfig{
RepoPath: path,
HeadRef: tTest.branch,
BaseRef: tTest.base,
MaxDepth: tTest.maxDepth,
Filter: tTest.filter,
}
if err := e.ScanGit(ctx, cfg); err != nil {
return
}
logFatalFunc := func(_ error, _ string, _ ...any) {
t.Fatalf("error logging function should not have been called")
}
go e.Finish(ctx, logFatalFunc)
resultCount := 0
for result := range e.ResultsChan() {
switch meta := result.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
}
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
}
e := Start(ctx,
WithConcurrency(1),
WithDecoders(decoders.DefaultDecoders()...),
WithDetectors(false, DefaultDetectors()...),
)
cfg := sources.GitConfig{
RepoPath: path,
HeadRef: tTest.branch,
BaseRef: tTest.base,
MaxDepth: tTest.maxDepth,
Filter: tTest.filter,
}
if err := e.ScanGit(ctx, cfg); err != nil {
return
}
go e.Finish(ctx)
resultCount := 0
for result := range e.ResultsChan() {
switch meta := result.SourceMetadata.GetData().(type) {
case *source_metadatapb.MetaData_Git:
if tTest.expected[meta.Git.Commit].B != string(result.Raw) {
t.Errorf("%s: unexpected result. Got: %s, Expected: %s", tName, string(result.Raw), tTest.expected[meta.Git.Commit].B)
}
resultCount++
if tTest.expected[meta.Git.Commit].LineNumber != result.SourceMetadata.GetGit().Line {
t.Errorf("%s: unexpected line number. Got: %d, Expected: %d", tName, result.SourceMetadata.GetGit().Line, tTest.expected[meta.Git.Commit].LineNumber)
}
}
resultCount++
}
if resultCount != len(tTest.expected) {
t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
}
})
}
if resultCount != len(tTest.expected) {
t.Errorf("%s: unexpected number of results. Got: %d, Expected: %d", tName, resultCount, len(tTest.expected))
}
}
}
@@ -130,8 +124,5 @@ func BenchmarkGitEngine(b *testing.B) {
return
}
}
logFatalFunc := func(_ error, _ string, _ ...any) {
b.Fatalf("error logging function should not have been called")
}
e.Finish(ctx, logFatalFunc)
e.Finish(ctx)
}
+5 -6
View File
@@ -1,8 +1,6 @@
package engine
import (
"fmt"
gogit "github.com/go-git/go-git/v5"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -60,13 +58,14 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
scanOptions := git.NewScanOptions(opts...)
source.WithScanOptions(scanOptions)
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
err := source.Chunks(ctx, e.ChunksChan())
if err != nil {
return fmt.Errorf("could not scan github: %w", err)
ctx.Logger().Error(err, "could not scan github")
}
return nil
})
}()
return nil
}
+5 -4
View File
@@ -63,13 +63,14 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
}
gitlabSource.WithScanOptions(scanOptions)
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
err := gitlabSource.Chunks(ctx, e.ChunksChan())
if err != nil {
return fmt.Errorf("error scanning GitLab: %w", err)
ctx.Logger().Error(err, "error scanning GitLab")
}
return nil
})
}()
return nil
}
+5 -4
View File
@@ -65,13 +65,14 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
return errors.WrapPrefix(err, "failed to init S3 source", 0)
}
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
err := s3Source.Chunks(ctx, e.ChunksChan())
if err != nil {
return fmt.Errorf("error scanning S3: %w", err)
ctx.Logger().Error(err, "error scanning S3")
}
return nil
})
}()
return nil
}
+5 -5
View File
@@ -1,7 +1,6 @@
package engine
import (
"fmt"
"os"
"github.com/go-errors/errors"
@@ -54,13 +53,14 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
return err
}
e.sourcesWg.Go(func() error {
e.sourcesWg.Add(1)
go func() {
defer common.RecoverWithExit(ctx)
defer e.sourcesWg.Done()
err := source.Chunks(ctx, e.ChunksChan())
if err != nil {
return fmt.Errorf("could not scan syslog: %w", err)
ctx.Logger().Error(err, "could not scan syslog")
}
return nil
})
}()
return nil
}
+3 -3
View File
@@ -140,10 +140,10 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
return c.executeCommand(ctx, cmd)
}
// Staged parses the output of the `git diff` command for the `source` path.
func (c *Parser) Staged(ctx context.Context, source string) (chan Commit, error) {
// Unstaged parses the output of the `git diff` command for the `source` path.
func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, error) {
// Provide the --cached flag to diff to get the diff of the staged changes.
args := []string{"-C", source, "diff", "-p", "--cached", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z"}
args := []string{"-C", source, "diff", "-p", "--cached", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
cmd := exec.Command("git", args...)
+2 -6
View File
@@ -991,7 +991,6 @@ const (
DetectorType_Aiven DetectorType = 917
DetectorType_Prefect DetectorType = 918
DetectorType_Docusign DetectorType = 919
DetectorType_Couchbase DetectorType = 920
)
// Enum value maps for DetectorType.
@@ -1913,7 +1912,6 @@ var (
917: "Aiven",
918: "Prefect",
919: "Docusign",
920: "Couchbase",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2832,7 +2830,6 @@ var (
"Aiven": 917,
"Prefect": 918,
"Docusign": 919,
"Couchbase": 920,
}
)
@@ -3197,7 +3194,7 @@ var file_detectors_proto_rawDesc = []byte{
0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x89, 0x73, 0x0a,
0x02, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xf9, 0x72, 0x0a,
0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d,
0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a,
@@ -4117,8 +4114,7 @@ var file_detectors_proto_rawDesc = []byte{
0x0b, 0x4e, 0x75, 0x47, 0x65, 0x74, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, 0x10, 0x94, 0x07, 0x12,
0x0a, 0x0a, 0x05, 0x41, 0x69, 0x76, 0x65, 0x6e, 0x10, 0x95, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x50,
0x72, 0x65, 0x66, 0x65, 0x63, 0x74, 0x10, 0x96, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x44, 0x6f, 0x63,
0x75, 0x73, 0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x43, 0x6f, 0x75, 0x63,
0x68, 0x62, 0x61, 0x73, 0x65, 0x10, 0x98, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
0x75, 0x73, 0x69, 0x67, 0x6e, 0x10, 0x97, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,
+1 -3
View File
@@ -31,12 +31,10 @@ type Source struct {
jobPool *errgroup.Group
sources.Progress
client *http.Client
sources.CommonSourceUnitUnmarshaller
}
// Ensure the Source satisfies the interfaces at compile time.
// Ensure the Source satisfies the interface at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
+1 -3
View File
@@ -30,14 +30,12 @@ type Source struct {
verify bool
conn sourcespb.Docker
sources.Progress
sources.CommonSourceUnitUnmarshaller
}
var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
// Ensure the Source satisfies the interfaces at compile time.
// Ensure the Source satisfies the interface at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
+1 -3
View File
@@ -39,12 +39,10 @@ type Source struct {
log logr.Logger
filter *common.Filter
sources.Progress
sources.CommonSourceUnitUnmarshaller
}
// Ensure the Source satisfies the interfaces at compile time
// Ensure the Source satisfies the interface at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
+1 -3
View File
@@ -30,9 +30,8 @@ import (
const defaultCachePersistIncrement = 2500
// Ensure the Source satisfies the interfaces at compile time.
// Ensure the Source satisfies the interface at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
@@ -70,7 +69,6 @@ type Source struct {
mu sync.Mutex
sources.Progress // progress is not thread safe
sources.CommonSourceUnitUnmarshaller
}
// persistableCache is a wrapper around cache.Cache that allows
+4 -9
View File
@@ -67,9 +67,8 @@ func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName s
}
}
// Ensure the Source satisfies the interfaces at compile time.
// Ensure the Source satisfies the interface at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
@@ -482,7 +481,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
// Get the URL metadata for reporting (may be empty).
urlMetadata := getSafeRemoteURL(repo, "origin")
commitChan, err := gitparse.NewParser().Staged(ctx, path)
commitChan, err := gitparse.NewParser().Unstaged(ctx, path)
if err != nil {
return err
}
@@ -739,7 +738,7 @@ func PrepareRepoSinceCommit(ctx context.Context, uriString, commitHash string) (
var path string
switch {
case uri.User != nil:
ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri.Redacted())
ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
password, ok := uri.User.Password()
if !ok {
return "", true, fmt.Errorf("password must be included in Git repo URL when username is provided")
@@ -777,7 +776,7 @@ func PrepareRepo(ctx context.Context, uriString string) (string, bool, error) {
remote = true
switch {
case uri.User != nil:
ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri.Redacted())
ctx.Logger().V(1).Info("cloning repo with authentication", "uri", uri)
password, ok := uri.User.Password()
if !ok {
return "", remote, fmt.Errorf("password must be included in Git repo URL when username is provided")
@@ -876,7 +875,3 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so
return nil
}
func (s *Source) UnmarshalSourceUnit(data []byte) (sources.SourceUnit, error) {
return UnmarshalUnit(data)
}
-40
View File
@@ -1,40 +0,0 @@
package git
import (
"encoding/json"
"fmt"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
const (
UnitRepo string = "repo"
UnitDir string = "dir"
)
// Ensure SourceUnit implements the interface at compile time.
var _ sources.SourceUnit = SourceUnit{}
// A git source unit can be two kinds of units: either a local directory path
// or a remote repository.
type SourceUnit struct {
Kind string `json:"kind"`
ID string `json:"id"`
}
// Implement sources.SourceUnit interface.
func (u SourceUnit) SourceUnitID() string {
return u.ID
}
// Helper function to unmarshal raw bytes into our SourceUnit struct.
func UnmarshalUnit(data []byte) (sources.SourceUnit, error) {
var unit SourceUnit
if err := json.Unmarshal(data, &unit); err != nil {
return nil, err
}
if unit.ID == "" || (unit.Kind != UnitRepo && unit.Kind != UnitDir) {
return nil, fmt.Errorf("not a git.SourceUnit")
}
return unit, nil
}
+86 -4
View File
@@ -69,17 +69,14 @@ type Source struct {
mu sync.Mutex
publicMap map[string]source_metadatapb.Visibility
sources.Progress
sources.CommonSourceUnitUnmarshaller
}
func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
s.scanOptions = scanOptions
}
// Ensure the Source satisfies the interfaces at compile time
// Ensure the Source satisfies the interface at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
// Type returns the type of source.
@@ -666,11 +663,17 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
defer func(start time.Time) {
logger.V(2).Info(fmt.Sprintf("scanned %d/%d repos", scanned, len(s.repos)), "repo_size", repoSize, "duration_seconds", time.Since(start).Seconds())
}(now)
fmt.Printf("Scanning repoURL %s, path %+v, repo %+v \n ", repoURL, path, repo)
if err = s.git.ScanRepo(ctx, repo, path, s.scanOptions, chunksChan); err != nil {
scanErrs.Add(fmt.Errorf("error scanning repo %s: %w", repoURL, err))
return nil
}
atomic.AddUint64(&scanned, 1)
fmt.Println("SCANNNNN COMMENTS")
if err = s.scanComments(ctx, repoURL, chunksChan); err != nil {
fmt.Println("test")
}
return nil
})
@@ -919,3 +922,82 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
}
func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error {
s.log.Info("scanning comments")
fmt.Print("repoPath: ", repoPath, "\n")
trimmedURL := removeURLAndSplit(repoPath)
owner := trimmedURL[1]
repo := trimmedURL[2]
//for _, part := range trimmedURL {
// fmt.Println(part)
//}
sortType := "created"
directionType := "desc"
issueOpts := &github.IssueListCommentsOptions{
Sort: &sortType,
Direction: &directionType,
ListOptions: github.ListOptions{
PerPage: defaultPagination,
Page: 1,
},
}
issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, 0, issueOpts)
prOpts := &github.PullRequestListCommentsOptions{
Sort: sortType,
Direction: directionType,
ListOptions: github.ListOptions{
PerPage: defaultPagination,
Page: 1,
},
}
prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, 0, prOpts)
fmt.Println("issue comments: ", issueComments)
fmt.Println("pr comments: ", prComments)
fmt.Println("resp: ", resp)
fmt.Println("err: ", err)
//for _, comment := range comments {
// // Create chunk and send it to the channel.
// chunk := &sources.Chunk{
// SourceName: s.name,
// SourceID: s.SourceID(),
// SourceType: s.Type(),
// SourceMetadata: &source_metadatapb.MetaData{
// Data: &source_metadatapb.MetaData_Github{
// Github: &source_metadatapb.Github{
// Link: sanitizer.UTF8(comment.GetHTMLURL()),
// Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
// Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
// Repository: sanitizer.UTF8(repo),
// Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
// },
// },
// },
// Data: []byte(sanitizer.UTF8(comment.GetBody())),
// Verify: s.verify,
// }
//
// select {
// case <-ctx.Done():
// return ctx.Err()
// case chunksChan <- chunk:
// }
//}
return nil
}
func removeURLAndSplit(url string) []string {
trimmedURL := strings.TrimPrefix(url, "https://")
trimmedURL = strings.TrimSuffix(trimmedURL, ".git")
splitURL := strings.Split(trimmedURL, "/")
return splitURL
}
+1 -3
View File
@@ -47,12 +47,10 @@ type Source struct {
resumeInfoMutex sync.Mutex
sources.Progress
jobPool *errgroup.Group
sources.CommonSourceUnitUnmarshaller
}
// Ensure the Source satisfies the interfaces at compile time.
// Ensure the Source satisfies the interface at compile time.
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source.
// It is used for matching source types in configuration and job input.
+2 -7
View File
@@ -46,12 +46,10 @@ type Source struct {
conn *sourcespb.S3
jobPool *errgroup.Group
maxObjectSize int64
sources.CommonSourceUnitUnmarshaller
}
// Ensure the Source satisfies the interfaces at compile time
// Ensure the Source satisfies the interface at compile time
var _ sources.Source = (*Source)(nil)
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
// Type returns the type of source
func (s *Source) Type() sourcespb.SourceType {
@@ -197,10 +195,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
})
if err != nil {
return fmt.Errorf(
"could not list objects in s3 bucket: bucket %s: %w",
bucket,
err)
s.log.Error(err, "could not list objects in s3 bucket", "bucket", bucket)
}
}
s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
+1 -26
View File
@@ -1,37 +1,12 @@
package sources
import (
"encoding/json"
"fmt"
)
// Ensure CommonSourceUnit implements SourceUnit at compile time.
var _ SourceUnit = CommonSourceUnit{}
// CommonSourceUnit is a common implementation of SourceUnit that Sources can
// use instead of implementing their own types.
type CommonSourceUnit struct {
ID string `json:"source_unit_id"`
}
// SourceUnitID implements the SourceUnit interface.
// Implement the SourceUnit interface.
func (c CommonSourceUnit) SourceUnitID() string {
return c.ID
}
// CommonSourceUnitUnmarshaller is an implementation of SourceUnitUnmarshaller
// for the CommonSourceUnit. A source can embed this struct to gain the
// functionality of converting []byte to a CommonSourceUnit.
type CommonSourceUnitUnmarshaller struct{}
// UnmarshalSourceUnit implements the SourceUnitUnmarshaller interface.
func (c CommonSourceUnitUnmarshaller) UnmarshalSourceUnit(data []byte) (SourceUnit, error) {
var unit CommonSourceUnit
if err := json.Unmarshal(data, &unit); err != nil {
return nil, err
}
if unit.ID == "" {
return nil, fmt.Errorf("not a CommonSourceUnit")
}
return unit, nil
}
File diff suppressed because it is too large Load Diff
-1
View File
@@ -928,7 +928,6 @@ enum DetectorType {
Aiven = 917;
Prefect = 918;
Docusign = 919;
Couchbase = 920;
}
message Result {