Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Miccah Castorina
|
1eae96e283 |
@@ -31,30 +31,6 @@ jobs:
|
||||
service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
|
||||
- name: Test
|
||||
run: make test-integration
|
||||
- name: Set up gotestsum
|
||||
run: |
|
||||
go install gotest.tools/gotestsum@latest
|
||||
mkdir -p tmp/test-results
|
||||
- name: Test
|
||||
run: |
|
||||
CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=sources $(go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
|
||||
if: ${{ success() || failure() }} # always run this step, even if there were previous errors
|
||||
- name: Upload test results to BuildPulse for flaky test detection
|
||||
if: '!cancelled()' # Run this step even when the tests fail. Skip if the workflow is cancelled.
|
||||
uses: buildpulse/buildpulse-action@main
|
||||
with:
|
||||
account: 79229934
|
||||
repository: 77726177
|
||||
path: |
|
||||
tmp/test-results/*.xml
|
||||
key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
|
||||
secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
|
||||
tags: integration
|
||||
- name: Annotate test results
|
||||
uses: mikepenz/action-junit-report@v3
|
||||
if: success() || failure() # always run even if the previous step fails
|
||||
with:
|
||||
report_paths: 'tmp/test-results/*.xml'
|
||||
test-detectors:
|
||||
if: ${{ ! github.event.pull_request.head.repo.fork }}
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/trufflesecurity/trufflehog/v3
|
||||
|
||||
go 1.21
|
||||
go 1.20
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
@@ -11,7 +11,6 @@ require (
|
||||
cloud.google.com/go/secretmanager v1.11.1
|
||||
cloud.google.com/go/storage v1.31.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
@@ -51,7 +50,6 @@ require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
|
||||
github.com/mattn/go-isatty v0.0.18
|
||||
@@ -172,8 +170,8 @@ require (
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
|
||||
github.com/gorilla/css v1.0.0 // indirect
|
||||
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
|
||||
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
|
||||
github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
@@ -194,6 +192,7 @@ require (
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 // indirect
|
||||
github.com/launchdarkly/go-semver v1.0.2 // indirect
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
@@ -204,7 +203,7 @@ require (
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.6.6 // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
github.com/mtibben/percent v0.2.1 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
|
||||
@@ -39,7 +39,6 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJ
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
|
||||
@@ -66,7 +65,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4=
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
@@ -95,14 +93,12 @@ github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1L
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg=
|
||||
github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw=
|
||||
github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA=
|
||||
@@ -241,7 +237,6 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf
|
||||
github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM=
|
||||
github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
|
||||
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819 h1:RIB4cRk+lBqKK3Oy0r2gRX4ui7tuhiZq2SuTtTCi0/0=
|
||||
github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
@@ -267,7 +262,6 @@ github.com/getsentry/sentry-go v0.22.0 h1:XNX9zKbv7baSEI65l+H1GEJgSeIC1c7EN5kluW
|
||||
github.com/getsentry/sentry-go v0.22.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
|
||||
github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
@@ -277,7 +271,6 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
|
||||
github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A=
|
||||
github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
@@ -304,7 +297,6 @@ github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
|
||||
github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
|
||||
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
|
||||
github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
@@ -374,7 +366,6 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
|
||||
github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
@@ -383,7 +374,6 @@ github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8I
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
|
||||
github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
@@ -437,7 +427,6 @@ github.com/jpillora/s3 v1.1.4 h1:YCCKDWzb/Ye9EBNd83ATRF/8wPEy0xd43Rezb6u6fzc=
|
||||
github.com/jpillora/s3 v1.1.4/go.mod h1:yedE603V+crlFi1Kl/5vZJaBu9pUzE9wvKegU/lF2zs=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
|
||||
github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8=
|
||||
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
|
||||
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
@@ -456,7 +445,6 @@ github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
@@ -479,10 +467,7 @@ github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 h1:PAM0GvE0nIUBeOkjdi
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2/go.mod h1:Mztipcz+7ZMatXVun3k/IfPa8IOgUnAqiZawtFh2MRg=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 h1:pp/VBIon5JNbtSw7ih7I5MXN9mXHfo6T5xikKVy52dI=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0/go.mod h1:3TfS6vNsNksPT4LGeGuGKtT3zTjTbbOsCUK3nrj1neA=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0 h1:L3kGILP/6ewikhzhdNkHy1b5y4zs50LueWenVF0sBbs=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw=
|
||||
github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU=
|
||||
github.com/launchdarkly/go-test-helpers/v3 v3.0.2/go.mod h1:u2ZvJlc/DDJTFrshWW50tWMZHLVYXofuSHUfTU/eIwM=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
|
||||
@@ -520,9 +505,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ=
|
||||
github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
|
||||
github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
|
||||
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
|
||||
@@ -564,11 +548,9 @@ github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChl
|
||||
github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=
|
||||
github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
|
||||
github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
|
||||
github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
|
||||
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
|
||||
@@ -644,7 +626,6 @@ github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o
|
||||
github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8=
|
||||
github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck=
|
||||
github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ=
|
||||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM=
|
||||
github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY=
|
||||
github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw=
|
||||
@@ -668,7 +649,6 @@ github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGj
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
|
||||
@@ -905,7 +885,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E=
|
||||
gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
@@ -1001,7 +980,6 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
|
||||
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
||||
@@ -44,18 +44,6 @@ If you think that something should be included outside of these guidelines, plea
|
||||
- Go 1.17+
|
||||
- Make
|
||||
|
||||
### Adding New Token Formats to an Existing Scanner
|
||||
|
||||
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
|
||||
|
||||
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
|
||||
|
||||
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
|
||||
|
||||
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
|
||||
|
||||
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
|
||||
### Creating a new Secret Scanner
|
||||
|
||||
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.
|
||||
@@ -108,7 +96,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
Note: `/tmp/s` is a valid path on Linux. You will need to change that for Windows or OSX, otherwise you will see an error. On Windows you will also need to install [WSL](https://docs.microsoft.com/en-us/windows/wsl/install).
|
||||
|
||||
```bash
|
||||
gcloud secrets versions access --project trufflehog-testing --secret detectors5 latest > /tmp/s
|
||||
gcloud secrets versions access --project trufflehog-testing --secret detectors3 latest > /tmp/s
|
||||
```
|
||||
|
||||
2. Add the secret that you need for testing.
|
||||
@@ -125,9 +113,8 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
|
||||
3. Update the secret version with your modification.
|
||||
|
||||
```bash
|
||||
gcloud secrets versions add --project trufflehog-testing detectors5 --data-file /tmp/s
|
||||
gcloud secrets versions add --project trufflehog-testing detectors3 --data-file /tmp/s
|
||||
```
|
||||
Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb).
|
||||
|
||||
4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go).
|
||||
|
||||
|
||||
@@ -40,18 +40,6 @@ If you think that something should be included outside of these guidelines, plea
|
||||
- Go 1.17+
|
||||
- Make
|
||||
|
||||
### Adding New Token Formats to an Existing Scanner
|
||||
|
||||
In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
|
||||
|
||||
1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
|
||||
|
||||
Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
|
||||
|
||||
2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
|
||||
|
||||
3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
|
||||
|
||||
### Creating a new Secret Scanner
|
||||
|
||||
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.
|
||||
|
||||
@@ -105,11 +105,11 @@ var (
|
||||
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
|
||||
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
|
||||
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
|
||||
s3Scan = cli.Command("s3", "Find credentials in S3 buckets.")
|
||||
s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String()
|
||||
s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings()
|
||||
s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String()
|
||||
s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String()
|
||||
s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
|
||||
s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
|
||||
s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes()
|
||||
@@ -354,10 +354,6 @@ func run(state overseer.State) {
|
||||
printer = new(output.PlainPrinter)
|
||||
}
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
e, err := engine.Start(ctx,
|
||||
engine.WithConcurrency(uint8(*concurrency)),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
@@ -473,7 +469,7 @@ func run(state overseer.State) {
|
||||
Secret: *s3ScanSecret,
|
||||
SessionToken: *s3ScanSessionToken,
|
||||
Buckets: *s3ScanBuckets,
|
||||
Roles: *s3ScanRoleArns,
|
||||
Roles: *s3ScanRoleArns,
|
||||
CloudCred: *s3ScanCloudEnv,
|
||||
MaxObjectSize: int64(*s3ScanMaxObjectSize),
|
||||
}
|
||||
@@ -529,6 +525,10 @@ func run(state overseer.State) {
|
||||
}
|
||||
}
|
||||
|
||||
if !*jsonLegacy && !*jsonOut {
|
||||
fmt.Fprintf(os.Stderr, "🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷\n\n")
|
||||
}
|
||||
|
||||
// Wait for all workers to finish.
|
||||
if err = e.Finish(ctx); err != nil {
|
||||
logFatal(err, "engine failed to finish execution")
|
||||
|
||||
+110
-130
@@ -118,21 +118,118 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
verified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
|
||||
s1.Verified = verified
|
||||
s1.ExtraData = extraData
|
||||
s1.VerificationError = verificationErr
|
||||
// REQUEST VALUES.
|
||||
method := "GET"
|
||||
service := "sts"
|
||||
host := "sts.amazonaws.com"
|
||||
region := "us-east-1"
|
||||
endpoint := "https://sts.amazonaws.com"
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
amzDate := now.Format("20060102T150405Z0700")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// TASK 1: CREATE A CANONICAL REQUEST.
|
||||
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
||||
canonicalURI := "/"
|
||||
canonicalHeaders := "host:" + host + "\n"
|
||||
signedHeaders := "host"
|
||||
algorithm := "AWS4-HMAC-SHA256"
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
|
||||
|
||||
params := req.URL.Query()
|
||||
params.Add("Action", "GetCallerIdentity")
|
||||
params.Add("Version", "2011-06-15")
|
||||
params.Add("X-Amz-Algorithm", algorithm)
|
||||
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
|
||||
params.Add("X-Amz-Date", amzDate)
|
||||
params.Add("X-Amz-Expires", "30")
|
||||
params.Add("X-Amz-SignedHeaders", signedHeaders)
|
||||
|
||||
canonicalQuerystring := params.Encode()
|
||||
payloadHash := GetHash("") // empty payload
|
||||
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
|
||||
|
||||
// TASK 2: CREATE THE STRING TO SIGN.
|
||||
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
|
||||
|
||||
// TASK 3: CALCULATE THE SIGNATURE.
|
||||
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
|
||||
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
|
||||
hash = GetHMAC(hash, []byte(region))
|
||||
hash = GetHMAC(hash, []byte(service))
|
||||
hash = GetHMAC(hash, []byte("aws4_request"))
|
||||
|
||||
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
|
||||
signature := hex.EncodeToString(signature2)
|
||||
|
||||
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
|
||||
params.Add("X-Amz-Signature", signature)
|
||||
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
client := s.verificationClient
|
||||
if client == nil {
|
||||
client = defaultVerificationClient
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
identityInfo := identityRes{}
|
||||
err := json.NewDecoder(res.Body).Decode(&identityInfo)
|
||||
if err == nil {
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{
|
||||
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
res.Body.Close()
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if res.StatusCode == 403 {
|
||||
var body awsErrorResponseBody
|
||||
err = json.NewDecoder(res.Body).Decode(&body)
|
||||
res.Body.Close()
|
||||
if err == nil {
|
||||
// All instances of the code I've seen in the wild are PascalCased but this check is
|
||||
// case-insensitive out of an abundance of caution
|
||||
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
|
||||
// determinate failure - nothing to do
|
||||
} else {
|
||||
// We see a surprising number of false-negative signature mismatch errors here
|
||||
// (The official SDK somehow elicits even more than just making the request
|
||||
// ourselves)
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified {
|
||||
// Unverified results that contain common test words are probably not secrets
|
||||
if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
// Unverified results that look like hashes are probably not secrets
|
||||
if falsePositiveSecretCheck.MatchString(resSecretMatch) {
|
||||
continue
|
||||
}
|
||||
// If the result is unverified and matches something like a git hash, don't include it in the results.
|
||||
if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -145,123 +242,6 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
return awsCustomCleanResults(results), nil
|
||||
}
|
||||
|
||||
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
|
||||
// REQUEST VALUES.
|
||||
method := "GET"
|
||||
service := "sts"
|
||||
host := "sts.amazonaws.com"
|
||||
region := "us-east-1"
|
||||
endpoint := "https://sts.amazonaws.com"
|
||||
now := time.Now().UTC()
|
||||
datestamp := now.Format("20060102")
|
||||
amzDate := now.Format("20060102T150405Z0700")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// TASK 1: CREATE A CANONICAL REQUEST.
|
||||
// http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
||||
canonicalURI := "/"
|
||||
canonicalHeaders := "host:" + host + "\n"
|
||||
signedHeaders := "host"
|
||||
algorithm := "AWS4-HMAC-SHA256"
|
||||
credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
|
||||
|
||||
params := req.URL.Query()
|
||||
params.Add("Action", "GetCallerIdentity")
|
||||
params.Add("Version", "2011-06-15")
|
||||
params.Add("X-Amz-Algorithm", algorithm)
|
||||
params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
|
||||
params.Add("X-Amz-Date", amzDate)
|
||||
params.Add("X-Amz-Expires", "30")
|
||||
params.Add("X-Amz-SignedHeaders", signedHeaders)
|
||||
|
||||
canonicalQuerystring := params.Encode()
|
||||
payloadHash := GetHash("") // empty payload
|
||||
canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
|
||||
|
||||
// TASK 2: CREATE THE STRING TO SIGN.
|
||||
stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
|
||||
|
||||
// TASK 3: CALCULATE THE SIGNATURE.
|
||||
// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
|
||||
hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
|
||||
hash = GetHMAC(hash, []byte(region))
|
||||
hash = GetHMAC(hash, []byte(service))
|
||||
hash = GetHMAC(hash, []byte("aws4_request"))
|
||||
|
||||
signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
|
||||
signature := hex.EncodeToString(signature2)
|
||||
|
||||
// TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
|
||||
params.Add("X-Amz-Signature", signature)
|
||||
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
|
||||
req.URL.RawQuery = params.Encode()
|
||||
|
||||
client := s.verificationClient
|
||||
if client == nil {
|
||||
client = defaultVerificationClient
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
identityInfo := identityRes{}
|
||||
err := json.NewDecoder(res.Body).Decode(&identityInfo)
|
||||
if err == nil {
|
||||
extraData := map[string]string{
|
||||
"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
|
||||
"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
|
||||
"arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
|
||||
}
|
||||
return true, extraData, nil
|
||||
} else {
|
||||
return false, nil, err
|
||||
}
|
||||
} else if res.StatusCode == 403 {
|
||||
// Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
|
||||
// share a key ID but are signed with different secrets the second one will be rejected with a 403 that
|
||||
// carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
|
||||
// valid. Since this is exactly our access pattern, we need to work around it.
|
||||
//
|
||||
// Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
|
||||
// response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
|
||||
// the response to the original second request, or read to its end, or the resubmission will also yield a
|
||||
// SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
|
||||
// SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
|
||||
// after decoding its response body, which requires reading the entire response body, which disables the
|
||||
// workaround.
|
||||
//
|
||||
// We are clearly deep in the guts of AWS implementation details here, so this all might change with no
|
||||
// notice. If you're here because something in this detector broke, you have my condolences.
|
||||
if retryOn403 {
|
||||
return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
|
||||
}
|
||||
var body awsErrorResponseBody
|
||||
err = json.NewDecoder(res.Body).Decode(&body)
|
||||
if err == nil {
|
||||
// All instances of the code I've seen in the wild are PascalCased but this check is
|
||||
// case-insensitive out of an abundance of caution
|
||||
if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
|
||||
return false, nil, nil
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
|
||||
}
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
|
||||
}
|
||||
} else {
|
||||
return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
return false, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
|
||||
if len(results) == 0 {
|
||||
return results
|
||||
|
||||
@@ -248,28 +248,6 @@ func TestAWS_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
wantVerificationError: true,
|
||||
},
|
||||
{
|
||||
name: "verified secret checked directly after unverified secret with same key id",
|
||||
s: scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_AWS,
|
||||
Verified: true,
|
||||
Redacted: "AKIASP2TPHJSQH3FJRUX",
|
||||
ExtraData: map[string]string{
|
||||
"account": "171436882533",
|
||||
"arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
|
||||
"user_id": "AIDASP2TPHJSUFRSTTZX4",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
|
||||
@@ -66,7 +66,7 @@ type ResultWithMetadata struct {
|
||||
// SourceMetadata contains source-specific contextual information.
|
||||
SourceMetadata *source_metadatapb.MetaData
|
||||
// SourceID is the ID of the source that the API uses to map secrets to specific sources.
|
||||
SourceID sources.SourceID
|
||||
SourceID int64
|
||||
// SourceType is the type of Source.
|
||||
SourceType sourcespb.SourceType
|
||||
// SourceName is the name of the Source.
|
||||
|
||||
@@ -96,7 +96,7 @@ func constructRequest(secret, keyID string) (*http.Request, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
params := map[string]any{
|
||||
params := map[string]interface{}{
|
||||
"request": accountDetail,
|
||||
"nonce": time.Now().UnixNano(),
|
||||
}
|
||||
|
||||
@@ -13,18 +13,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
var _ detectors.Versioner = (*Scanner)(nil)
|
||||
|
||||
func (Scanner) Version() int { return 1 }
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
|
||||
@@ -75,11 +70,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
data := fmt.Sprintf("%s:%s", resEmail, resToken)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
|
||||
@@ -95,15 +85,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// If the request is successful and the login reason is not failed authentication, then the token is valid.
|
||||
// This is because Jira returns a 200 status code even if the token is invalid.
|
||||
// Jira returns a default dashboard page.
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if res.Header.Get(loginReasonHeaderKey) != failedAuth {
|
||||
s1.Verified = true
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
|
||||
s1.Verified = true
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,8 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -20,12 +19,12 @@ import (
|
||||
func TestJiraToken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("JIRA_TOKEN")
|
||||
inactiveSecret := testSecrets.MustGetField("JIRA_INACTIVE")
|
||||
token := testSecrets.MustGetField("JIRA_TOKEN")
|
||||
inactiveToken := testSecrets.MustGetField("JIRA_INACTIVE")
|
||||
email := testSecrets.MustGetField("JIRA_EMAIL")
|
||||
domain := testSecrets.MustGetField("JIRA_DOMAIN")
|
||||
|
||||
@@ -35,19 +34,18 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
@@ -56,15 +54,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
@@ -73,8 +70,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -84,48 +80,14 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -134,12 +96,10 @@ func TestJiraToken_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,129 +0,0 @@
|
||||
package jiratoken_v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
var _ detectors.Versioner = (*Scanner)(nil)
|
||||
|
||||
func (Scanner) Version() int { return 2 }
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
|
||||
// Tokens created after Jan 18 2023 use a variable length
|
||||
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`)
|
||||
domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\.[a-zA-Z-0-9]{3,16}\.[a-zA-Z-0-9]{3,16})\b`)
|
||||
emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`)
|
||||
)
|
||||
|
||||
const (
|
||||
failedAuth = "AUTHENTICATED_FAILED"
|
||||
loginReasonHeaderKey = "X-Seraph-LoginReason"
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"jira"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify JiraToken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
tokens := tokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
domains := domainPat.FindAllStringSubmatch(dataStr, -1)
|
||||
emails := emailPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, email := range emails {
|
||||
email = strings.Split(email[0], " ")
|
||||
if len(email) != 2 {
|
||||
continue
|
||||
}
|
||||
resEmail := strings.TrimSpace(email[1])
|
||||
for _, token := range tokens {
|
||||
if len(token) != 2 {
|
||||
continue
|
||||
}
|
||||
resToken := strings.TrimSpace(token[1])
|
||||
for _, domain := range domains {
|
||||
if len(domain) != 2 {
|
||||
continue
|
||||
}
|
||||
resDomain := strings.TrimSpace(domain[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Raw: []byte(resToken),
|
||||
RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
data := fmt.Sprintf("%s:%s", resEmail, resToken)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
// If the request is successful and the login reason is not failed authentication, then the token is valid.
|
||||
// This is because Jira returns a 200 status code even if the token is invalid.
|
||||
// Jira returns a default dashboard page.
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
if res.Header.Get(loginReasonHeaderKey) != failedAuth {
|
||||
s1.Verified = true
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified {
|
||||
if detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_JiraToken
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package jiratoken_v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestJiraToken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("JIRA_V2_TOKEN")
|
||||
inactiveSecret := testSecrets.MustGetField("JIRA_V2_INACTIVE")
|
||||
email := testSecrets.MustGetField("JIRA_V2_EMAIL")
|
||||
domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveSecret, email, domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", secret, email, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_JiraToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -83,8 +83,8 @@ func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
type typeRes struct {
|
||||
Types []struct {
|
||||
Name string `json:"name"`
|
||||
Type any `json:"type"`
|
||||
Parameters []any `json:"parameters"`
|
||||
Name string `json:"name"`
|
||||
Type interface{} `json:"type"`
|
||||
Parameters []interface{} `json:"parameters"`
|
||||
} `json:"types"`
|
||||
}
|
||||
|
||||
@@ -21,13 +21,13 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pd"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty"}) + `\b([a-z]{1}\+[a-zA-Z]{9}\-[a-z]{2}\-[a-z0-9]{5})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"pagerduty", "pd"}
|
||||
return []string{"pagerduty"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify PagerDutyApiKey secrets in a given set of bytes.
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
//go:embed "list.txt"
|
||||
//go:embed "rockyou-15.txt"
|
||||
var rawCrackList []byte
|
||||
var passphrases [][]byte
|
||||
|
||||
@@ -21,17 +21,17 @@ var (
|
||||
ErrUncrackable = errors.New("unable to crack encryption")
|
||||
)
|
||||
|
||||
func crack(in []byte) (any, string, error) {
|
||||
func crack(in []byte) (interface{}, error) {
|
||||
for _, passphrase := range passphrases {
|
||||
parsed, err := ssh.ParseRawPrivateKeyWithPassphrase(in, passphrase)
|
||||
if err != nil {
|
||||
if errors.Is(err, x509.IncorrectPasswordError) {
|
||||
continue
|
||||
} else {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return parsed, string(passphrase), nil
|
||||
return parsed, nil
|
||||
}
|
||||
return nil, "", ErrUncrackable
|
||||
return nil, ErrUncrackable
|
||||
}
|
||||
|
||||
@@ -56,29 +56,23 @@ Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
|
||||
|
||||
func Test_crack(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in []byte
|
||||
wantedPassphrase string
|
||||
wantErr bool
|
||||
name string
|
||||
in []byte
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "crackable",
|
||||
wantErr: false,
|
||||
in: testEncryptedKey,
|
||||
wantedPassphrase: string(testEncryptedKeyCorrectPassword),
|
||||
name: "crackable",
|
||||
wantErr: false,
|
||||
in: testEncryptedKey,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, passphrase, err := crack(tt.in)
|
||||
_, err := crack(tt.in)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("crack() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if passphrase != string(tt.wantedPassphrase) {
|
||||
t.Errorf("crack() passphrase = %v, want %v", passphrase, string(testEncryptedKeyCorrectPassword))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,12 +5,10 @@ import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rsa"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
@@ -20,8 +18,18 @@ var (
|
||||
ErrEncryptedKey = errors.New("key is encrypted")
|
||||
)
|
||||
|
||||
func FingerprintPEMKey(parsedKey any) (string, error) {
|
||||
var pubKey any
|
||||
func FingerprintPEMKey(in []byte) (string, error) {
|
||||
parsedKey, err := ssh.ParseRawPrivateKey(in)
|
||||
if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
|
||||
parsedKey, err = crack(in)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var pubKey interface{}
|
||||
switch privateKey := parsedKey.(type) {
|
||||
case *rsa.PrivateKey:
|
||||
pubKey = &privateKey.PublicKey
|
||||
@@ -36,10 +44,6 @@ func FingerprintPEMKey(parsedKey any) (string, error) {
|
||||
return "", ErrNotSupported
|
||||
}
|
||||
|
||||
return fingerprintPublicKey(pubKey)
|
||||
}
|
||||
|
||||
func fingerprintPublicKey(pubKey any) (string, error) {
|
||||
publickeyBytes, err := x509.MarshalPKIXPublicKey(pubKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -49,8 +53,3 @@ func fingerprintPublicKey(pubKey any) (string, error) {
|
||||
publicKeyFingerprintInHex := hex.EncodeToString(publicKeyFingerprint[:])
|
||||
return publicKeyFingerprintInHex, nil
|
||||
}
|
||||
|
||||
func fingerprintSSHPublicKey(pubKey ssh.PublicKey) string {
|
||||
publicKeyFingerprint := sha256.Sum256(pubKey.Marshal())
|
||||
return fmt.Sprintf("SHA256:%s", base64.RawStdEncoding.EncodeToString(publicKeyFingerprint[:]))
|
||||
}
|
||||
|
||||
@@ -6,14 +6,11 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/AzureAD/microsoft-authentication-library-for-go/apps/errors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
@@ -56,67 +53,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
|
||||
Redacted: token[0:64],
|
||||
}
|
||||
|
||||
secret.ExtraData = make(map[string]string)
|
||||
|
||||
var passphrase string
|
||||
parsedKey, err := ssh.ParseRawPrivateKey([]byte(token))
|
||||
if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
|
||||
secret.ExtraData["encrypted"] = "true"
|
||||
parsedKey, passphrase, err = crack([]byte(token))
|
||||
if err != nil {
|
||||
secret.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if passphrase != "" {
|
||||
secret.ExtraData["cracked_encryption_passphrase"] = "true"
|
||||
}
|
||||
} else if err != nil {
|
||||
// couldn't parse key, probably invalid
|
||||
continue
|
||||
}
|
||||
|
||||
fingerprint, err := FingerprintPEMKey(parsedKey)
|
||||
fingerprint, err := FingerprintPEMKey([]byte(token))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if verify {
|
||||
verificationErrors := []string{}
|
||||
data, err := lookupFingerprint(fingerprint, s.IncludeExpired)
|
||||
if err == nil {
|
||||
secret.StructuredData = data
|
||||
if data != nil {
|
||||
secret.Verified = true
|
||||
secret.ExtraData["certificate_urls"] = strings.Join(data.CertificateURLs, ", ")
|
||||
}
|
||||
} else {
|
||||
verificationErrors = append(verificationErrors, err.Error())
|
||||
}
|
||||
|
||||
user, err := verifyGitHubUser(parsedKey)
|
||||
if err != nil && !errors.Is(err, errPermissionDenied) {
|
||||
verificationErrors = append(verificationErrors, err.Error())
|
||||
}
|
||||
if user != nil {
|
||||
secret.Verified = true
|
||||
secret.ExtraData["github_user"] = *user
|
||||
}
|
||||
|
||||
user, err = verifyGitLabUser(parsedKey)
|
||||
if err != nil && !errors.Is(err, errPermissionDenied) {
|
||||
verificationErrors = append(verificationErrors, err.Error())
|
||||
}
|
||||
if user != nil {
|
||||
secret.Verified = true
|
||||
secret.ExtraData["gitlab_user"] = *user
|
||||
}
|
||||
|
||||
if !secret.Verified && len(verificationErrors) > 0 {
|
||||
secret.VerificationError = fmt.Errorf("verification failures: %s", strings.Join(verificationErrors, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
if len(secret.ExtraData) == 0 {
|
||||
secret.ExtraData = nil
|
||||
}
|
||||
|
||||
results = append(results, secret)
|
||||
@@ -125,30 +74,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
|
||||
return results, nil
|
||||
}
|
||||
|
||||
type result struct {
|
||||
CertificateURLs []string
|
||||
GitHubUsername string
|
||||
}
|
||||
|
||||
func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*result, error) {
|
||||
func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (data *detectorspb.StructuredData, err error) {
|
||||
req, err := http.NewRequest("GET", fmt.Sprintf("https://keychecker.trufflesecurity.com/fingerprint/%s", publicKeyFingerprintInHex), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
results := DriftwoodResult{}
|
||||
err = json.NewDecoder(res.Body).Decode(&results)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return
|
||||
}
|
||||
|
||||
var data *result
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
for _, r := range results.CertificateResults {
|
||||
if _, ok := seen[r.CertificateFingerprint]; ok {
|
||||
@@ -158,13 +100,36 @@ func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*
|
||||
continue
|
||||
}
|
||||
if data == nil {
|
||||
data = &result{}
|
||||
data = &detectorspb.StructuredData{}
|
||||
}
|
||||
data.CertificateURLs = append(data.CertificateURLs, fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint))
|
||||
if data.TlsPrivateKey == nil {
|
||||
data.TlsPrivateKey = make([]*detectorspb.TlsPrivateKey, 0)
|
||||
}
|
||||
data.TlsPrivateKey = append(data.TlsPrivateKey, &detectorspb.TlsPrivateKey{
|
||||
CertificateFingerprint: r.CertificateFingerprint,
|
||||
ExpirationTimestamp: r.ExpirationTimestamp.Unix(),
|
||||
VerificationUrl: fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint),
|
||||
})
|
||||
seen[r.CertificateFingerprint] = struct{}{}
|
||||
}
|
||||
|
||||
return data, nil
|
||||
for _, r := range results.GitHubSSHResults {
|
||||
if _, ok := seen[r.Username]; ok {
|
||||
continue
|
||||
}
|
||||
if data == nil {
|
||||
data = &detectorspb.StructuredData{}
|
||||
}
|
||||
if data.GithubSshKey == nil {
|
||||
data.GithubSshKey = make([]*detectorspb.GitHubSSHKey, 0)
|
||||
}
|
||||
data.GithubSshKey = append(data.GithubSshKey, &detectorspb.GitHubSSHKey{
|
||||
User: r.Username,
|
||||
})
|
||||
seen[r.Username] = struct{}{}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type DriftwoodResult struct {
|
||||
|
||||
@@ -20,15 +20,13 @@ import (
|
||||
func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
|
||||
secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
|
||||
secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
|
||||
secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
@@ -77,8 +75,19 @@ func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PrivateKey,
|
||||
Verified: true,
|
||||
Redacted: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
|
||||
ExtraData: map[string]string{
|
||||
"certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
|
||||
StructuredData: &detectorspb.StructuredData{
|
||||
TlsPrivateKey: []*detectorspb.TlsPrivateKey{
|
||||
{
|
||||
CertificateFingerprint: "1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
|
||||
VerificationUrl: "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
|
||||
ExpirationTimestamp: 1497337731,
|
||||
},
|
||||
{
|
||||
CertificateFingerprint: "0e9de31fb2ee16465a4d5d93b227d54f870326d1",
|
||||
VerificationUrl: "https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
|
||||
ExpirationTimestamp: 1528192832,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -97,8 +106,12 @@ func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PrivateKey,
|
||||
Verified: true,
|
||||
Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
|
||||
ExtraData: map[string]string{
|
||||
"github_user": "sirdetectsalot",
|
||||
StructuredData: &detectorspb.StructuredData{
|
||||
GithubSshKey: []*detectorspb.GitHubSSHKey{
|
||||
{
|
||||
User: "thisisforgithub0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -108,8 +121,47 @@ func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
name: "found encrypted GitHub SSH private key, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(secretGitHubEncrypted),
|
||||
ctx: context.Background(),
|
||||
data: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
|
||||
b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun
|
||||
xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz
|
||||
utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu
|
||||
inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF
|
||||
OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3
|
||||
sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH
|
||||
NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5
|
||||
WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X
|
||||
lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W
|
||||
UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2
|
||||
uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE
|
||||
Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG
|
||||
SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5
|
||||
kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe
|
||||
wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi
|
||||
TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl
|
||||
nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg
|
||||
TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W
|
||||
8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR
|
||||
wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3
|
||||
akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx
|
||||
ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx
|
||||
Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B
|
||||
e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a
|
||||
s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ
|
||||
Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E
|
||||
kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8
|
||||
Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm
|
||||
fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e
|
||||
S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ
|
||||
sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5
|
||||
c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD
|
||||
/90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02
|
||||
muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx
|
||||
iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3
|
||||
f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e
|
||||
Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
|
||||
-----END OPENSSH PRIVATE KEY-----
|
||||
`),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
@@ -117,10 +169,12 @@ func TestPrivatekey_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_PrivateKey,
|
||||
Verified: true,
|
||||
Redacted: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
|
||||
ExtraData: map[string]string{
|
||||
"github_user": "sirdetectsalot",
|
||||
"encrypted": "true",
|
||||
"cracked_encryption_passphrase": "true",
|
||||
StructuredData: &detectorspb.StructuredData{
|
||||
GithubSshKey: []*detectorspb.GitHubSSHKey{
|
||||
{
|
||||
User: "thisisforgithub0",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -208,7 +262,7 @@ func Test_lookupFingerprint(t *testing.T) {
|
||||
t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
|
||||
if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.TlsPrivateKey) > 0, tt.wantFingerprints) {
|
||||
t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,125 +0,0 @@
|
||||
package privatekey
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
|
||||
var githubFingerprints = map[string]string{
|
||||
"SHA256:uNiVztksCsDhcc0u9e8BujQXVUpKZIDTMczCvj3tD2s": "RSA",
|
||||
"SHA256:br9IjFspm1vxR3iA35FWE+4VTyz1hYVLIE2t1/CeyWQ": "DSA - deprecated",
|
||||
"SHA256:p2QAMXNIC1TJYWeIOttrVc98/R1BUFWu3/LiyKgUfQM": "ECDSA",
|
||||
"SHA256:+DiY3wvvV6TuJJhbpZisF/zLDA0zPMSvHdkr4UvCOqU": "ED25519",
|
||||
}
|
||||
|
||||
// https://docs.gitlab.com/ee/user/gitlab_com/index.html#ssh-host-keys-fingerprints
|
||||
var gitlabFingerprints = map[string]string{
|
||||
"SHA256:HbW3g8zUjNSksFbqTiUWPWg2Bq1x8xdGUrliXFzSnUw": "ECDSA",
|
||||
"SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8": "ED25519",
|
||||
"SHA256:ROQFvPThGrW4RuWLoL9tq9I9zJ42fK4XywyRtbOz/EQ": "RSA",
|
||||
}
|
||||
|
||||
func firstResponseFromSSH(parsedKey any, username, hostport string) (string, error) {
|
||||
signer, err := ssh.NewSignerFromKey(parsedKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Verify the server fingerprint to ensure that there is no MITM replay attack
|
||||
config := &ssh.ClientConfig{
|
||||
Timeout: 3 * time.Second,
|
||||
User: username,
|
||||
Auth: []ssh.AuthMethod{
|
||||
ssh.PublicKeys(signer),
|
||||
},
|
||||
HostKeyCallback: func(hostname string, _ net.Addr, key ssh.PublicKey) error {
|
||||
switch hostname {
|
||||
case "github.com:22":
|
||||
fingerprint := fingerprintSSHPublicKey(key)
|
||||
if _, ok := githubFingerprints[fingerprint]; !ok {
|
||||
return fmt.Errorf("unknown host fingerprint for github.com, got %s", fingerprint)
|
||||
}
|
||||
case "gitlab.com:22":
|
||||
fingerprint := fingerprintSSHPublicKey(key)
|
||||
if _, ok := gitlabFingerprints[fingerprint]; !ok {
|
||||
return fmt.Errorf("unknown host fingerprint for gitlab.com, got %s", fingerprint)
|
||||
}
|
||||
default:
|
||||
return errors.New("unknown host in fingerprint db")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
client, err := ssh.Dial("tcp", hostport, config)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "unable to authenticate") {
|
||||
return "", errPermissionDenied
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer session.Close()
|
||||
|
||||
var output bytes.Buffer
|
||||
session.Stderr = &output
|
||||
|
||||
err = session.Shell()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_ = session.Wait()
|
||||
|
||||
return output.String(), err
|
||||
}
|
||||
|
||||
var errPermissionDenied = errors.New("permission denied")
|
||||
|
||||
func verifyGitHubUser(parsedKey any) (*string, error) {
|
||||
output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.Contains(output, "Permission denied") {
|
||||
return nil, errPermissionDenied
|
||||
}
|
||||
|
||||
if strings.Contains(output, "successfully authenticated") {
|
||||
username := strings.TrimSuffix(strings.Split(output, " ")[1], "!")
|
||||
return &username, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func verifyGitLabUser(parsedKey any) (*string, error) {
|
||||
output, err := firstResponseFromSSH(parsedKey, "git", "gitlab.com:22")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.Contains(output, "Permission denied") {
|
||||
return nil, errPermissionDenied
|
||||
}
|
||||
|
||||
if strings.Contains(output, "Welcome to GitLab") {
|
||||
split := strings.Split(output, " ")
|
||||
username := strings.TrimPrefix(strings.TrimSuffix(split[len(split)-1], "!"), "@")
|
||||
return &username, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
package privatekey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
func TestFirstResponseFromSSH(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
|
||||
|
||||
parsedKey, err := ssh.ParseRawPrivateKey([]byte(normalize(secretGitHub)))
|
||||
if err != nil {
|
||||
t.Fatalf("could not parse test secret: %s", err)
|
||||
}
|
||||
|
||||
output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
|
||||
if err != nil {
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
if len(output) == 0 {
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package sendbird
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -13,25 +12,19 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{40})\b`)
|
||||
appIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12})\b`)
|
||||
)
|
||||
|
||||
type userResp struct {
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
@@ -68,40 +61,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Api-Token", resMatch)
|
||||
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 400 { // Sendbird returns 400 for all errors
|
||||
var userResp userResp
|
||||
err := json.NewDecoder(res.Body).Decode(&userResp)
|
||||
if err != nil {
|
||||
s1.VerificationError = fmt.Errorf("error decoding json response body: %w", err)
|
||||
} else if userResp.Code != 400401 {
|
||||
// https://sendbird.com/docs/chat/platform-api/v3/error-codes
|
||||
// Sendbird always includes its own error codes with 400 responses
|
||||
// 400401 (InvalidApiToken) is the only one that indicates a bad token
|
||||
s1.VerificationError = fmt.Errorf("unexpected response code: %d", userResp.Code)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func TestSendbird_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -33,12 +33,11 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -54,8 +53,7 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -71,8 +69,7 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -82,82 +79,14 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sendbird,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sendbird,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected error code",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, `{"code":400402}`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sendbird,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but failed to decode json",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, `{`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird application ID %s with sendbird secret %s within", appId, secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sendbird,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Sendbird.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
@@ -166,12 +95,9 @@ func TestSendbird_FromChunk(t *testing.T) {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2,7 +2,6 @@ package sendbirdorganizationapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,15 +11,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{24})\b`)
|
||||
@@ -50,33 +47,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://gate.sendbird.com/api/v2/applications", nil)
|
||||
if err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
req.Header.Add("SENDBIRDORGANIZATIONAPITOKEN", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
s1.VerificationError = err
|
||||
} else {
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode != http.StatusForbidden {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
|
||||
@@ -9,8 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -20,7 +19,7 @@ import (
|
||||
func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -33,12 +32,11 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -54,8 +52,7 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -71,42 +68,7 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -122,23 +84,20 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("SendbirdOrganizationAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("SendbirdOrganizationAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -13,15 +13,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendgrid"}) + `(SG\.[\w\-_]{20,24}\.[\w\-_]{39,50})\b`)
|
||||
)
|
||||
@@ -44,7 +42,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
res := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Raw: []byte(res),
|
||||
}
|
||||
@@ -54,11 +52,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 403 will be issued if the scope is wrong but the key is correct
|
||||
baseURL := "https://api.sendgrid.com/v3/templates"
|
||||
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
// test `read_user` scope
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", baseURL, nil)
|
||||
if err != nil {
|
||||
@@ -74,18 +67,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
// 403 means good key but not the right scope
|
||||
// 401 is bad key
|
||||
if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode != http.StatusUnauthorized {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
s.Verified = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
results = append(results, s)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
@@ -9,8 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -19,7 +18,7 @@ import (
|
||||
func TestSendgrid_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
@@ -31,12 +30,11 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -52,8 +50,7 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
@@ -69,8 +66,7 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -80,54 +76,22 @@ func TestSendgrid_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sendgrid secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SendGrid,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Sendgrid.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
for i := range got {
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Sendgrid.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -2,10 +2,7 @@ package sentrytoken
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -15,20 +12,16 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sentry"}) + `\b([a-f0-9]{64})\b`)
|
||||
|
||||
errUnauthorized = fmt.Errorf("token unauthorized")
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -54,81 +47,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://sentry.io/api/0/projects/", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
isVerified, verificationErr := verifyToken(ctx, client, resMatch)
|
||||
|
||||
switch {
|
||||
case errors.Is(verificationErr, errUnauthorized):
|
||||
s1.Verified = false
|
||||
case isVerified:
|
||||
s1.Verified = true
|
||||
default:
|
||||
s1.VerificationError = verificationErr
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
// req.Header.Add("Cookie", "sc=g2ceDPcsBwPLGXugHUv0coGBi09TQYKqmtp0u4JYdIvOTFd28yVvn1yiuXgmJJa8; session=.eJwNyE0OwiAQQGGsPwtP4kqoGQa4gUdgZ2CGiaZJCUqjGxOPbnfve7_h29QpHpVSvbw61To9StvEwzre9TkVbkPcr7jN5dPbNl7X1jUt_a4vLpEHgUycQSgFZ0YBRkGxnthxwIyBAgdngcF4YxBssiMjekGv2245_wHyySMG:1m0J5S:FjpRlBVuFx577UQp6GcSGWTq0uk")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
type Response []Project
|
||||
|
||||
type Project struct {
|
||||
Organization Organization `json:"organization"`
|
||||
}
|
||||
|
||||
type Organization struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
func verifyToken(ctx context.Context, client *http.Client, token string) (bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://sentry.io/api/0/projects/", nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
var isVerified bool
|
||||
switch res.StatusCode {
|
||||
case http.StatusOK, http.StatusForbidden:
|
||||
isVerified = true
|
||||
case http.StatusUnauthorized:
|
||||
return false, errUnauthorized
|
||||
default:
|
||||
return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
|
||||
bytes, readErr := io.ReadAll(res.Body)
|
||||
if readErr != nil {
|
||||
return false, readErr
|
||||
}
|
||||
|
||||
var resp Response
|
||||
if err = json.Unmarshal(bytes, &resp); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(resp) == 0 {
|
||||
return false, fmt.Errorf("unexpected response body: %s", string(bytes))
|
||||
}
|
||||
|
||||
return isVerified, err
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_SentryToken
|
||||
}
|
||||
|
||||
@@ -9,9 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
@@ -34,12 +32,11 @@ func TestSentryToken_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -73,90 +70,6 @@ func TestSentryToken_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified but for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found and valid but unexpected api response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(500, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, good key but wrong scope",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(403, responseBody403)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, account deactivated",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, reponseAccountDeactivated)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, account deactivated",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, responseEnmpty)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sentry super secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SentryToken,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
@@ -169,50 +82,27 @@ func TestSentryToken_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("SentryToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError)
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, opts); diff != "" {
|
||||
t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("SentryToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
responseBody403 = `
|
||||
[
|
||||
{
|
||||
"organization": {
|
||||
"id": "911964",
|
||||
"slug": "wigslap",
|
||||
"status": {
|
||||
"id": "active",
|
||||
"name": "active"
|
||||
},
|
||||
"name": "wigslap"
|
||||
}
|
||||
}
|
||||
]
|
||||
`
|
||||
reponseAccountDeactivated = `{"detail": "Authentication credentials were not provided"}`
|
||||
responseEnmpty = `[]`
|
||||
)
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
|
||||
@@ -13,16 +13,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Check that the Slack scanner implements the SecretScanner interface at compile time.
|
||||
var _ detectors.Detector = Scanner{}
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
tokenPats = map[string]*regexp.Regexp{
|
||||
tokenPats = map[string]*regexp.Regexp{
|
||||
"Slack Bot Token": regexp.MustCompile(`xoxb\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
"Slack User Token": regexp.MustCompile(`xoxp\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
"Slack Workspace Access Token": regexp.MustCompile(`xoxa\-[0-9]{10,13}\-[0-9]{10,13}[a-zA-Z0-9\-]*`),
|
||||
@@ -56,54 +53,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
tokens := tokenPat.FindAllString(dataStr, -1)
|
||||
|
||||
for _, token := range tokens {
|
||||
s1 := detectors.Result{
|
||||
s := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Raw: []byte(token),
|
||||
}
|
||||
if verify {
|
||||
client := s.client
|
||||
if s.client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
client := common.SaneHttpClient()
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", verifyURL, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("Content-Type", "application/json; charset=utf-8")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
var authResponse authRes
|
||||
if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
|
||||
s1.VerificationError = fmt.Errorf("failed to decode auth response: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if authResponse.Ok {
|
||||
s1.Verified = true
|
||||
// Slack API returns 200 even if the token is invalid. We need to check the error field.
|
||||
} else if authResponse.Error == "invalid_auth" {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected error auth response %+v", authResponse.Error)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
s.Verified = authResponse.Ok
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
results = append(results, s)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
return
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
|
||||
@@ -9,15 +9,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestSlack_FromChunk(t *testing.T) {
|
||||
func TestScanner_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
@@ -26,28 +24,17 @@ func TestSlack_FromChunk(t *testing.T) {
|
||||
}
|
||||
secret := testSecrets.MustGetField("SLACK")
|
||||
secretInactive := testSecrets.MustGetField("SLACK_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
wantResults []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
data []byte
|
||||
verify bool
|
||||
wantResults []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
name: "found, verified",
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
@@ -57,12 +44,9 @@ func TestSlack_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found but unverified",
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
|
||||
verify: true,
|
||||
},
|
||||
name: "found but unverified",
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secretInactive)),
|
||||
verify: true,
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
@@ -71,62 +55,23 @@ func TestSlack_FromChunk(t *testing.T) {
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "unexpected auth response",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(200, `{"ok": false, "error": "unexpected_error"}`)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slack secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
wantResults: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Slack,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(context.Background(), tt.verify, tt.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Slack.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Scanner.FromData) error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.wantResults, ignoreOpts); diff != "" {
|
||||
t.Errorf("Slack.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.wantResults); diff != "" {
|
||||
t.Errorf("Scanner.FromData) %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package slackwebhook
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -70,10 +69,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (bytes.Equal(bodyBytes, []byte("no_text")) || bytes.Equal(bodyBytes, []byte("missing_text")))) {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 || res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
|
||||
@@ -6,14 +6,14 @@ package slackwebhook
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
@@ -57,40 +57,6 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, verified, 400 no_text",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, "no_text")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SlackWebhook,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, verified, 400 missing_text",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, "missing_text")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SlackWebhook,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
@@ -154,23 +120,6 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, account disabled",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(400, disabledAccountResponse)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_SlackWebhook,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -195,8 +144,6 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
const disabledAccountResponse = `missing_text_or_fallback_or_attachments`
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
package sourcegraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(sgp_[a-f0-9]{40})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"sgp_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Sourcegraph secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
payload := strings.NewReader("{\"query\": \"query { currentUser { username } }\"}")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://sourcegraph.com/.api/graphql", payload)
|
||||
req.Header.Add("Authorization", "token "+resMatch)
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Sourcegraph
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package sourcegraph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestSourcegraph_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("SOURCEGRAPH")
|
||||
inactiveSecret := testSecrets.MustGetField("SOURCEGRAPH_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a sourcegraph secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Sourcegraph,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Sourcegraph.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Sourcegraph.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package tailscale
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\btskey-[a-z]+-[0-9A-Za-z_]+-[0-9A-Za-z_]+\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"tskey-", "tskey-api-", "tskey-oauth-"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Tailscaleapi secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
resMatch := strings.TrimSpace(match[0])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
const u = "https://api.tailscale.com/api/v2/secret-scanning/verify"
|
||||
vals := url.Values{"key": []string{resMatch}}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, strings.NewReader(vals.Encode()))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
switch res.StatusCode {
|
||||
case http.StatusNoContent:
|
||||
s1.Verified = true
|
||||
case http.StatusUnauthorized:
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
default:
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Tailscale
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package tailscale
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestTailscaleapi_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil && false {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("TAILSCALEAPI")
|
||||
inactiveSecret := testSecrets.MustGetField("TAILSCALEAPI_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a tailscaleapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Tailscale,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Tailscaleapi.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Tailscaleapi.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package twilio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
@@ -11,15 +10,12 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
identifierPat = regexp.MustCompile(`(?i)sid.{0,20}AC[0-9a-f]{32}`) // Should we have this? Seems restrictive.
|
||||
sidPat = regexp.MustCompile(`\bAC[0-9a-f]{32}\b`)
|
||||
keyPat = regexp.MustCompile(`\b[0-9a-f]{32}\b`)
|
||||
@@ -55,11 +51,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
client = s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx, "GET", "https://verify.twilio.com/v2/Services", nil)
|
||||
if err != nil {
|
||||
@@ -74,13 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 || res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,11 +6,11 @@ package twilio
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
@@ -33,12 +33,11 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
@@ -53,7 +52,6 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Verified: true,
|
||||
Redacted: id,
|
||||
RawV2: []byte(id + secret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -71,7 +69,6 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Verified: false,
|
||||
Redacted: id,
|
||||
RawV2: []byte(id + secretInactive),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -87,62 +84,23 @@ func TestTwilio_FromChunk(t *testing.T) {
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(100 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Verified: false,
|
||||
Redacted: id,
|
||||
RawV2: []byte(id + secret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a twillio secret %s within awsId %s", secret, id)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Twilio,
|
||||
Verified: false,
|
||||
Redacted: id,
|
||||
RawV2: []byte(id + secret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Twilio.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
t.Fatal("no raw secret present")
|
||||
}
|
||||
got[i].Raw = nil
|
||||
got[i].RawV2 = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Twilio.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
|
||||
+11
-6
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
|
||||
)
|
||||
|
||||
@@ -26,13 +27,17 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - Circle CI"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, circleci.SourceType)
|
||||
|
||||
circleSource := &circleci.Source{}
|
||||
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
circleSource := circleci.Source{}
|
||||
if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &circleSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, circleSource)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraph"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
@@ -1542,8 +1540,6 @@ func DefaultDetectors() []detectors.Detector {
|
||||
huggingface.Scanner{},
|
||||
trufflehogenterprise.Scanner{},
|
||||
salesforce.Scanner{},
|
||||
sourcegraph.Scanner{},
|
||||
tailscale.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+11
-6
@@ -6,18 +6,23 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
|
||||
)
|
||||
|
||||
// ScanDocker scans a given docker connection.
|
||||
func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
|
||||
sourceName := "trufflehog - docker"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, docker.SourceType)
|
||||
|
||||
dockerSource := &docker.Source{}
|
||||
if err := dockerSource.Init(ctx, sourceName, jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
dockerSource := docker.Source{}
|
||||
if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dockerSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := e.sourceManager.Run(ctx, sourceName, dockerSource)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+4
-11
@@ -239,7 +239,7 @@ func (m *Metrics) getScanDuration() time.Duration {
|
||||
func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
logger := context.Background().Logger()
|
||||
avgTime := map[string][]time.Duration{}
|
||||
e.metrics.detectorAvgTime.Range(func(k, v any) bool {
|
||||
e.metrics.detectorAvgTime.Range(func(k, v interface{}) bool {
|
||||
key, ok := k.(string)
|
||||
if !ok {
|
||||
logger.Info("expected detectorAvgTime key to be a string")
|
||||
@@ -565,18 +565,11 @@ func (e *Engine) notifyResults(ctx context.Context) {
|
||||
}
|
||||
atomic.AddUint32(&e.numFoundResults, 1)
|
||||
|
||||
// Dedupe results by comparing the detector type, raw result, and source metadata.
|
||||
// We want to avoid duplicate results with different decoder types, but we also
|
||||
// want to include duplicate results with the same decoder type.
|
||||
// Duplicate results with the same decoder type SHOULD have their own entry in the
|
||||
// results list, this would happen if the same secret is found multiple times.
|
||||
key := fmt.Sprintf("%s%s%s%+v", r.DetectorType.String(), r.Raw, r.RawV2, r.SourceMetadata)
|
||||
if val, ok := e.dedupeCache.Get(key); ok {
|
||||
if res, ok := val.(detectorspb.DecoderType); ok && res != r.DecoderType {
|
||||
continue
|
||||
}
|
||||
if _, ok := e.dedupeCache.Get(key); ok {
|
||||
continue
|
||||
}
|
||||
e.dedupeCache.Add(key, r.DecoderType)
|
||||
e.dedupeCache.Add(key, struct{}{})
|
||||
|
||||
if r.Verified {
|
||||
atomic.AddUint64(&e.metrics.VerifiedSecretsFound, 1)
|
||||
|
||||
@@ -2,13 +2,8 @@ package engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
@@ -176,32 +171,3 @@ func BenchmarkSupportsLineNumbersLoop(b *testing.B) {
|
||||
_ = SupportsLineNumbers(sourceType)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEngine_DuplicatSecrets is a test that detects ALL duplicate secrets with the same decoder.
|
||||
func TestEngine_DuplicatSecrets(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
absPath, err := filepath.Abs("./testdata")
|
||||
assert.Nil(t, err)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(true, DefaultDetectors()...),
|
||||
WithPrinter(new(discardPrinter)),
|
||||
)
|
||||
assert.Nil(t, err)
|
||||
|
||||
cfg := sources.FilesystemConfig{Paths: []string{absPath}}
|
||||
if err := e.ScanFileSystem(ctx, cfg); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Wait for all the chunks to be processed.
|
||||
assert.Nil(t, e.Finish(ctx))
|
||||
want := uint64(5)
|
||||
assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
|
||||
}
|
||||
|
||||
@@ -24,14 +24,18 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
|
||||
return err
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - filesystem"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, filesystem.SourceType)
|
||||
|
||||
fileSystemSource := &filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
fileSystemSource := filesystem.Source{}
|
||||
fileSystemSource.WithFilter(c.Filter)
|
||||
if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &fileSystemSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+10
-6
@@ -44,14 +44,18 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
|
||||
return fmt.Errorf("failed to marshal GCS connection: %w", err)
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - gcs"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gcs.SourceType)
|
||||
|
||||
gcsSource := &gcs.Source{}
|
||||
if err := gcsSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gcsSource := gcs.Source{}
|
||||
if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &gcsSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, gcsSource)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+14
-11
@@ -51,18 +51,21 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - git"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, git.SourceType)
|
||||
|
||||
gitSource := &git.Source{}
|
||||
if err := gitSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitSource := git.Source{}
|
||||
if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitSource.WithScanOptions(scanOptions)
|
||||
// Don't try to clean up the provided directory. That's handled by the
|
||||
// caller of ScanGit.
|
||||
gitSource.WithPreserveTempDirs(true)
|
||||
return &gitSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gitSource.WithScanOptions(scanOptions)
|
||||
// Don't try to clean up the provided directory. That's handled by the
|
||||
// caller of ScanGit.
|
||||
gitSource.WithPreserveTempDirs(true)
|
||||
|
||||
_, err := e.sourceManager.Run(ctx, sourceName, gitSource)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+11
-7
@@ -47,14 +47,18 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
|
||||
}
|
||||
scanOptions := git.NewScanOptions(opts...)
|
||||
|
||||
sourceName := "trufflehog - github"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, github.SourceType)
|
||||
|
||||
githubSource := &github.Source{}
|
||||
if err := githubSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
githubSource := github.Source{}
|
||||
if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
githubSource.WithScanOptions(scanOptions)
|
||||
return &githubSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
githubSource.WithScanOptions(scanOptions)
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, githubSource)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+11
-7
@@ -50,14 +50,18 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
|
||||
return err
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - gitlab"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gitlab.SourceType)
|
||||
|
||||
gitlabSource := &gitlab.Source{}
|
||||
if err := gitlabSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
gitlabSource := gitlab.Source{}
|
||||
if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
return &gitlabSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gitlabSource.WithScanOptions(scanOptions)
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, gitlabSource)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+10
-6
@@ -58,13 +58,17 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
|
||||
return err
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - s3"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, s3.SourceType)
|
||||
|
||||
s3Source := &s3.Source{}
|
||||
if err := s3Source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
s3Source := s3.Source{}
|
||||
if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3Source, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, s3Source)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
+11
-7
@@ -41,14 +41,18 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
|
||||
sourceName := "trufflehog - syslog"
|
||||
sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, syslog.SourceType)
|
||||
syslogSource := &syslog.Source{}
|
||||
if err := syslogSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),
|
||||
func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
|
||||
syslogSource := syslog.Source{}
|
||||
if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
syslogSource.InjectConnection(connection)
|
||||
return &syslogSource, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
syslogSource.InjectConnection(connection)
|
||||
|
||||
_, err = e.sourceManager.Run(ctx, sourceName, syslogSource)
|
||||
_, err = e.sourceManager.ScheduleRun(ctx, handle)
|
||||
return err
|
||||
}
|
||||
|
||||
Vendored
-6
@@ -1,6 +0,0 @@
|
||||
1 aws AKIAWARWQKZNHMZBLY4I
|
||||
2 secret s6NbZeygUrUdM95K683Lb6IsILWXOJlJ8ZVd1Kw0
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
sentry 27ac84f4bcdb4fca9701f4d6f6f58cd7d96b69c9d9754d40800645a51d668f90
|
||||
+10
-11
@@ -119,18 +119,7 @@ func GenerateLink(repo, commit, file string, line int64) string {
|
||||
case providerBitbucket:
|
||||
return repo[:len(repo)-4] + "/commits/" + commit
|
||||
|
||||
case providerAzure:
|
||||
baseLink := repo + "/commit/" + commit + "/" + file
|
||||
if line > 0 {
|
||||
baseLink += "?line=" + strconv.FormatInt(line, 10)
|
||||
}
|
||||
return baseLink
|
||||
|
||||
case providerGithub, providerGitlab:
|
||||
// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
|
||||
// So do the same thing.
|
||||
fallthrough
|
||||
default:
|
||||
var baseLink string
|
||||
if file == "" {
|
||||
baseLink = repo[:len(repo)-4] + "/commit/" + commit
|
||||
@@ -141,5 +130,15 @@ func GenerateLink(repo, commit, file string, line int64) string {
|
||||
}
|
||||
}
|
||||
return baseLink
|
||||
|
||||
case providerAzure:
|
||||
baseLink := repo + "?path=" + file + "&version=GB" + commit
|
||||
if line > 0 {
|
||||
baseLink += "&line=" + strconv.FormatInt(line, 10)
|
||||
}
|
||||
return baseLink
|
||||
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,7 +148,7 @@ func TestGenerateLink(t *testing.T) {
|
||||
commit: "abcdef",
|
||||
file: "main.go",
|
||||
},
|
||||
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go",
|
||||
want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef",
|
||||
},
|
||||
{
|
||||
name: "Azure link gen with line",
|
||||
@@ -158,25 +158,7 @@ func TestGenerateLink(t *testing.T) {
|
||||
file: "main.go",
|
||||
line: int64(20),
|
||||
},
|
||||
want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
|
||||
},
|
||||
{
|
||||
name: "Unknown provider on-prem instance",
|
||||
args: args{
|
||||
repo: "https://onprem.customdomain.com/org/repo.git",
|
||||
commit: "xyz123",
|
||||
file: "main.go",
|
||||
line: int64(30),
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
|
||||
},
|
||||
{
|
||||
name: "Unknown provider on-prem instance - no file",
|
||||
args: args{
|
||||
repo: "https://onprem.customdomain.com/org/repo.git",
|
||||
commit: "xyz123",
|
||||
},
|
||||
want: "https://onprem.customdomain.com/org/repo/commit/xyz123",
|
||||
want: "https://dev.azure.com/org/project/_git/repo?path=main.go&version=GBabcdef&line=20",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
|
||||
+34
-57
@@ -17,15 +17,12 @@ import (
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type ctxKey int
|
||||
|
||||
const (
|
||||
depthKey ctxKey = iota
|
||||
|
||||
errMaxArchiveDepthReached = "max archive depth reached"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -78,76 +75,57 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
|
||||
if errors.Is(err, archiver.ErrNoMatch) {
|
||||
return
|
||||
}
|
||||
logger.Error(err, "error unarchiving chunk.")
|
||||
logger.V(2).Info("Error unarchiving chunk.")
|
||||
}
|
||||
}()
|
||||
return archiveChan
|
||||
}
|
||||
|
||||
type decompressorInfo struct {
|
||||
depth int
|
||||
reader io.Reader
|
||||
archiveChan chan []byte
|
||||
archiver archiver.Decompressor
|
||||
}
|
||||
|
||||
// openArchive takes a reader and extracts the contents up to the maximum depth.
|
||||
func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
|
||||
if depth >= maxDepth {
|
||||
return fmt.Errorf(errMaxArchiveDepthReached)
|
||||
return fmt.Errorf("max archive depth reached")
|
||||
}
|
||||
|
||||
format, arReader, err := archiver.Identify("", reader)
|
||||
if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
|
||||
return a.handleNonArchiveContent(ctx, arReader, archiveChan)
|
||||
}
|
||||
|
||||
format, reader, err := archiver.Identify("", reader)
|
||||
if err != nil {
|
||||
if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
|
||||
chunkSize := 10 * 1024
|
||||
for {
|
||||
chunk := make([]byte, chunkSize)
|
||||
n, _ := reader.Read(chunk)
|
||||
archiveChan <- chunk
|
||||
if n < chunkSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
switch archive := format.(type) {
|
||||
case archiver.Decompressor:
|
||||
info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}
|
||||
return a.handleDecompressor(ctx, info)
|
||||
case archiver.Extractor:
|
||||
return archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
|
||||
default:
|
||||
return fmt.Errorf("unknown archive type: %s", format.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Archive) handleNonArchiveContent(ctx context.Context, reader io.Reader, archiveChan chan []byte) error {
|
||||
aCtx := logContext.AddLogger(ctx)
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(aCtx, reader)
|
||||
for data := range chunkResChan {
|
||||
if err := data.Error(); err != nil {
|
||||
aCtx.Logger().Error(err, "error reading chunk")
|
||||
continue
|
||||
}
|
||||
if err := common.CancellableWrite(ctx, archiveChan, data.Bytes()); err != nil {
|
||||
compReader, err := archive.OpenReader(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes, err := a.ReadToMax(ctx, compReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newReader := bytes.NewReader(fileBytes)
|
||||
return a.openArchive(ctx, depth+1, newReader, archiveChan)
|
||||
case archiver.Extractor:
|
||||
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *Archive) handleDecompressor(ctx context.Context, info decompressorInfo) error {
|
||||
compReader, err := info.archiver.OpenReader(info.reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes, err := a.ReadToMax(ctx, compReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newReader := bytes.NewReader(fileBytes)
|
||||
return a.openArchive(ctx, info.depth+1, newReader, info.archiveChan)
|
||||
return fmt.Errorf("Unknown archive type: %s", format.Name())
|
||||
}
|
||||
|
||||
// IsFiletype returns true if the provided reader is an archive.
|
||||
func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bool) {
|
||||
func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
|
||||
format, readerB, err := archiver.Identify("", reader)
|
||||
if err != nil {
|
||||
return readerB, false
|
||||
@@ -157,9 +135,8 @@ func (a *Archive) IsFiletype(_ context.Context, reader io.Reader) (io.Reader, bo
|
||||
return readerB, true
|
||||
case archiver.Decompressor:
|
||||
return readerB, true
|
||||
default:
|
||||
return readerB, false
|
||||
}
|
||||
return readerB, false
|
||||
}
|
||||
|
||||
// extractorHandler is applied to each file in an archiver.Extractor file.
|
||||
@@ -203,7 +180,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
|
||||
if e, ok := r.(error); ok {
|
||||
err = e
|
||||
} else {
|
||||
err = fmt.Errorf("panic occurred: %v", r)
|
||||
err = fmt.Errorf("Panic occurred: %v", r)
|
||||
}
|
||||
logger.Error(err, "Panic occurred when reading archive")
|
||||
}
|
||||
@@ -306,7 +283,7 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
|
||||
// The caller is responsible for closing the returned reader.
|
||||
func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) {
|
||||
if a.currentDepth >= maxDepth {
|
||||
return nil, fmt.Errorf(errMaxArchiveDepthReached)
|
||||
return nil, fmt.Errorf("max archive depth reached")
|
||||
}
|
||||
|
||||
tmpEnv, err := a.createTempEnv(ctx, file)
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -237,37 +236,3 @@ func TestExtractRPMContent(t *testing.T) {
|
||||
expectedLength := 1822720
|
||||
assert.Equal(t, expectedLength, len(string(content)))
|
||||
}
|
||||
|
||||
func TestOpenInvalidArchive(t *testing.T) {
|
||||
reader := strings.NewReader("invalid archive")
|
||||
|
||||
ctx := logContext.AddLogger(context.Background())
|
||||
a := &Archive{}
|
||||
|
||||
archiveChan := make(chan []byte)
|
||||
|
||||
err := a.openArchive(ctx, 0, reader, archiveChan)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestNestedDirArchive(t *testing.T) {
|
||||
file, err := os.Open("testdata/dir-archive.zip")
|
||||
assert.Nil(t, err)
|
||||
defer file.Close()
|
||||
|
||||
ctx, cancel := logContext.WithTimeout(logContext.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
sourceChan := make(chan *sources.Chunk, 1)
|
||||
|
||||
go func() {
|
||||
defer close(sourceChan)
|
||||
HandleFile(ctx, file, &sources.Chunk{}, sourceChan)
|
||||
}()
|
||||
|
||||
count := 0
|
||||
want := 4
|
||||
for range sourceChan {
|
||||
count++
|
||||
}
|
||||
assert.Equal(t, want, count)
|
||||
}
|
||||
|
||||
+27
-29
@@ -47,44 +47,42 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
|
||||
// an io.MultiReader, which is used by the SpecializedHandler.
|
||||
reReader, err := diskbufferreader.New(file)
|
||||
if err != nil {
|
||||
aCtx.Logger().Error(err, "error creating reusable reader")
|
||||
aCtx.Logger().Error(err, "error creating re-reader reader")
|
||||
return false
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
if success := processHandler(aCtx, h, reReader, chunkSkel, chunksChan); success {
|
||||
return true
|
||||
// Check if the handler implements SpecializedHandler and process accordingly.
|
||||
if specialHandler, ok := h.(SpecializedHandler); ok {
|
||||
file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
|
||||
if isSpecial {
|
||||
return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
aCtx.Logger().Error(err, "error handling file")
|
||||
}
|
||||
}
|
||||
|
||||
if err := reReader.Reset(); err != nil {
|
||||
aCtx.Logger().Error(err, "error resetting re-reader")
|
||||
return false
|
||||
}
|
||||
if _, isType := h.IsFiletype(aCtx, reReader); !isType {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := reReader.Reset(); err != nil {
|
||||
aCtx.Logger().Error(err, "error resetting re-reader")
|
||||
return false
|
||||
}
|
||||
reReader.Stop()
|
||||
return handleChunks(aCtx, h.FromFile(ctx, reReader), chunkSkel, chunksChan)
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func processHandler(ctx logContext.Context, h Handler, reReader *diskbufferreader.DiskBufferReader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
defer reReader.Close()
|
||||
defer reReader.Stop()
|
||||
|
||||
if specialHandler, ok := h.(SpecializedHandler); ok {
|
||||
file, isSpecial, err := specialHandler.HandleSpecialized(ctx, reReader)
|
||||
if isSpecial {
|
||||
return handleChunks(ctx, h.FromFile(ctx, file), chunkSkel, chunksChan)
|
||||
}
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "error handling file")
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := reReader.Seek(0, io.SeekStart); err != nil {
|
||||
ctx.Logger().Error(err, "error seeking to start of file")
|
||||
return false
|
||||
}
|
||||
|
||||
if _, isType := h.IsFiletype(ctx, reReader); !isType {
|
||||
return false
|
||||
}
|
||||
|
||||
return handleChunks(ctx, h.FromFile(ctx, reReader), chunkSkel, chunksChan)
|
||||
}
|
||||
|
||||
func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
|
||||
for {
|
||||
select {
|
||||
|
||||
BIN
Binary file not shown.
+1
-2
@@ -10,7 +10,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
// JSONPrinter is a printer that prints results in JSON format.
|
||||
@@ -21,7 +20,7 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
// SourceMetadata contains source-specific contextual information.
|
||||
SourceMetadata *source_metadatapb.MetaData
|
||||
// SourceID is the ID of the source that the API uses to map secrets to specific sources.
|
||||
SourceID sources.SourceID
|
||||
SourceID int64
|
||||
// SourceType is the type of Source.
|
||||
SourceType sourcespb.SourceType
|
||||
// SourceName is the name of the Source.
|
||||
|
||||
+2
-2
@@ -76,7 +76,7 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
}
|
||||
}
|
||||
|
||||
aggregateData := make(map[string]any)
|
||||
aggregateData := make(map[string]interface{})
|
||||
var aggregateDataKeys []string
|
||||
|
||||
for _, data := range meta {
|
||||
@@ -93,7 +93,7 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
|
||||
return nil
|
||||
}
|
||||
|
||||
func structToMap(obj any) (m map[string]map[string]any, err error) {
|
||||
func structToMap(obj interface{}) (m map[string]map[string]interface{}, err error) {
|
||||
data, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -999,8 +999,6 @@ const (
|
||||
DetectorType_Salesforce DetectorType = 925
|
||||
DetectorType_HuggingFace DetectorType = 926
|
||||
DetectorType_Snowflake DetectorType = 927
|
||||
DetectorType_Sourcegraph DetectorType = 928
|
||||
DetectorType_Tailscale DetectorType = 929
|
||||
)
|
||||
|
||||
// Enum value maps for DetectorType.
|
||||
@@ -1930,8 +1928,6 @@ var (
|
||||
925: "Salesforce",
|
||||
926: "HuggingFace",
|
||||
927: "Snowflake",
|
||||
928: "Sourcegraph",
|
||||
929: "Tailscale",
|
||||
}
|
||||
DetectorType_value = map[string]int32{
|
||||
"Alibaba": 0,
|
||||
@@ -2858,8 +2854,6 @@ var (
|
||||
"Salesforce": 925,
|
||||
"HuggingFace": 926,
|
||||
"Snowflake": 927,
|
||||
"Sourcegraph": 928,
|
||||
"Tailscale": 929,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3238,7 +3232,7 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
|
||||
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
|
||||
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xae, 0x74, 0x0a, 0x0c, 0x44,
|
||||
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0x8c, 0x74, 0x0a, 0x0c, 0x44,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
|
||||
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
|
||||
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
|
||||
@@ -4167,14 +4161,11 @@ var file_detectors_proto_rawDesc = []byte{
|
||||
0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x66,
|
||||
0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69,
|
||||
0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x6e, 0x6f,
|
||||
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x53, 0x6f, 0x75,
|
||||
0x72, 0x63, 0x65, 0x67, 0x72, 0x61, 0x70, 0x68, 0x10, 0xa0, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x54,
|
||||
0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x10, 0xa1, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,
|
||||
0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,
|
||||
0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,
|
||||
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x33,
|
||||
0x77, 0x66, 0x6c, 0x61, 0x6b, 0x65, 0x10, 0x9f, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74,
|
||||
0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73,
|
||||
0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68,
|
||||
0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74,
|
||||
0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -19,17 +19,13 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
|
||||
|
||||
baseURL = "https://circleci.com/api/v1.1/"
|
||||
)
|
||||
const baseURL = "https://circleci.com/api/v1.1/"
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
token string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
jobPool *errgroup.Group
|
||||
sources.Progress
|
||||
@@ -44,19 +40,19 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized CircleCI source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
@@ -79,7 +75,7 @@ func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourc
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
projects, err := s.projects(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting projects: %w", err)
|
||||
|
||||
+144
-227
@@ -2,6 +2,7 @@ package docker
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -13,30 +14,27 @@ import (
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
"github.com/google/go-containerregistry/pkg/v1/tarball"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const SourceType = sourcespb.SourceType_SOURCE_TYPE_DOCKER
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
verify bool
|
||||
concurrency int
|
||||
conn sourcespb.Docker
|
||||
name string
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
conn sourcespb.Docker
|
||||
sources.Progress
|
||||
sources.CommonSourceUnitUnmarshaller
|
||||
}
|
||||
|
||||
var FilesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
@@ -44,24 +42,23 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_DOCKER
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init initializes the source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
s.verify = verify
|
||||
s.concurrency = concurrency
|
||||
|
||||
// Reset metrics for this source at initialization time.
|
||||
dockerImagesScanned.WithLabelValues(s.name).Set(0)
|
||||
@@ -74,209 +71,166 @@ func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourc
|
||||
return nil
|
||||
}
|
||||
|
||||
type imageInfo struct {
|
||||
image v1.Image
|
||||
base string
|
||||
tag string
|
||||
}
|
||||
|
||||
type layerInfo struct {
|
||||
digest v1.Hash
|
||||
base string
|
||||
tag string
|
||||
}
|
||||
|
||||
// Chunks emits data over a channel that is decoded and scanned for secrets.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)
|
||||
|
||||
workers := new(errgroup.Group)
|
||||
workers.SetLimit(s.concurrency)
|
||||
|
||||
scanErrs := sources.NewScanErrors()
|
||||
for _, image := range s.conn.GetImages() {
|
||||
image := image
|
||||
workers.Go(func() error {
|
||||
if common.IsDone(ctx) {
|
||||
return nil
|
||||
}
|
||||
|
||||
imgInfo, err := s.processImage(ctx, image)
|
||||
if err != nil {
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
|
||||
ctx.Logger().V(2).Info("scanning image")
|
||||
|
||||
layers, err := imgInfo.image.Layers()
|
||||
if err != nil {
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, layer := range layers {
|
||||
if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
|
||||
scanErrs.Add(err)
|
||||
return nil
|
||||
}
|
||||
dockerLayersScanned.WithLabelValues(s.name).Inc()
|
||||
}
|
||||
|
||||
dockerImagesScanned.WithLabelValues(s.name).Inc()
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
_ = workers.Wait()
|
||||
if scanErrs.Count() > 0 {
|
||||
ctx.Logger().V(2).Info("scan errors", "errors", scanErrs.String())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// processImage processes an individual image and prepares it for further processing.
|
||||
func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
|
||||
var (
|
||||
imgInfo imageInfo
|
||||
hasDigest bool
|
||||
imageName name.Reference
|
||||
)
|
||||
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
remoteOpts, err := s.remoteOpts()
|
||||
if err != nil {
|
||||
return imgInfo, err
|
||||
return err
|
||||
}
|
||||
|
||||
const filePrefix = "file://"
|
||||
if strings.HasPrefix(image, filePrefix) {
|
||||
image = strings.TrimPrefix(image, filePrefix)
|
||||
imgInfo.base = image
|
||||
imgInfo.image, err = tarball.ImageFromPath(image, nil)
|
||||
if err != nil {
|
||||
return imgInfo, err
|
||||
}
|
||||
} else {
|
||||
imgInfo.base, imgInfo.tag, hasDigest = baseAndTagFromImage(image)
|
||||
logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
|
||||
|
||||
if hasDigest {
|
||||
imageName, err = name.NewDigest(image)
|
||||
for _, image := range s.conn.GetImages() {
|
||||
var img v1.Image
|
||||
var err error
|
||||
var base, tag string
|
||||
var hasDigest bool
|
||||
var imageName name.Reference
|
||||
|
||||
if strings.HasPrefix(image, "file://") {
|
||||
image = strings.TrimPrefix(image, "file://")
|
||||
base = image
|
||||
img, err = tarball.ImageFromPath(image, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
imageName, err = name.NewTag(image)
|
||||
}
|
||||
if err != nil {
|
||||
return imgInfo, err
|
||||
base, tag, hasDigest = baseAndTagFromImage(image)
|
||||
|
||||
if hasDigest {
|
||||
imageName, err = name.NewDigest(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
imageName, err = name.NewTag(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
img, err = remote.Image(imageName, remoteOpts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
imgInfo.image, err = remote.Image(imageName, remoteOpts...)
|
||||
if err != nil {
|
||||
return imgInfo, err
|
||||
}
|
||||
}
|
||||
logger = logger.WithValues("image", base, "tag", tag)
|
||||
logger.V(2).Info("scanning image")
|
||||
|
||||
ctx.Logger().WithValues("image", imgInfo.base, "tag", imgInfo.tag).V(2).Info("scanning image")
|
||||
|
||||
return imgInfo, nil
|
||||
}
|
||||
|
||||
// processLayer processes an individual layer of an image.
|
||||
func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
|
||||
layerInfo := layerInfo{
|
||||
base: imgInfo.base,
|
||||
tag: imgInfo.tag,
|
||||
}
|
||||
|
||||
var err error
|
||||
layerInfo.digest, err = layer.Digest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx.Logger().WithValues("layer", layerInfo.digest.String()).V(2).Info("scanning layer")
|
||||
|
||||
rc, err := layer.Compressed()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
gzipReader, err := gzip.NewReader(rc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gzipReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
layers, err := img.Layers()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info := chunkProcessingInfo{size: header.Size, name: header.Name, reader: tarReader, layer: layerInfo}
|
||||
if err := s.processChunk(ctx, info, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, layer := range layers {
|
||||
|
||||
return nil
|
||||
}
|
||||
digest, err := layer.Digest()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
type chunkProcessingInfo struct {
|
||||
size int64
|
||||
name string
|
||||
reader io.Reader
|
||||
layer layerInfo
|
||||
}
|
||||
logger = logger.WithValues("layer", digest.String())
|
||||
logger.V(2).Info("scanning layer")
|
||||
|
||||
// processChunk processes an individual chunk of a layer.
|
||||
func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chunksChan chan *sources.Chunk) error {
|
||||
const filesizeLimitBytes int64 = 50 * 1024 * 1024 // 50MB
|
||||
if info.size > filesizeLimitBytes {
|
||||
ctx.Logger().V(4).Info("skipping large file", "file", info.name, "size", info.size)
|
||||
return nil
|
||||
}
|
||||
rc, err := layer.Compressed()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunkReader := sources.NewChunkReader()
|
||||
chunkResChan := chunkReader(ctx, info.reader)
|
||||
defer rc.Close()
|
||||
|
||||
for data := range chunkResChan {
|
||||
if err := data.Error(); err != nil {
|
||||
ctx.Logger().Error(err, "error reading chunk.")
|
||||
continue
|
||||
}
|
||||
gzipReader, err := gzip.NewReader(rc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
Docker: &source_metadatapb.Docker{
|
||||
File: "/" + info.name,
|
||||
Image: info.layer.base,
|
||||
Tag: info.layer.tag,
|
||||
Layer: info.layer.digest.String(),
|
||||
defer gzipReader.Close()
|
||||
|
||||
tarReader := tar.NewReader(gzipReader)
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break // End of archive
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Skip files larger than FilesizeLimitBytes
|
||||
if header.Size > FilesizeLimitBytes {
|
||||
continue
|
||||
}
|
||||
|
||||
file := bytes.NewBuffer(make([]byte, 0, header.Size))
|
||||
|
||||
_, err = io.Copy(file, tarReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chunk := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
JobID: s.JobID(),
|
||||
Data: file.Bytes(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Docker{
|
||||
Docker: &source_metadatapb.Docker{
|
||||
File: "/" + header.Name, // Prepend /, because the file is relative to the root of the image
|
||||
Image: base,
|
||||
Tag: tag,
|
||||
Layer: digest.String(),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
chunk.Data = data.Bytes()
|
||||
Verify: s.verify,
|
||||
}
|
||||
|
||||
if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {
|
||||
return err
|
||||
select {
|
||||
case chunksChan <- chunk:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
dockerLayersScanned.WithLabelValues(s.name).Inc()
|
||||
}
|
||||
|
||||
dockerImagesScanned.WithLabelValues(s.name).Inc()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
|
||||
regRepoDelimiter := "/"
|
||||
tagDelim := ":"
|
||||
digestDelim := "@"
|
||||
|
||||
// Split on digest first, if present.
|
||||
parts := strings.SplitN(image, digestDelim, 2)
|
||||
if len(parts) > 1 {
|
||||
base = parts[0]
|
||||
tag = parts[1]
|
||||
hasDigest = true
|
||||
return
|
||||
}
|
||||
parts = strings.Split(image, tagDelim)
|
||||
// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
|
||||
if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) {
|
||||
base = strings.Join(parts[:len(parts)-1], tagDelim)
|
||||
tag = parts[len(parts)-1]
|
||||
} else {
|
||||
base = image
|
||||
tag = "latest"
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Source) remoteOpts() ([]remote.Option, error) {
|
||||
switch s.conn.GetCredential().(type) {
|
||||
case *sourcespb.Docker_Unauthenticated:
|
||||
@@ -302,40 +256,3 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
|
||||
return nil, fmt.Errorf("unknown credential type: %T", s.conn.Credential)
|
||||
}
|
||||
}
|
||||
|
||||
func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
|
||||
if base, tag, hasDigest = extractDigest(image); hasDigest {
|
||||
return base, tag, true
|
||||
}
|
||||
|
||||
base, tag = extractTagOrUseDefault(image)
|
||||
return base, tag, false
|
||||
}
|
||||
|
||||
// extractDigest tries to split the image string on the digest delimiter.
|
||||
// If successful, it means the image has a digest.
|
||||
func extractDigest(image string) (base, tag string, hasDigest bool) {
|
||||
const digestDelim = "@"
|
||||
|
||||
if parts := strings.SplitN(image, digestDelim, 2); len(parts) > 1 {
|
||||
return parts[0], parts[1], true
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// extractTagOrUseDefault extracts the tag from the image string.
|
||||
// If no tag is found, it defaults to "latest".
|
||||
func extractTagOrUseDefault(image string) (base, tag string) {
|
||||
const (
|
||||
tagDelim = ":"
|
||||
regRepoDelim = "/"
|
||||
)
|
||||
|
||||
parts := strings.Split(image, tagDelim)
|
||||
|
||||
// Check if the last part is not a hostname with a port (for weak validation)
|
||||
if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelim) {
|
||||
return strings.Join(parts[:len(parts)-1], tagDelim), parts[len(parts)-1]
|
||||
}
|
||||
return image, "latest"
|
||||
}
|
||||
|
||||
@@ -22,12 +22,10 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const SourceType = sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
paths []string
|
||||
log logr.Logger
|
||||
@@ -45,19 +43,19 @@ var _ sources.SourceUnitChunker = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized Filesystem source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, _ int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
|
||||
s.log = aCtx.Logger()
|
||||
|
||||
s.name = name
|
||||
@@ -79,7 +77,7 @@ func (s *Source) WithFilter(filter *common.Filter) {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
for i, path := range s.paths {
|
||||
logger := ctx.Logger().WithValues("path", path)
|
||||
if common.IsDone(ctx) {
|
||||
|
||||
+8
-12
@@ -28,11 +28,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_GCS
|
||||
|
||||
defaultCachePersistIncrement = 2500
|
||||
)
|
||||
const defaultCachePersistIncrement = 2500
|
||||
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
@@ -41,16 +37,16 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_GCS
|
||||
}
|
||||
|
||||
// SourceID number for GCS Source.
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
// JobID number for GCS Source.
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
@@ -62,8 +58,8 @@ type objectManager interface {
|
||||
// Source represents a GCS source.
|
||||
type Source struct {
|
||||
name string
|
||||
jobId sources.JobID
|
||||
sourceId sources.SourceID
|
||||
jobId int64
|
||||
sourceId int64
|
||||
concurrency int
|
||||
verify bool
|
||||
|
||||
@@ -111,7 +107,7 @@ func (c *persistableCache) shouldPersist() (bool, string) {
|
||||
}
|
||||
|
||||
// Init returns an initialized GCS source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, id sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.log = aCtx.Logger()
|
||||
|
||||
s.name = name
|
||||
@@ -252,7 +248,7 @@ func (s *Source) enumerate(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
persistableCache := s.setupCache(ctx)
|
||||
|
||||
objectCh, err := s.gcsManager.ListObjects(ctx)
|
||||
|
||||
+10
-38
@@ -34,12 +34,10 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const SourceType = sourcespb.SourceType_SOURCE_TYPE_GIT
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
git *Git
|
||||
sources.Progress
|
||||
@@ -53,8 +51,8 @@ type Source struct {
|
||||
type Git struct {
|
||||
sourceType sourcespb.SourceType
|
||||
sourceName string
|
||||
sourceID sources.SourceID
|
||||
jobID sources.JobID
|
||||
sourceID int64
|
||||
jobID int64
|
||||
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
|
||||
verify bool
|
||||
metrics metrics
|
||||
@@ -65,7 +63,7 @@ type metrics struct {
|
||||
commitsScanned uint64
|
||||
}
|
||||
|
||||
func NewGit(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
|
||||
func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
|
||||
sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
|
||||
) *Git {
|
||||
return &Git{
|
||||
@@ -86,14 +84,14 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_GIT
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
@@ -111,7 +109,7 @@ func (s *Source) WithPreserveTempDirs(preserve bool) {
|
||||
}
|
||||
|
||||
// Init returns an initialized GitHub source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
@@ -155,7 +153,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, so
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
if err := s.scanRepos(ctx, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -372,32 +370,6 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
|
||||
return clonePath, repo, nil
|
||||
}
|
||||
|
||||
// PingRepoUsingToken executes git ls-remote on a repo and returns any error that occurs. It can be used to validate
|
||||
// that a repo actually exists and is reachable.
|
||||
//
|
||||
// Pinging using other authentication methods is only unimplemented because there's been no pressing need for it yet.
|
||||
func PingRepoUsingToken(ctx context.Context, token, gitUrl, user string) error {
|
||||
if err := GitCmdCheck(); err != nil {
|
||||
return err
|
||||
}
|
||||
lsUrl, err := GitURLParse(gitUrl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if lsUrl.User == nil {
|
||||
lsUrl.User = url.UserPassword(user, token)
|
||||
}
|
||||
|
||||
// We don't actually care about any refs on the remote, we just care whether can can list them at all. So we query
|
||||
// only for a ref that we know won't exist to minimize the search time on the remote. (By default, ls-remote exits
|
||||
// with 0 even if it doesn't find any matching refs.)
|
||||
fakeRef := "TRUFFLEHOG_CHECK_GIT_REMOTE_URL_REACHABILITY"
|
||||
gitArgs := []string{"ls-remote", lsUrl.String(), "--quiet", fakeRef}
|
||||
cmd := exec.Command("git", gitArgs...)
|
||||
_, err = cmd.CombinedOutput()
|
||||
return err
|
||||
}
|
||||
|
||||
// CloneRepoUsingToken clones a repo using a provided token.
|
||||
func CloneRepoUsingToken(ctx context.Context, token, gitUrl, user string, args ...string) (string, *git.Repository, error) {
|
||||
userInfo := url.UserPassword(user, token)
|
||||
|
||||
@@ -39,8 +39,6 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB
|
||||
|
||||
unauthGithubOrgRateLimt = 30
|
||||
defaultPagination = 100
|
||||
membersAppPagination = 500
|
||||
@@ -53,8 +51,8 @@ type Source struct {
|
||||
githubUser string
|
||||
githubToken string
|
||||
|
||||
sourceID sources.SourceID
|
||||
jobID sources.JobID
|
||||
sourceID int64
|
||||
jobID int64
|
||||
verify bool
|
||||
repos []string
|
||||
members []string
|
||||
@@ -106,14 +104,14 @@ var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_GITHUB
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceID
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobID
|
||||
}
|
||||
|
||||
@@ -202,7 +200,7 @@ func (c *filteredRepoCache) includeRepo(s string) bool {
|
||||
}
|
||||
|
||||
// Init returns an initialized GitHub source.
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.log = aCtx.Logger()
|
||||
|
||||
s.name = name
|
||||
@@ -415,7 +413,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
apiEndpoint := s.conn.Endpoint
|
||||
if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {
|
||||
apiEndpoint = "https://api.github.com"
|
||||
|
||||
@@ -190,13 +190,13 @@ func TestAddMembersByApp(t *testing.T) {
|
||||
gock.New("https://api.github.com").
|
||||
Get("/app/installations").
|
||||
Reply(200).
|
||||
JSON([]map[string]any{
|
||||
JSON([]map[string]interface{}{
|
||||
{"account": map[string]string{"login": "super-secret-org", "type": "Organization"}},
|
||||
})
|
||||
gock.New("https://api.github.com").
|
||||
Get("/orgs/super-secret-org/members").
|
||||
Reply(200).
|
||||
JSON([]map[string]any{
|
||||
JSON([]map[string]interface{}{
|
||||
{"login": "ssm1"},
|
||||
{"login": "ssm2"},
|
||||
{"login": "ssm3"},
|
||||
@@ -221,7 +221,7 @@ func TestAddReposByApp(t *testing.T) {
|
||||
gock.New("https://api.github.com").
|
||||
Get("/installation/repositories").
|
||||
Reply(200).
|
||||
JSON(map[string]any{
|
||||
JSON(map[string]interface{}{
|
||||
"repositories": []map[string]string{
|
||||
{"clone_url": "https://github/ssr1.git", "full_name": "ssr1"},
|
||||
{"clone_url": "https://github/ssr2.git", "full_name": "ssr2"},
|
||||
@@ -247,7 +247,7 @@ func TestAddOrgsByUser(t *testing.T) {
|
||||
gock.New("https://api.github.com").
|
||||
Get("/user/orgs").
|
||||
Reply(200).
|
||||
JSON([]map[string]any{
|
||||
JSON([]map[string]interface{}{
|
||||
{"login": "sso2"},
|
||||
})
|
||||
|
||||
|
||||
+54
-139
@@ -29,12 +29,10 @@ import (
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
)
|
||||
|
||||
const SourceType = sourcespb.SourceType_SOURCE_TYPE_GITLAB
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
authMethod string
|
||||
user string
|
||||
@@ -55,24 +53,23 @@ type Source struct {
|
||||
// Ensure the Source satisfies the interfaces at compile time.
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
var _ sources.Validator = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_GITLAB
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized Gitlab source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
s.jobId = jobId
|
||||
@@ -142,16 +139,16 @@ func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourc
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
// Start client.
|
||||
apiClient, err := s.newClient()
|
||||
if err != nil {
|
||||
return errors.New(err)
|
||||
}
|
||||
// Get repo within target.
|
||||
repos, errs := normalizeRepos(s.repos)
|
||||
repos, errs := s.getRepos()
|
||||
for _, repoErr := range errs {
|
||||
ctx.Logger().Info("error normalizing repo", "error", repoErr)
|
||||
ctx.Logger().Info("error getting repo", "error", repoErr)
|
||||
}
|
||||
|
||||
// End early if we had errors getting specified repos but none were validated.
|
||||
@@ -161,14 +158,26 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
|
||||
|
||||
// Get all repos if not specified.
|
||||
if len(repos) == 0 {
|
||||
ignoreRepo := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {
|
||||
ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", pattern)
|
||||
})
|
||||
gitlabRepos, err2, done := s.getReposFromGitlab(ctx, apiClient, ignoreRepo)
|
||||
if done {
|
||||
return err2
|
||||
projects, err := s.getAllProjects(ctx, apiClient)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting all projects: %v", err)
|
||||
}
|
||||
// Turn projects into URLs for Git cloner.
|
||||
for _, prj := range projects {
|
||||
if s.ignoreRepo(ctx, prj.PathWithNamespace) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Ensure the urls are valid before adding them to the repo list.
|
||||
_, err := url.Parse(prj.HTTPURLToRepo)
|
||||
if err != nil {
|
||||
fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
|
||||
}
|
||||
repos = append(repos, prj.HTTPURLToRepo)
|
||||
}
|
||||
if len(repos) == 0 {
|
||||
return errors.Errorf("unable to discover any repos")
|
||||
}
|
||||
repos = gitlabRepos
|
||||
}
|
||||
|
||||
s.repos = repos
|
||||
@@ -178,69 +187,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
|
||||
return s.scanRepos(ctx, chunksChan)
|
||||
}
|
||||
|
||||
func (s *Source) Validate(ctx context.Context) []error {
|
||||
// The client is only used to query Gitlab for a repo list - it's not used to actually clone anything. Thus, we
|
||||
// don't use it if there is a list of explicitly configured repos. However, constructing it validates that the
|
||||
// configured authentication method is sensible, so we'll do it here.
|
||||
apiClient, err := s.newClient()
|
||||
if err != nil {
|
||||
return []error{err}
|
||||
}
|
||||
|
||||
_, _, err = apiClient.Users.CurrentUser()
|
||||
if err != nil {
|
||||
msg := fmt.Sprintf("gitlab authentication failed using method %v", s.authMethod)
|
||||
return []error{errors.WrapPrefix(err, msg, 0)}
|
||||
}
|
||||
|
||||
explicitlyConfiguredRepos, errs := normalizeRepos(s.repos)
|
||||
|
||||
if len(explicitlyConfiguredRepos) > 0 {
|
||||
user := s.user
|
||||
if user == "" {
|
||||
user = "placeholder"
|
||||
}
|
||||
|
||||
// We only check reachability for explicitly configured repositories. The purpose of source validation is to
|
||||
// help users validate their configuration files, and Gitlab telling us about repositories that it won't let us
|
||||
// access isn't a local configuration issue.
|
||||
for _, r := range explicitlyConfiguredRepos {
|
||||
if err := git.PingRepoUsingToken(ctx, s.token, r, user); err != nil {
|
||||
msg := fmt.Sprintf("could not reach git repository %q", r)
|
||||
err = errors.WrapPrefix(err, msg, 0)
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(s.ignoreRepos) > 0 {
|
||||
errs = append(errs, fmt.Errorf("both repositories and ignore patterns were explicitly configured; ignore patterns will not be used"))
|
||||
}
|
||||
}
|
||||
|
||||
if len(explicitlyConfiguredRepos) > 0 || len(errs) > 0 {
|
||||
return errs
|
||||
}
|
||||
|
||||
ignoreProject := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {
|
||||
msg := fmt.Sprintf("could not compile ignore repo pattern %q", pattern)
|
||||
errs = append(errs, errors.WrapPrefix(err, msg, 0))
|
||||
})
|
||||
|
||||
projects, err := s.getAllProjects(ctx, apiClient)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
return errs
|
||||
}
|
||||
|
||||
for _, p := range projects {
|
||||
if !ignoreProject(p.PathWithNamespace) {
|
||||
return errs
|
||||
}
|
||||
}
|
||||
|
||||
return append(errs, fmt.Errorf("ignore patterns excluded all projects"))
|
||||
}
|
||||
|
||||
func (s *Source) newClient() (*gitlab.Client, error) {
|
||||
// Initialize a new api instance.
|
||||
switch s.authMethod {
|
||||
@@ -376,30 +322,23 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (
|
||||
return projectList, nil
|
||||
}
|
||||
|
||||
func (s *Source) getReposFromGitlab(ctx context.Context, apiClient *gitlab.Client, ignoreRepo func(repo string) bool) ([]string, error, bool) {
|
||||
projects, err := s.getAllProjects(ctx, apiClient)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting all projects: %v", err), true
|
||||
func (s *Source) getRepos() ([]string, []error) {
|
||||
if len(s.repos) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Turn projects into URLs for Git cloner.
|
||||
var repos []string
|
||||
for _, prj := range projects {
|
||||
if ignoreRepo(prj.PathWithNamespace) {
|
||||
var validRepos []string
|
||||
var errs []error
|
||||
for _, prj := range s.repos {
|
||||
repo, err := giturl.NormalizeGitlabRepo(prj)
|
||||
if err != nil {
|
||||
errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
|
||||
continue
|
||||
}
|
||||
|
||||
// Ensure the urls are valid before adding them to the repo list.
|
||||
_, err := url.Parse(prj.HTTPURLToRepo)
|
||||
if err != nil {
|
||||
fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
|
||||
}
|
||||
repos = append(repos, prj.HTTPURLToRepo)
|
||||
validRepos = append(validRepos, repo)
|
||||
}
|
||||
if len(repos) == 0 {
|
||||
return nil, errors.Errorf("unable to discover any repos"), true
|
||||
}
|
||||
return repos, nil, false
|
||||
return validRepos, errs
|
||||
}
|
||||
|
||||
func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
@@ -472,6 +411,21 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) ignoreRepo(ctx context.Context, r string) bool {
|
||||
for _, ignore := range s.ignoreRepos {
|
||||
g, err := glob.Compile(ignore)
|
||||
if err != nil {
|
||||
ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore)
|
||||
continue
|
||||
}
|
||||
if g.Match(r) {
|
||||
ctx.Logger().V(2).Info("Ignoring repo", "repo", r)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
|
||||
func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
|
||||
s.resumeInfoMutex.Lock()
|
||||
@@ -491,42 +445,3 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
|
||||
func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
|
||||
s.scanOptions = scanOptions
|
||||
}
|
||||
|
||||
func buildIgnorer(patterns []string, onCompileErr func(err error, pattern string)) func(repo string) bool {
|
||||
var globs []glob.Glob
|
||||
|
||||
for _, pattern := range patterns {
|
||||
g, err := glob.Compile(pattern)
|
||||
if err != nil {
|
||||
onCompileErr(err, pattern)
|
||||
continue
|
||||
}
|
||||
globs = append(globs, g)
|
||||
}
|
||||
|
||||
f := func(repo string) bool {
|
||||
for _, g := range globs {
|
||||
if g.Match(repo) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return f
|
||||
}
|
||||
|
||||
func normalizeRepos(repos []string) ([]string, []error) {
|
||||
var validRepos []string
|
||||
var errs []error
|
||||
for _, prj := range repos {
|
||||
repo, err := giturl.NormalizeGitlabRepo(prj)
|
||||
if err != nil {
|
||||
errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
|
||||
continue
|
||||
}
|
||||
|
||||
validRepos = append(validRepos, repo)
|
||||
}
|
||||
return validRepos, errs
|
||||
}
|
||||
|
||||
@@ -177,150 +177,6 @@ func TestSource_Scan(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSource_Validate(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
token := secret.MustGetField("GITLAB_TOKEN")
|
||||
tokenWrongScope := secret.MustGetField("GITLAB_TOKEN_WRONG_SCOPE")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
connection *sourcespb.GitLab
|
||||
wantErrCount int
|
||||
wantErrs []string
|
||||
}{
|
||||
{
|
||||
name: "basic auth did not authenticate",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_BasicAuth{
|
||||
BasicAuth: &credentialspb.BasicAuth{
|
||||
Username: "bad-user",
|
||||
Password: "bad-password",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "token did not authenticate",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: "bad-token",
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "bad repo urls",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy", // valid
|
||||
"https://gitlab.com/testermctestface/testy/", // trailing slash
|
||||
"ssh:git@gitlab.com/testermctestface/testy", // bad protocol
|
||||
"https://gitlab.com", // no path
|
||||
"https://gitlab.com/", // no org name
|
||||
"https://gitlab.com//testy", // no org name
|
||||
"https://gitlab.com/testermctestface/", // no repo name
|
||||
},
|
||||
},
|
||||
wantErrCount: 6,
|
||||
},
|
||||
{
|
||||
name: "token does not have permission to list projects",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: tokenWrongScope,
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "repositories and ignore globs both configured",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy", // valid
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"tes1188/*-gitlab",
|
||||
"[", // glob doesn't compile, but this won't be checked
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "could not compile ignore glob(s)",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"tes1188/*-gitlab",
|
||||
"[", // glob doesn't compile
|
||||
"[a-]", // glob doesn't compile
|
||||
},
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "repositories do not exist or are not accessible",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
Repositories: []string{
|
||||
"https://gitlab.com/testermctestface/testy",
|
||||
"https://gitlab.com/testermctestface/doesn't-exist",
|
||||
"https://gitlab.com/testermctestface/also-doesn't-exist",
|
||||
},
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "ignore globs exclude all repos",
|
||||
connection: &sourcespb.GitLab{
|
||||
Credential: &sourcespb.GitLab_Token{
|
||||
Token: token,
|
||||
},
|
||||
IgnoreRepos: []string{
|
||||
"*",
|
||||
},
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Source{}
|
||||
|
||||
conn, err := anypb.New(tt.connection)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.name, 0, 0, false, conn, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Source.Init() error: %v", err)
|
||||
}
|
||||
|
||||
errs := s.Validate(ctx)
|
||||
|
||||
assert.Equal(t, tt.wantErrCount, len(errs))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
|
||||
tests := []struct {
|
||||
startingResumeInfoSlice []string
|
||||
|
||||
@@ -43,8 +43,8 @@ type JobProgressHook interface {
|
||||
// If the job supports it, the reference can also be used to cancel running via
|
||||
// CancelRun.
|
||||
type JobProgressRef struct {
|
||||
JobID JobID
|
||||
SourceID SourceID
|
||||
JobID int64
|
||||
SourceID int64
|
||||
SourceName string
|
||||
jobProgress *JobProgress
|
||||
}
|
||||
@@ -101,8 +101,8 @@ func (f ChunkError) Unwrap() error { return f.err }
|
||||
// JobProgress aggregates information about a run of a Source.
|
||||
type JobProgress struct {
|
||||
// Unique identifiers for this job.
|
||||
JobID JobID
|
||||
SourceID SourceID
|
||||
JobID int64
|
||||
SourceID int64
|
||||
SourceName string
|
||||
// Tracks whether the job is finished or not.
|
||||
ctx context.Context
|
||||
@@ -155,7 +155,7 @@ func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
|
||||
}
|
||||
|
||||
// NewJobProgress creates a new job report for the given source and job ID.
|
||||
func NewJobProgress(jobID JobID, sourceID SourceID, sourceName string, opts ...func(*JobProgress)) *JobProgress {
|
||||
func NewJobProgress(jobID, sourceID int64, sourceName string, opts ...func(*JobProgress)) *JobProgress {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
jp := &JobProgress{
|
||||
JobID: jobID,
|
||||
|
||||
@@ -38,8 +38,8 @@ func TestJobProgressFatalErrors(t *testing.T) {
|
||||
func TestJobProgressRef(t *testing.T) {
|
||||
jp := NewJobProgress(123, 456, "source name")
|
||||
ref := jp.Ref()
|
||||
assert.Equal(t, JobID(123), ref.JobID)
|
||||
assert.Equal(t, SourceID(456), ref.SourceID)
|
||||
assert.Equal(t, int64(123), ref.JobID)
|
||||
assert.Equal(t, int64(456), ref.SourceID)
|
||||
|
||||
// Test Done() blocks until Finish() is called.
|
||||
select {
|
||||
|
||||
@@ -41,7 +41,7 @@ func (m *MockJobProgressHook) End(arg0 JobProgressRef, arg1 time.Time) {
|
||||
}
|
||||
|
||||
// End indicates an expected call of End.
|
||||
func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "End", reflect.TypeOf((*MockJobProgressHook)(nil).End), arg0, arg1)
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func (m *MockJobProgressHook) EndEnumerating(arg0 JobProgressRef, arg1 time.Time
|
||||
}
|
||||
|
||||
// EndEnumerating indicates an expected call of EndEnumerating.
|
||||
func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).EndEnumerating), arg0, arg1)
|
||||
}
|
||||
@@ -65,7 +65,7 @@ func (m *MockJobProgressHook) EndUnitChunking(arg0 JobProgressRef, arg1 SourceUn
|
||||
}
|
||||
|
||||
// EndUnitChunking indicates an expected call of EndUnitChunking.
|
||||
func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).EndUnitChunking), arg0, arg1, arg2)
|
||||
}
|
||||
@@ -77,7 +77,7 @@ func (m *MockJobProgressHook) Finish(arg0 JobProgressRef) {
|
||||
}
|
||||
|
||||
// Finish indicates an expected call of Finish.
|
||||
func (mr *MockJobProgressHookMockRecorder) Finish(arg0 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) Finish(arg0 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockJobProgressHook)(nil).Finish), arg0)
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func (m *MockJobProgressHook) ReportChunk(arg0 JobProgressRef, arg1 SourceUnit,
|
||||
}
|
||||
|
||||
// ReportChunk indicates an expected call of ReportChunk.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportChunk", reflect.TypeOf((*MockJobProgressHook)(nil).ReportChunk), arg0, arg1, arg2)
|
||||
}
|
||||
@@ -101,7 +101,7 @@ func (m *MockJobProgressHook) ReportError(arg0 JobProgressRef, arg1 error) {
|
||||
}
|
||||
|
||||
// ReportError indicates an expected call of ReportError.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportError", reflect.TypeOf((*MockJobProgressHook)(nil).ReportError), arg0, arg1)
|
||||
}
|
||||
@@ -113,7 +113,7 @@ func (m *MockJobProgressHook) ReportUnit(arg0 JobProgressRef, arg1 SourceUnit) {
|
||||
}
|
||||
|
||||
// ReportUnit indicates an expected call of ReportUnit.
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportUnit", reflect.TypeOf((*MockJobProgressHook)(nil).ReportUnit), arg0, arg1)
|
||||
}
|
||||
@@ -125,7 +125,7 @@ func (m *MockJobProgressHook) Start(arg0 JobProgressRef, arg1 time.Time) {
|
||||
}
|
||||
|
||||
// Start indicates an expected call of Start.
|
||||
func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockJobProgressHook)(nil).Start), arg0, arg1)
|
||||
}
|
||||
@@ -137,7 +137,7 @@ func (m *MockJobProgressHook) StartEnumerating(arg0 JobProgressRef, arg1 time.Ti
|
||||
}
|
||||
|
||||
// StartEnumerating indicates an expected call of StartEnumerating.
|
||||
func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).StartEnumerating), arg0, arg1)
|
||||
}
|
||||
@@ -149,7 +149,7 @@ func (m *MockJobProgressHook) StartUnitChunking(arg0 JobProgressRef, arg1 Source
|
||||
}
|
||||
|
||||
// StartUnitChunking indicates an expected call of StartUnitChunking.
|
||||
func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 any) *gomock.Call {
|
||||
func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).StartUnitChunking), arg0, arg1, arg2)
|
||||
}
|
||||
|
||||
+48
-130
@@ -31,17 +31,14 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_S3
|
||||
|
||||
defaultAWSRegion = "us-east-1"
|
||||
defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
|
||||
maxObjectSizeLimit = 250 * 1024 * 1024 // 250 MiB
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
concurrency int
|
||||
log logr.Logger
|
||||
@@ -56,23 +53,22 @@ type Source struct {
|
||||
// Ensure the Source satisfies the interfaces at compile time
|
||||
var _ sources.Source = (*Source)(nil)
|
||||
var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
|
||||
var _ sources.Validator = (*Source)(nil)
|
||||
|
||||
// Type returns the type of source
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_S3
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
// Init returns an initialized AWS source
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
s.log = context.WithValues(aCtx, "source", s.Type(), "name", name).Logger()
|
||||
|
||||
s.name = name
|
||||
@@ -97,23 +93,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, so
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) Validate(ctx context.Context) []error {
|
||||
var errors []error
|
||||
visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
|
||||
roleErrs := s.validateBucketAccess(c, defaultRegionClient, roleArn, buckets)
|
||||
if len(roleErrs) > 0 {
|
||||
errors = append(errors, roleErrs...)
|
||||
}
|
||||
}
|
||||
|
||||
err := s.visitRoles(ctx, visitor)
|
||||
if err != nil {
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
// setMaxObjectSize sets the maximum size of objects that will be scanned. If
|
||||
// not set, set to a negative number, or set larger than the
|
||||
// maxObjectSizeLimit, the defaultMaxObjectSizeLimit will be used.
|
||||
@@ -174,7 +153,7 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
|
||||
|
||||
res, err := client.ListBuckets(&s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("could not list s3 buckets: %w", err)
|
||||
}
|
||||
|
||||
var bucketsToScan []string
|
||||
@@ -184,30 +163,34 @@ func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) {
|
||||
return bucketsToScan, nil
|
||||
}
|
||||
|
||||
func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) {
|
||||
func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) error {
|
||||
const defaultAWSRegion = "us-east-1"
|
||||
objectCount := uint64(0)
|
||||
|
||||
logger := s.log
|
||||
if role != "" {
|
||||
logger = logger.WithValues("roleArn", role)
|
||||
}
|
||||
|
||||
for i, bucket := range bucketsToScan {
|
||||
logger := logger.WithValues("bucket", bucket)
|
||||
|
||||
if common.IsDone(ctx) {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
|
||||
logger.V(3).Info("Scanning bucket")
|
||||
|
||||
regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
|
||||
s.log.Info("Scanning bucket", "bucket", bucket)
|
||||
region, err := s3manager.GetBucketRegionWithClient(ctx, client, bucket)
|
||||
if err != nil {
|
||||
logger.Error(err, "could not get regional client for bucket")
|
||||
s.log.Error(err, "could not get s3 region for bucket", "bucket: ", bucket)
|
||||
continue
|
||||
}
|
||||
|
||||
var regionalClient *s3.S3
|
||||
if region != defaultAWSRegion {
|
||||
regionalClient, err = s.newClient(region, role)
|
||||
if err != nil {
|
||||
s.log.Error(err, "could not make regional s3 client")
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
regionalClient = client
|
||||
}
|
||||
|
||||
errorCount := sync.Map{}
|
||||
|
||||
err = regionalClient.ListObjectsV2PagesWithContext(
|
||||
@@ -218,46 +201,40 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if role == "" {
|
||||
logger.Error(err, "could not list objects in bucket")
|
||||
} else {
|
||||
// Our documentation blesses specifying a role to assume without specifying buckets to scan, which will
|
||||
// often cause this to happen a lot (because in that case the scanner tries to scan every bucket in the
|
||||
// account, but the role probably doesn't have access to all of them). This makes it expected behavior
|
||||
// and therefore not an error.
|
||||
logger.V(3).Info("could not list objects in bucket",
|
||||
"err", err)
|
||||
}
|
||||
s.log.Error(err, "could not list objects in s3 bucket", "bucket: ", bucket)
|
||||
continue
|
||||
}
|
||||
}
|
||||
s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {
|
||||
s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
const defaultAWSRegion = "us-east-1"
|
||||
|
||||
roles := s.conn.Roles
|
||||
if len(roles) == 0 {
|
||||
roles = []string{""}
|
||||
}
|
||||
|
||||
return s.visitRoles(ctx, visitor)
|
||||
}
|
||||
for _, role := range roles {
|
||||
client, err := s.newClient(defaultAWSRegion, role)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not create s3 client", 0)
|
||||
}
|
||||
|
||||
func (s *Source) getRegionalClientForBucket(ctx context.Context, defaultRegionClient *s3.S3, role, bucket string) (*s3.S3, error) {
|
||||
region, err := s3manager.GetBucketRegionWithClient(ctx, defaultRegionClient, bucket)
|
||||
if err != nil {
|
||||
return nil, errors.WrapPrefix(err, "could not get s3 region for bucket", 0)
|
||||
bucketsToScan, err := s.getBucketsToScan(client)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if region == defaultAWSRegion {
|
||||
return defaultRegionClient, nil
|
||||
}
|
||||
|
||||
regionalClient, err := s.newClient(region, role)
|
||||
if err != nil {
|
||||
return nil, errors.WrapPrefix(err, "could not create regional s3 client", 0)
|
||||
}
|
||||
|
||||
return regionalClient, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// pageChunker emits chunks onto the given channel from a page
|
||||
@@ -280,7 +257,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
|
||||
// ignore large files
|
||||
if *obj.Size > s.maxObjectSize {
|
||||
s.log.V(5).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
|
||||
s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -419,65 +396,6 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
|
||||
_ = s.jobPool.Wait()
|
||||
}
|
||||
|
||||
func (s *Source) validateBucketAccess(ctx context.Context, client *s3.S3, roleArn string, buckets []string) []error {
|
||||
shouldHaveAccessToAllBuckets := roleArn == ""
|
||||
wasAbleToListAnyBucket := false
|
||||
var errors []error
|
||||
|
||||
for _, bucket := range buckets {
|
||||
if common.IsDone(ctx) {
|
||||
return append(errors, ctx.Err())
|
||||
}
|
||||
|
||||
regionalClient, err := s.getRegionalClientForBucket(ctx, client, roleArn, bucket)
|
||||
if err != nil {
|
||||
errors = append(errors, fmt.Errorf("could not get regional client for bucket %q: %w", bucket, err))
|
||||
continue
|
||||
}
|
||||
|
||||
_, err = regionalClient.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &bucket})
|
||||
|
||||
if err == nil {
|
||||
wasAbleToListAnyBucket = true
|
||||
} else if shouldHaveAccessToAllBuckets {
|
||||
errors = append(errors, fmt.Errorf("could not list objects in bucket %q: %w", bucket, err))
|
||||
}
|
||||
}
|
||||
|
||||
if !wasAbleToListAnyBucket {
|
||||
if roleArn == "" {
|
||||
errors = append(errors, fmt.Errorf("could not list objects in any bucket"))
|
||||
} else {
|
||||
errors = append(errors, fmt.Errorf("role %q could not list objects in any bucket", roleArn))
|
||||
}
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
func (s *Source) visitRoles(ctx context.Context, f func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string)) error {
|
||||
roles := s.conn.Roles
|
||||
if len(roles) == 0 {
|
||||
roles = []string{""}
|
||||
}
|
||||
|
||||
for _, role := range roles {
|
||||
client, err := s.newClient(defaultAWSRegion, role)
|
||||
if err != nil {
|
||||
return errors.WrapPrefix(err, "could not create s3 client", 0)
|
||||
}
|
||||
|
||||
bucketsToScan, err := s.getBucketsToScan(client)
|
||||
if err != nil {
|
||||
return fmt.Errorf("role %q could not list any s3 buckets for scanning: %w", role, err)
|
||||
}
|
||||
|
||||
f(ctx, client, role, bucketsToScan)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// S3 links currently have the general format of:
|
||||
// https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
|
||||
func makeS3Link(bucket, region, key string) string {
|
||||
|
||||
@@ -4,14 +4,10 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
@@ -49,127 +45,3 @@ func TestSource_ChunksCount(t *testing.T) {
|
||||
}
|
||||
assert.Greater(t, got, wantChunkCount)
|
||||
}
|
||||
|
||||
func TestSource_Validate(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
|
||||
defer cancel()
|
||||
|
||||
secret, err := common.GetTestSecret(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
|
||||
}
|
||||
|
||||
s3key := secret.MustGetField("AWS_S3_KEY")
|
||||
s3secret := secret.MustGetField("AWS_S3_SECRET")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
roles []string
|
||||
buckets []string
|
||||
wantErrCount int
|
||||
}{
|
||||
{
|
||||
name: "buckets without roles, can access all buckets",
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-tests",
|
||||
},
|
||||
wantErrCount: 0,
|
||||
},
|
||||
{
|
||||
name: "buckets without roles, one error per inaccessible bucket",
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-tests",
|
||||
"truffletestbucket-s3-role-assumption",
|
||||
"truffletestbucket-no-access",
|
||||
},
|
||||
wantErrCount: 2,
|
||||
},
|
||||
{
|
||||
name: "roles without buckets, all can access at least one account bucket",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
},
|
||||
wantErrCount: 0,
|
||||
},
|
||||
{
|
||||
name: "roles without buckets, one error per role that cannot access any account buckets",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
"arn:aws:iam::619888638459:role/test-no-access",
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "role and buckets, can access at least one bucket",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
},
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-role-assumption",
|
||||
"truffletestbucket-no-access",
|
||||
},
|
||||
wantErrCount: 0,
|
||||
},
|
||||
{
|
||||
name: "roles and buckets, one error per role that cannot access at least one bucket",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
"arn:aws:iam::619888638459:role/test-no-access",
|
||||
},
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-role-assumption",
|
||||
"truffletestbucket-no-access",
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
{
|
||||
name: "role and buckets, a bucket doesn't even exist",
|
||||
roles: []string{
|
||||
"arn:aws:iam::619888638459:role/s3-test-assume-role",
|
||||
},
|
||||
buckets: []string{
|
||||
"truffletestbucket-s3-role-assumption",
|
||||
"not-a-real-bucket-asljdhmglasjgvklhsdaljfh", // need a bucket name that nobody is likely to ever create
|
||||
},
|
||||
wantErrCount: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
|
||||
var cancelOnce sync.Once
|
||||
defer cancelOnce.Do(cancel)
|
||||
|
||||
// These are used by the tests that assume roles
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", s3key)
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", s3secret)
|
||||
|
||||
s := &Source{}
|
||||
|
||||
conn, err := anypb.New(&sourcespb.S3{
|
||||
// These are used by the tests that don't assume roles
|
||||
Credential: &sourcespb.S3_AccessKey{
|
||||
AccessKey: &credentialspb.KeySecret{
|
||||
Key: s3key,
|
||||
Secret: s3secret,
|
||||
},
|
||||
},
|
||||
Buckets: tt.buckets,
|
||||
Roles: tt.roles,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = s.Init(ctx, tt.name, 0, 0, false, conn, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
errs := s.Validate(ctx)
|
||||
|
||||
assert.Equal(t, tt.wantErrCount, len(errs))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+148
-23
@@ -13,9 +13,29 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
)
|
||||
|
||||
// handle uniquely identifies a Source given to the manager to manage. If the
|
||||
// SourceManager is connected to the API, it will be equivalent to the unique
|
||||
// source ID, otherwise it behaves as a counter.
|
||||
type handle int64
|
||||
|
||||
// SourceInitFunc is a function that takes a source and job ID and returns an
|
||||
// initialized Source.
|
||||
type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, error)
|
||||
|
||||
// sourceInfo is an aggregate struct to store source information provided on
|
||||
// initialization.
|
||||
type sourceInfo struct {
|
||||
initFunc SourceInitFunc
|
||||
sourceID int64
|
||||
name string
|
||||
}
|
||||
|
||||
type SourceManager struct {
|
||||
api apiClient
|
||||
hooks []JobProgressHook
|
||||
// Map of handle to source initializer.
|
||||
handles map[handle]sourceInfo
|
||||
handlesLock sync.Mutex
|
||||
// Pool limiting the amount of concurrent sources running.
|
||||
pool errgroup.Group
|
||||
poolLimit int
|
||||
@@ -32,9 +52,10 @@ type SourceManager struct {
|
||||
|
||||
// apiClient is an interface for optionally communicating with an external API.
|
||||
type apiClient interface {
|
||||
// GetIDs informs the API of the source that's about to run and returns
|
||||
// two identifiers used during source initialization.
|
||||
GetIDs(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, JobID, error)
|
||||
// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
|
||||
RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
|
||||
// GetJobID queries the API for an existing job or new job ID.
|
||||
GetJobID(ctx context.Context, id int64) (int64, error)
|
||||
}
|
||||
|
||||
// WithAPI adds an API client to the manager for tracking jobs and progress. If
|
||||
@@ -79,6 +100,7 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
|
||||
mgr := SourceManager{
|
||||
// Default to the headless API. Can be overwritten by the WithAPI option.
|
||||
api: &headlessAPI{},
|
||||
handles: make(map[handle]sourceInfo),
|
||||
outputChunks: make(chan *Chunk),
|
||||
}
|
||||
for _, opt := range opts {
|
||||
@@ -87,33 +109,89 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
|
||||
return &mgr
|
||||
}
|
||||
|
||||
func (s *SourceManager) GetIDs(ctx context.Context, sourceName string, kind sourcespb.SourceType) (SourceID, JobID, error) {
|
||||
return s.api.GetIDs(ctx, sourceName, kind)
|
||||
// Enroll informs the SourceManager to track and manage a Source.
|
||||
func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
|
||||
if s.done {
|
||||
return 0, fmt.Errorf("manager is done")
|
||||
}
|
||||
id, err := s.api.RegisterSource(ctx, name, kind)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
handleID := handle(id)
|
||||
s.handlesLock.Lock()
|
||||
defer s.handlesLock.Unlock()
|
||||
if _, ok := s.handles[handleID]; ok {
|
||||
// TODO: smartly handle this?
|
||||
return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
|
||||
}
|
||||
s.handles[handleID] = sourceInfo{
|
||||
initFunc: f,
|
||||
sourceID: int64(handleID),
|
||||
name: name,
|
||||
}
|
||||
return handleID, nil
|
||||
}
|
||||
|
||||
// Run blocks until a resource is available to run the source, then
|
||||
// synchronously runs it. The first fatal error, if any, will be returned.
|
||||
func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
progress, err := s.ScheduleRun(ctx, handle)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
<-progress.Done()
|
||||
return progress, progress.Snapshot().FatalError()
|
||||
}
|
||||
|
||||
// RunOnce blocks until a resource is available to run the source, then
|
||||
// synchronously runs it. The first fatal error, if any, will be returned.
|
||||
func (s *SourceManager) RunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
|
||||
progress, err := s.ScheduleRunOnce(ctx, name, source)
|
||||
if err != nil {
|
||||
return progress, err
|
||||
}
|
||||
<-progress.Done()
|
||||
return progress, progress.Snapshot().FatalError()
|
||||
}
|
||||
|
||||
// ScheduleRun blocks until a resource is available to run the source, then
|
||||
// asynchronously runs it. Error information is stored and accessible via the
|
||||
// JobProgressRef as it becomes available.
|
||||
func (s *SourceManager) Run(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
|
||||
return s.asyncRun(ctx, sourceName, source)
|
||||
func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
return s.asyncRun(ctx, handle)
|
||||
}
|
||||
|
||||
// ScheduleRunOnce blocks until a resource is available to run the source, then
|
||||
// asynchronously runs it. Error information is stored and accessible via the
|
||||
// JobProgressRef as it becomes available.
|
||||
func (s *SourceManager) ScheduleRunOnce(ctx context.Context, name string, source Source) (JobProgressRef, error) {
|
||||
return s.asyncRunOnce(ctx, name, source)
|
||||
}
|
||||
|
||||
// asyncRun is a helper method to asynchronously run the Source. It calls out
|
||||
// to the API to get a job ID for this run, creates a JobProgress object, then
|
||||
// waits for an available goroutine to asynchronously run it.
|
||||
func (s *SourceManager) asyncRun(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
|
||||
sourceID, jobID := source.SourceID(), source.JobID()
|
||||
func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
|
||||
// Do preflight checks before waiting on the pool.
|
||||
if err := s.preflightChecks(ctx); err != nil {
|
||||
return JobProgressRef{
|
||||
SourceName: sourceName,
|
||||
SourceID: sourceID,
|
||||
JobID: jobID,
|
||||
}, err
|
||||
return JobProgressRef{}, err
|
||||
}
|
||||
// Get the name.
|
||||
sourceInfo, ok := s.getSourceInfo(handle)
|
||||
if !ok {
|
||||
return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
|
||||
}
|
||||
sourceName := sourceInfo.name
|
||||
// Get a Job ID.
|
||||
ctx = context.WithValue(ctx, "source_id", int64(handle))
|
||||
jobID, err := s.api.GetJobID(ctx, int64(handle))
|
||||
if err != nil {
|
||||
return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
|
||||
}
|
||||
// Create a JobProgress object for tracking progress.
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
atomic.AddInt32(&s.currentRunningCount, 1)
|
||||
defer atomic.AddInt32(&s.currentRunningCount, -1)
|
||||
@@ -123,7 +201,36 @@ func (s *SourceManager) asyncRun(ctx context.Context, sourceName string, source
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel(nil)
|
||||
return s.run(ctx, source, progress)
|
||||
return s.run(ctx, sourceInfo, jobID, progress)
|
||||
})
|
||||
return progress.Ref(), nil
|
||||
}
|
||||
|
||||
// asyncRunOnce is a helper method to asynchronously run an already initialized
|
||||
// Source once. Unlike asyncRun, this method doesn't call out to the API for a
|
||||
// job ID since it's already available in the Source.
|
||||
func (s *SourceManager) asyncRunOnce(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
|
||||
if err := s.preflightChecks(ctx); err != nil {
|
||||
return JobProgressRef{}, err
|
||||
}
|
||||
ctx = context.WithValue(ctx, "source_id", source.SourceID())
|
||||
jobID := source.JobID()
|
||||
// Create a JobProgress object for tracking progress.
|
||||
ctx, cancel := context.WithCancelCause(ctx)
|
||||
progress := NewJobProgress(jobID, source.SourceID(), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
|
||||
s.pool.Go(func() error {
|
||||
ctx := context.WithValues(ctx,
|
||||
"job_id", jobID,
|
||||
"source_manager_worker_id", common.RandomID(5),
|
||||
)
|
||||
defer common.Recover(ctx)
|
||||
defer cancel(nil)
|
||||
sourceInfo := sourceInfo{
|
||||
initFunc: func(context.Context, int64, int64) (Source, error) { return source, nil },
|
||||
sourceID: source.SourceID(),
|
||||
name: sourceName,
|
||||
}
|
||||
return s.run(ctx, sourceInfo, jobID, progress)
|
||||
})
|
||||
return progress.Ref(), nil
|
||||
}
|
||||
@@ -167,8 +274,7 @@ func (s *SourceManager) AvailableCapacity() int {
|
||||
return s.poolLimit - int(runCount)
|
||||
}
|
||||
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't
|
||||
// done.
|
||||
// preflightChecks is a helper method to check the Manager or the context isn't done.
|
||||
func (s *SourceManager) preflightChecks(ctx context.Context) error {
|
||||
// Check if the manager has been Waited.
|
||||
if s.done {
|
||||
@@ -180,7 +286,7 @@ func (s *SourceManager) preflightChecks(ctx context.Context) error {
|
||||
// run is a helper method to sychronously run the source. It does not check for
|
||||
// acquired resources. An error is returned if there was a fatal error during
|
||||
// the run. This information is also recorded in the JobProgress.
|
||||
func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgress) error {
|
||||
func (s *SourceManager) run(ctx context.Context, sourceInfo sourceInfo, jobID int64, report *JobProgress) error {
|
||||
defer report.Finish()
|
||||
report.Start(time.Now())
|
||||
defer func() { report.End(time.Now()) }()
|
||||
@@ -191,10 +297,16 @@ func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgr
|
||||
}
|
||||
}()
|
||||
|
||||
// Initialize the source.
|
||||
source, err := sourceInfo.initFunc(ctx, jobID, sourceInfo.sourceID)
|
||||
if err != nil {
|
||||
report.ReportError(Fatal{err})
|
||||
return Fatal{err}
|
||||
}
|
||||
report.TrackProgress(source.GetProgress())
|
||||
ctx = context.WithValues(ctx,
|
||||
"source_type", source.Type().String(),
|
||||
"source_name", report.SourceName,
|
||||
"source_name", sourceInfo.name,
|
||||
)
|
||||
// Check for the preferred method of tracking source units.
|
||||
if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
|
||||
@@ -310,15 +422,28 @@ func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumC
|
||||
}
|
||||
}
|
||||
|
||||
// getSourceInfo is a helper method for safe concurrent access to the
|
||||
// map[handle]SourceInitFunc map.
|
||||
func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) {
|
||||
s.handlesLock.Lock()
|
||||
defer s.handlesLock.Unlock()
|
||||
f, ok := s.handles[handle]
|
||||
return f, ok
|
||||
}
|
||||
|
||||
// headlessAPI implements the apiClient interface locally.
|
||||
type headlessAPI struct {
|
||||
// Counters for assigning source and job IDs.
|
||||
// Counters for assigning handle and job IDs.
|
||||
sourceIDCounter int64
|
||||
jobIDCounter int64
|
||||
}
|
||||
|
||||
func (api *headlessAPI) GetIDs(context.Context, string, sourcespb.SourceType) (SourceID, JobID, error) {
|
||||
return SourceID(atomic.AddInt64(&api.sourceIDCounter, 1)), JobID(atomic.AddInt64(&api.jobIDCounter, 1)), nil
|
||||
func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
|
||||
return atomic.AddInt64(&api.sourceIDCounter, 1), nil
|
||||
}
|
||||
|
||||
func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
|
||||
return atomic.AddInt64(&api.jobIDCounter, 1), nil
|
||||
}
|
||||
|
||||
// mgrUnitReporter implements the UnitReporter interface.
|
||||
|
||||
@@ -6,24 +6,25 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
)
|
||||
|
||||
// DummySource implements Source and is used for testing a SourceManager.
|
||||
type DummySource struct {
|
||||
sourceID SourceID
|
||||
jobID JobID
|
||||
sourceID int64
|
||||
jobID int64
|
||||
initCalled bool
|
||||
chunker
|
||||
}
|
||||
|
||||
func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
|
||||
func (d *DummySource) SourceID() SourceID { return d.sourceID }
|
||||
func (d *DummySource) JobID() JobID { return d.jobID }
|
||||
func (d *DummySource) Init(_ context.Context, _ string, jobID JobID, sourceID SourceID, _ bool, _ *anypb.Any, _ int) error {
|
||||
func (d *DummySource) SourceID() int64 { return d.sourceID }
|
||||
func (d *DummySource) JobID() int64 { return d.jobID }
|
||||
func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
|
||||
d.initCalled = true
|
||||
d.sourceID = sourceID
|
||||
d.jobID = jobID
|
||||
return nil
|
||||
@@ -32,7 +33,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }
|
||||
|
||||
// Interface to easily test different chunking methods.
|
||||
type chunker interface {
|
||||
Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error
|
||||
Chunks(context.Context, chan *Chunk) error
|
||||
ChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) error
|
||||
Enumerate(ctx context.Context, reporter UnitReporter) error
|
||||
}
|
||||
@@ -43,7 +44,7 @@ type counterChunker struct {
|
||||
count int
|
||||
}
|
||||
|
||||
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
|
||||
for i := 0; i < c.count; i++ {
|
||||
select {
|
||||
case ch <- &Chunk{Data: []byte{c.chunkCounter}}:
|
||||
@@ -76,17 +77,20 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte
|
||||
// Chunk method that always returns an error.
|
||||
type errorChunker struct{ error }
|
||||
|
||||
func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }
|
||||
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
|
||||
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
|
||||
func (c errorChunker) Chunks(context.Context, chan *Chunk) error { return c }
|
||||
func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }
|
||||
func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }
|
||||
|
||||
// buildDummy is a helper function to enroll a DummySource with a SourceManager.
|
||||
func buildDummy(chunkMethod chunker) (Source, error) {
|
||||
source := &DummySource{chunker: chunkMethod}
|
||||
if err := source.Init(context.Background(), "dummy", 123, 456, true, nil, 42); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return source, nil
|
||||
// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
|
||||
func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
|
||||
return mgr.Enroll(context.Background(), "dummy", 1337,
|
||||
func(ctx context.Context, jobID, sourceID int64) (Source, error) {
|
||||
source := &DummySource{chunker: chunkMethod}
|
||||
if err := source.Init(ctx, "dummy", jobID, sourceID, true, nil, 42); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return source, nil
|
||||
})
|
||||
}
|
||||
|
||||
// tryRead is a helper function that will try to read from a channel and return
|
||||
@@ -102,13 +106,11 @@ func tryRead(ch <-chan *Chunk) (*Chunk, error) {
|
||||
|
||||
func TestSourceManagerRun(t *testing.T) {
|
||||
mgr := NewManager(WithBufferedOutput(8))
|
||||
source, err := buildDummy(&counterChunker{count: 1})
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 1})
|
||||
assert.NoError(t, err)
|
||||
for i := 0; i < 3; i++ {
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
<-ref.Done()
|
||||
_, err = mgr.Run(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, ref.Snapshot().FatalError())
|
||||
chunk, err := tryRead(mgr.Chunks())
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, []byte{byte(i)}, chunk.Data)
|
||||
@@ -120,31 +122,35 @@ func TestSourceManagerRun(t *testing.T) {
|
||||
|
||||
func TestSourceManagerWait(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
source, err := buildDummy(&counterChunker{count: 1})
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 1})
|
||||
assert.NoError(t, err)
|
||||
// Asynchronously run the source.
|
||||
_, err = mgr.Run(context.Background(), "dummy", source)
|
||||
_, err = mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
// Read the 1 chunk we're expecting so Waiting completes.
|
||||
<-mgr.Chunks()
|
||||
// Wait for all resources to complete.
|
||||
assert.NoError(t, mgr.Wait())
|
||||
// Run should return an error now.
|
||||
_, err = buildDummy(&counterChunker{count: 1})
|
||||
assert.NoError(t, err)
|
||||
_, err = mgr.Run(context.Background(), "dummy", source)
|
||||
// Enroll and run should return an error now.
|
||||
_, err = enrollDummy(mgr, &counterChunker{count: 1})
|
||||
assert.Error(t, err)
|
||||
_, err = mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestSourceManagerError(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
source, err := buildDummy(errorChunker{fmt.Errorf("oops")})
|
||||
handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
// A synchronous run should fail.
|
||||
_, err = mgr.Run(context.Background(), handle)
|
||||
assert.Error(t, err)
|
||||
// Scheduling a run should not fail, but the error should surface in
|
||||
// Wait().
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
assert.Error(t, ref.Snapshot().FatalError())
|
||||
assert.Error(t, mgr.Wait())
|
||||
assert.Error(t, ref.Snapshot().FatalError())
|
||||
}
|
||||
|
||||
func TestSourceManagerReport(t *testing.T) {
|
||||
@@ -154,11 +160,11 @@ func TestSourceManagerReport(t *testing.T) {
|
||||
{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},
|
||||
} {
|
||||
mgr := NewManager(opts...)
|
||||
source, err := buildDummy(&counterChunker{count: 4})
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 4})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
// Synchronously run the source.
|
||||
ref, err := mgr.Run(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
assert.Equal(t, 0, len(ref.Snapshot().Errors))
|
||||
assert.Equal(t, uint64(4), ref.Snapshot().TotalChunks)
|
||||
}
|
||||
@@ -172,7 +178,7 @@ type unitChunk struct {
|
||||
|
||||
type unitChunker struct{ steps []unitChunk }
|
||||
|
||||
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {
|
||||
for _, step := range c.steps {
|
||||
if step.err != "" {
|
||||
continue
|
||||
@@ -216,11 +222,10 @@ func TestSourceManagerNonFatalError(t *testing.T) {
|
||||
{unit: "three", err: "not again"},
|
||||
}
|
||||
mgr := NewManager(WithBufferedOutput(8), WithSourceUnits())
|
||||
source, err := buildDummy(&unitChunker{input})
|
||||
handle, err := enrollDummy(mgr, &unitChunker{input})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
ref, err := mgr.Run(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
report := ref.Snapshot()
|
||||
assert.Equal(t, len(input), int(report.TotalUnits))
|
||||
assert.Equal(t, len(input), int(report.FinishedUnits))
|
||||
@@ -231,11 +236,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {
|
||||
|
||||
func TestSourceManagerContextCancelled(t *testing.T) {
|
||||
mgr := NewManager(WithBufferedOutput(8))
|
||||
source, err := buildDummy(&counterChunker{count: 100})
|
||||
handle, err := enrollDummy(mgr, &counterChunker{count: 100})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ref, err := mgr.Run(ctx, "dummy", source)
|
||||
ref, err := mgr.ScheduleRun(ctx, handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cancel()
|
||||
@@ -245,33 +250,60 @@ func TestSourceManagerContextCancelled(t *testing.T) {
|
||||
}
|
||||
|
||||
type DummyAPI struct {
|
||||
registerSource func(context.Context, string, sourcespb.SourceType) (SourceID, error)
|
||||
getJobID func(context.Context, SourceID) (JobID, error)
|
||||
registerSource func(context.Context, string, sourcespb.SourceType) (int64, error)
|
||||
getJobID func(context.Context, int64) (int64, error)
|
||||
}
|
||||
|
||||
func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, error) {
|
||||
func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
|
||||
return api.registerSource(ctx, name, kind)
|
||||
}
|
||||
|
||||
func (api DummyAPI) GetJobID(ctx context.Context, id SourceID) (JobID, error) {
|
||||
func (api DummyAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
|
||||
return api.getJobID(ctx, id)
|
||||
}
|
||||
|
||||
func TestSourceManagerJobAndSourceIDs(t *testing.T) {
|
||||
mgr := NewManager(WithAPI(DummyAPI{
|
||||
registerSource: func(context.Context, string, sourcespb.SourceType) (int64, error) {
|
||||
return 1337, nil
|
||||
},
|
||||
getJobID: func(context.Context, int64) (int64, error) {
|
||||
return 9001, nil
|
||||
},
|
||||
}))
|
||||
var (
|
||||
initializedJobID int64
|
||||
initializedSourceID int64
|
||||
)
|
||||
handle, err := mgr.Enroll(context.Background(), "dummy", 1337,
|
||||
func(ctx context.Context, jobID, sourceID int64) (Source, error) {
|
||||
initializedJobID = jobID
|
||||
initializedSourceID = sourceID
|
||||
return nil, fmt.Errorf("ignore")
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ref, _ := mgr.Run(context.Background(), handle)
|
||||
assert.Equal(t, int64(1337), initializedSourceID)
|
||||
assert.Equal(t, int64(1337), ref.SourceID)
|
||||
assert.Equal(t, int64(9001), initializedJobID)
|
||||
assert.Equal(t, int64(9001), ref.JobID)
|
||||
assert.Equal(t, "dummy", ref.SourceName)
|
||||
}
|
||||
|
||||
// Chunk method that has a custom callback for the Chunks method.
|
||||
type callbackChunker struct {
|
||||
cb func(context.Context, chan *Chunk) error
|
||||
}
|
||||
|
||||
func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {
|
||||
return c.cb(ctx, ch)
|
||||
}
|
||||
func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error { return c.cb(ctx, ch) }
|
||||
func (c callbackChunker) Enumerate(context.Context, UnitReporter) error { return nil }
|
||||
func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }
|
||||
|
||||
func TestSourceManagerCancelRun(t *testing.T) {
|
||||
mgr := NewManager(WithBufferedOutput(8))
|
||||
var returnedErr error
|
||||
source, err := buildDummy(callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
|
||||
handle, err := enrollDummy(mgr, callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
|
||||
// The context passed to Chunks should get cancelled when ref.CancelRun() is called.
|
||||
<-ctx.Done()
|
||||
returnedErr = fmt.Errorf("oh no: %w", ctx.Err())
|
||||
@@ -279,7 +311,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
|
||||
}})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
cancelErr := fmt.Errorf("abort! abort!")
|
||||
@@ -293,7 +325,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
|
||||
func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
mgr := NewManager(WithConcurrentSources(1337))
|
||||
start, end := make(chan struct{}), make(chan struct{})
|
||||
source, err := buildDummy(callbackChunker{func(context.Context, chan *Chunk) error {
|
||||
handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
|
||||
start <- struct{}{} // Send start signal.
|
||||
<-end // Wait for end signal.
|
||||
return nil
|
||||
@@ -301,7 +333,7 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
ref, err := mgr.ScheduleRun(context.Background(), handle)
|
||||
assert.NoError(t, err)
|
||||
|
||||
<-start // Wait for start signal.
|
||||
@@ -310,3 +342,21 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
<-ref.Done() // Wait for the job to finish.
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
}
|
||||
|
||||
func TestSourceManagerRunOnce(t *testing.T) {
|
||||
mgr := NewManager()
|
||||
var chunksCalled bool
|
||||
source := &DummySource{sourceID: 123, jobID: 456, chunker: callbackChunker{
|
||||
func(context.Context, chan *Chunk) error {
|
||||
chunksCalled = true
|
||||
return nil
|
||||
},
|
||||
}}
|
||||
ref, err := mgr.RunOnce(context.Background(), "test", source)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, source.jobID, ref.JobID)
|
||||
assert.Equal(t, source.sourceID, ref.SourceID)
|
||||
assert.Equal(t, "test", ref.SourceName)
|
||||
assert.True(t, chunksCalled)
|
||||
assert.False(t, source.initCalled)
|
||||
}
|
||||
|
||||
+7
-26
@@ -11,19 +11,14 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
|
||||
)
|
||||
|
||||
type (
|
||||
SourceID int64
|
||||
JobID int64
|
||||
)
|
||||
|
||||
// Chunk contains data to be decoded and scanned along with context on where it came from.
|
||||
type Chunk struct {
|
||||
// SourceName is the name of the Source that produced the chunk.
|
||||
SourceName string
|
||||
// SourceID is the ID of the source that the Chunk originated from.
|
||||
SourceID SourceID
|
||||
SourceID int64
|
||||
// JobID is the ID of the job that the Chunk originated from.
|
||||
JobID JobID
|
||||
JobID int64
|
||||
// SourceType is the type of Source that produced the chunk.
|
||||
SourceType sourcespb.SourceType
|
||||
// SourceMetadata holds the context of where the Chunk was found.
|
||||
@@ -35,32 +30,18 @@ type Chunk struct {
|
||||
Verify bool
|
||||
}
|
||||
|
||||
// ChunkingTarget specifies criteria for a targeted chunking process.
|
||||
// Instead of collecting data indiscriminately, this struct allows the caller
|
||||
// to specify particular subsets of data they're interested in. This becomes
|
||||
// especially useful when one needs to verify or recheck specific data points
|
||||
// without processing the entire dataset.
|
||||
type ChunkingTarget struct {
|
||||
// QueryCriteria represents specific parameters or conditions to target the chunking process.
|
||||
QueryCriteria source_metadatapb.MetaData
|
||||
}
|
||||
|
||||
// Source defines the interface required to implement a source chunker.
|
||||
type Source interface {
|
||||
// Type returns the source type, used for matching against configuration and jobs.
|
||||
Type() sourcespb.SourceType
|
||||
// SourceID returns the initialized source ID used for tracking relationships in the DB.
|
||||
SourceID() SourceID
|
||||
SourceID() int64
|
||||
// JobID returns the initialized job ID used for tracking relationships in the DB.
|
||||
JobID() JobID
|
||||
JobID() int64
|
||||
// Init initializes the source.
|
||||
Init(aCtx context.Context, name string, jobId JobID, sourceId SourceID, verify bool, connection *anypb.Any, concurrency int) error
|
||||
// Chunks emits data over a channel which is then decoded and scanned for secrets.
|
||||
// By default, data is obtained indiscriminately. However, by providing one or more
|
||||
// ChunkingTarget parameters, the caller can direct the function to retrieve
|
||||
// specific chunks of data. This targeted approach allows for efficient and
|
||||
// intentional data processing, beneficial when verifying or rechecking specific data points.
|
||||
Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error
|
||||
Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
|
||||
// Chunks emits data over a channel that is decoded and scanned for secrets.
|
||||
Chunks(ctx context.Context, chunksChan chan *Chunk) error
|
||||
// GetProgress is the completion progress (percentage) for Scanned Source.
|
||||
GetProgress() *Progress
|
||||
}
|
||||
|
||||
@@ -23,16 +23,12 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
const (
|
||||
SourceType = sourcespb.SourceType_SOURCE_TYPE_SYSLOG
|
||||
|
||||
nilString = ""
|
||||
)
|
||||
const nilString = ""
|
||||
|
||||
type Source struct {
|
||||
name string
|
||||
sourceId sources.SourceID
|
||||
jobId sources.JobID
|
||||
sourceId int64
|
||||
jobId int64
|
||||
verify bool
|
||||
syslog *Syslog
|
||||
sources.Progress
|
||||
@@ -42,14 +38,14 @@ type Source struct {
|
||||
type Syslog struct {
|
||||
sourceType sourcespb.SourceType
|
||||
sourceName string
|
||||
sourceID sources.SourceID
|
||||
jobID sources.JobID
|
||||
sourceID int64
|
||||
jobID int64
|
||||
sourceMetadataFunc func(hostname, appname, procid, timestamp, facility, client string) *source_metadatapb.MetaData
|
||||
verify bool
|
||||
concurrency *semaphore.Weighted
|
||||
}
|
||||
|
||||
func NewSyslog(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
|
||||
func NewSyslog(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
|
||||
sourceMetadataFunc func(hostname, appname, procid, timestamp, facility, client string) *source_metadatapb.MetaData,
|
||||
) *Syslog {
|
||||
return &Syslog{
|
||||
@@ -107,14 +103,14 @@ var _ sources.Source = (*Source)(nil)
|
||||
// Type returns the type of source.
|
||||
// It is used for matching source types in configuration and job input.
|
||||
func (s *Source) Type() sourcespb.SourceType {
|
||||
return SourceType
|
||||
return sourcespb.SourceType_SOURCE_TYPE_SYSLOG
|
||||
}
|
||||
|
||||
func (s *Source) SourceID() sources.SourceID {
|
||||
func (s *Source) SourceID() int64 {
|
||||
return s.sourceId
|
||||
}
|
||||
|
||||
func (s *Source) JobID() sources.JobID {
|
||||
func (s *Source) JobID() int64 {
|
||||
return s.jobId
|
||||
}
|
||||
|
||||
@@ -123,7 +119,7 @@ func (s *Source) InjectConnection(conn *sourcespb.Syslog) {
|
||||
}
|
||||
|
||||
// Init returns an initialized Syslog source.
|
||||
func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
|
||||
|
||||
s.name = name
|
||||
s.sourceId = sourceId
|
||||
@@ -186,7 +182,7 @@ func (s *Source) verifyConnectionConfig() error {
|
||||
}
|
||||
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
switch {
|
||||
case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:
|
||||
cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))
|
||||
|
||||
@@ -936,8 +936,6 @@ enum DetectorType {
|
||||
Salesforce = 925;
|
||||
HuggingFace = 926;
|
||||
Snowflake = 927;
|
||||
Sourcegraph = 928;
|
||||
Tailscale = 929;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
Reference in New Issue
Block a user