diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f96e2e93..a82368c1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,7 +17,7 @@ jobs: steps: - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 60ce7844..3fa4d64f 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -1,4 +1,4 @@ -name: Performance Test +name: Performance Test on: [pull_request] @@ -7,13 +7,13 @@ jobs: # skip if PR is from a fork. # TODO: this could probabaly be refactored a bit so that it runs on forks if: ${{ ! github.event.pull_request.head.repo.fork }} - + runs-on: ubuntu-latest steps: - name: Install Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59729001..3428f28b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - name: Run GoReleaser uses: goreleaser/goreleaser-action@v4 with: diff --git a/.github/workflows/secrets.yml b/.github/workflows/secrets.yml index 68e72098..8a6b6907 100644 --- a/.github/workflows/secrets.yml +++ b/.github/workflows/secrets.yml @@ -16,7 +16,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - name: Checkout code uses: actions/checkout@v3 with: diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 7c7a73fe..8aea23b3 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -11,7 +11,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - name: Checkout code uses: actions/checkout@v3 - name: Smoke diff --git a/.github/workflows/snifftest.yml b/.github/workflows/snifftest.yml index b451d924..234bc541 100644 --- a/.github/workflows/snifftest.yml +++ b/.github/workflows/snifftest.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - uses: actions/checkout@v3 - name: Run Snifftest - run: make snifftest \ No newline at end of file + run: make snifftest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 88ff3402..b0ec735d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - name: Checkout code uses: actions/checkout@v3 - id: 'auth' @@ -42,7 +42,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - name: Checkout code uses: actions/checkout@v3 - id: 'auth' @@ -63,7 +63,7 @@ jobs: - name: Install Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: '1.21' - name: Checkout code uses: actions/checkout@v3 - name: Test diff --git a/README.md b/README.md index c7b2c195..cc658311 100644 --- a/README.md +++ b/README.md @@ -116,25 +116,31 @@ Expected output: trufflehog s3 --bucket= --only-verified ``` -## 5: Scan a Github Repo using SSH authentication in docker +## 5: Scan S3 buckets using IAM Roles + +```bash +trufflehog s3 --role-arn= +``` + +## 6: Scan a Github Repo using SSH authentication in docker ```bash docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys ``` -## 6: Scan individual files or directories +## 7: Scan individual files or directories ```bash trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir ``` -## 7: Scan GCS buckets for verified secrets. +## 8: Scan GCS buckets for verified secrets. ```bash trufflehog gcs --project-id= --cloud-environment --only-verified ``` -# 8: Scan a Docker image for verified secrets. +## 9: Scan a Docker image for verified secrets. Use the `--image` flag multiple times to scan multiple images. @@ -218,6 +224,30 @@ For example, to scan a `git` repository, start with $ trufflehog git https://github.com/trufflesecurity/trufflehog.git ``` +## S3 + +The S3 source supports assuming IAM roles for scanning in addition to IAM users. This makes it easier for users to scan multiple AWS accounts without needing to rely on hardcoded credentials for each account. + +The IAM identity that TruffleHog uses initially will need to have `AssumeRole` privileges as a principal in the [trust policy](https://aws.amazon.com/blogs/security/how-to-use-trust-policies-with-iam-roles/) of each IAM role to assume. + +To scan a specific bucket using locally set credentials or instance metadata if on an EC2 instance: + +```bash +trufflehog s3 --bucket= +``` + +To scan a specific bucket using an assumed role: + +```bash +trufflehog s3 --bucket= --role-arn= +``` + +Multiple roles can be passed as separate arguments. The following command will attempt to scan every bucket each role has permissions to list in the S3 API: + +```bash +trufflehog s3 --role-arn= --role-arn= +``` + Exit Codes: - 0: No errors and no results were found. - 1: An error was encountered. Sources may not have completed scans. @@ -263,9 +293,9 @@ jobs: extra_args: --debug --only-verified ``` -# Precommit Hook +# Pre-commit Hook -Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer. +Trufflehog can be used in a pre-commit hook to prevent credentials from leaking before they ever leave your computer. An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation). ```yaml diff --git a/go.mod b/go.mod index fb21c1bf..a0818678 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,15 @@ require ( github.com/BobuSumisu/aho-corasick v1.0.3 github.com/TheZeroSlave/zapsentry v1.17.0 github.com/aws/aws-sdk-go v1.44.83 + github.com/aymanbagabas/go-osc52 v1.2.1 github.com/bill-rich/disk-buffer-reader v0.1.7 github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb github.com/bradleyfalzon/ghinstallation/v2 v2.6.0 + github.com/charmbracelet/bubbles v0.16.1 + github.com/charmbracelet/bubbletea v0.24.1 + github.com/charmbracelet/glamour v0.6.0 + github.com/charmbracelet/lipgloss v0.7.1 github.com/couchbase/gocb/v2 v2.6.3 github.com/crewjam/rfc5424 v0.1.0 github.com/denisenkom/go-mssqldb v0.12.3 @@ -26,7 +31,7 @@ require ( github.com/felixge/fgprof v0.9.3 github.com/getsentry/sentry-go v0.22.0 github.com/go-errors/errors v1.4.2 - github.com/go-git/go-git/v5 v5.8.0 + github.com/go-git/go-git/v5 v5.8.1 github.com/go-ldap/ldap/v3 v3.4.5 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.4 @@ -38,6 +43,7 @@ require ( github.com/google/go-containerregistry v0.15.2 github.com/google/go-github/v42 v42.0.0 github.com/googleapis/gax-go/v2 v2.12.0 + github.com/h2non/filetype v1.1.3 github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/golang-lru v0.5.1 github.com/jlaffaye/ftp v0.2.0 @@ -45,8 +51,11 @@ require ( github.com/jpillora/overseer v1.1.6 github.com/kylelemons/godebug v1.1.0 github.com/lib/pq v1.10.9 + github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e + github.com/mattn/go-isatty v0.0.18 github.com/mattn/go-sqlite3 v1.14.17 github.com/mholt/archiver/v4 v4.0.0-alpha.8 + github.com/muesli/reflow v0.3.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/paulbellamy/ratecounter v0.2.0 github.com/pkg/errors v0.9.1 @@ -77,6 +86,7 @@ require ( cloud.google.com/go/compute v1.20.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.0 // indirect + dario.cat/mergo v1.0.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.24 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect @@ -88,9 +98,13 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/alecthomas/chroma v0.10.0 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andybalholm/brotli v1.0.5 // indirect + github.com/atotto/clipboard v0.1.4 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bodgit/plumbing v1.2.0 // indirect @@ -99,10 +113,12 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/connesc/cipherio v0.2.1 // indirect + github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/couchbase/gocbcore/v10 v10.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/dlclark/regexp2 v1.4.0 // indirect github.com/docker/cli v23.0.5+incompatible // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/docker v23.0.5+incompatible // indirect @@ -125,6 +141,7 @@ require ( github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -136,13 +153,20 @@ require ( github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/pgzip v1.2.5 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/magefile/mage v1.14.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/microcosm-cc/bluemonday v1.0.23 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.15.1 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.23.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect @@ -154,9 +178,11 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect + github.com/rivo/uniseg v0.4.2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/sahilm/fuzzy v0.1.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect - github.com/skeema/knownhosts v1.1.1 // indirect + github.com/skeema/knownhosts v1.2.0 // indirect github.com/tetratelabs/wazero v1.2.1 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.10 // indirect @@ -166,6 +192,8 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + github.com/yuin/goldmark v1.5.2 // indirect + github.com/yuin/goldmark-emoji v1.0.1 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.7.0 // indirect @@ -174,6 +202,7 @@ require ( golang.org/x/mod v0.11.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/sys v0.10.0 // indirect + golang.org/x/term v0.10.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index 81fce99f..3f71514c 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0= @@ -66,6 +68,8 @@ github.com/TheZeroSlave/zapsentry v1.17.0 h1:RIQCG89U7vWWZVmmCxeUz/g32WEcAYXUrXH github.com/TheZeroSlave/zapsentry v1.17.0/go.mod h1:D1YMfSuu6xnkhwFXxrronesmsiyDhIqo+86I3Ok+r64= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= +github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= +github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= @@ -77,8 +81,17 @@ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= +github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.44.83 h1:7+Rtc2Eio6EKUNoZeMV/IVxzVrY5oBQcNPtCcgIHYJA= github.com/aws/aws-sdk-go v1.44.83/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= +github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E= +github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -103,6 +116,14 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= +github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= +github.com/charmbracelet/bubbletea v0.24.1 h1:LpdYfnu+Qc6XtvMz6d/6rRY71yttHTP5HtrjMgWvixc= +github.com/charmbracelet/bubbletea v0.24.1/go.mod h1:rK3g/2+T8vOSEkNHvtq40umJpeVYDn6bLaqbgzhL/hg= +github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= +github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= +github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= +github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -118,6 +139,8 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/connesc/cipherio v0.2.1 h1:FGtpTPMbKNNWByNrr9aEBtaJtXjqOzkIXNYJp6OEycw= github.com/connesc/cipherio v0.2.1/go.mod h1:ukY0MWJDFnJEbXMQtOcn2VmTpRfzcTz4OoVrWGGJZcA= +github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= +github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c= @@ -138,6 +161,8 @@ github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+ github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= +github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/cli v23.0.5+incompatible h1:ufWmAOuD3Vmr7JP2G5K3cyuNC4YZWiAsuDEvFVVDafE= github.com/docker/cli v23.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -181,8 +206,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmS github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git/v5 v5.8.0 h1:Rc543s6Tyq+YcyPwZRvU4jzZGM8rB/wWu94TnTIYALQ= -github.com/go-git/go-git/v5 v5.8.0/go.mod h1:coJHKEOk5kUClpsNlXrUvPrDxY3w3gjHvhcZd8Fodw8= +github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+A= +github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8= @@ -286,8 +311,12 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= +github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -306,7 +335,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= @@ -343,96 +371,78 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo= +github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= -github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mholt/archiver/v4 v4.0.0-alpha.8 h1:tRGQuDVPh66WCOelqe6LIGh0gwmfwxUrSSDunscGsRM= github.com/mholt/archiver/v4 v4.0.0-alpha.8/go.mod h1:5f7FUYGXdJWUjESffJaYR4R60VhnHxb2X3T1teMyv5A= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= +github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= +github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc3 h1:fzg1mXZFj8YdPeNkRXMg+zb88BFV0Ys52cJydRwBkb8= github.com/opencontainers/image-spec v1.1.0-rc3/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= -github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= -github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/rabbitmq/amqp091-go v1.8.1 h1:RejT1SBUim5doqcL6s7iN6SBmsQqyTgXb1xMlH0h1hA= github.com/rabbitmq/amqp091-go v1.8.1/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/sahilm/fuzzy v0.1.0/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/skeema/knownhosts v1.1.1 h1:MTk78x9FPgDFVFkDLTrsnnfCJl7g1C/nnKvePgrIngE= -github.com/skeema/knownhosts v1.1.1/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= -github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= +github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= -github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt8ZaU= github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -444,63 +454,41 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= -github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4qlUWs= github.com/tetratelabs/wazero v1.2.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= -github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w= github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= -github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= -github.com/wasilibs/go-re2 v1.3.0 h1:LFhBNzoStM3wMie6rN2slD1cuYH2CGiHpvNL3UtcsMw= github.com/wasilibs/go-re2 v1.3.0/go.mod h1:AafrCXVvGRJJOImMajgJ2M7rVmWyisVK7sFshbxnVrg= -github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ= -github.com/xanzy/go-gitlab v0.88.0 h1:9GHBrxyCUNZZNuAsbJ1NbEH6XAYsKyTn6NfE0wYO5SY= github.com/xanzy/go-gitlab v0.88.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= -github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yuin/goldmark v1.5.2/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE= go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/mock v0.2.0 h1:TaP3xedm7JaAgScZO7tlvlKrqT0p7I6OsdGB5YNSMDU= go.uber.org/mock v0.2.0/go.mod h1:J0y0rp9L3xiff1+ZBfKxlC1fz2+aO16tw0tsDOixfuM= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go4.org v0.0.0-20200411211856-f5505b9728dd h1:BNJlw5kRTzdmyfh5U8F93HA2OwkP7ZGwA51eJ/0wKOU= go4.org v0.0.0-20200411211856-f5505b9728dd/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -514,7 +502,6 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -524,7 +511,6 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7 h1:lhh9BuEmjIBOdOhMIGSQzdW5LbTtFbUV1k9O/Rlq3SE= golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -548,7 +534,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -575,11 +560,11 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -587,7 +572,6 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -599,7 +583,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -635,14 +618,15 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -650,7 +634,7 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -663,11 +647,9 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -698,13 +680,11 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -714,14 +694,12 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M= google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -738,11 +716,8 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs= google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk= google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc= google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -756,7 +731,6 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -770,36 +744,27 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -808,5 +773,4 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/main.go b/main.go index 931e7ec3..67cfbd7e 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( "github.com/felixge/fgprof" "github.com/go-logr/logr" "github.com/jpillora/overseer" + "github.com/mattn/go-isatty" "google.golang.org/protobuf/types/known/anypb" "gopkg.in/alecthomas/kingpin.v2" @@ -28,6 +29,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui" "github.com/trufflesecurity/trufflehog/v3/pkg/updater" "github.com/trufflesecurity/trufflehog/v3/pkg/version" ) @@ -66,21 +68,25 @@ var ( gitScanSinceCommit = gitScan.Flag("since-commit", "Commit to start scan from.").String() gitScanBranch = gitScan.Flag("branch", "Branch to scan.").String() gitScanMaxDepth = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int() + gitScanBare = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool() _ = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool() _ = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool() _ = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool() - githubScan = cli.Command("github", "Find credentials in GitHub repositories.") - githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String() - githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings() - githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() - githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String() - githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool() - githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool() - githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings() - githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings() - githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() - githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() + githubScan = cli.Command("github", "Find credentials in GitHub repositories.") + githubScanEndpoint = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String() + githubScanRepos = githubScan.Flag("repo", `GitHub repository to scan. You can repeat this flag. Example: "https://github.com/dustin-decker/secretsandstuff"`).Strings() + githubScanOrgs = githubScan.Flag("org", `GitHub organization to scan. You can repeat this flag. Example: "trufflesecurity"`).Strings() + githubScanToken = githubScan.Flag("token", "GitHub token. Can be provided with environment variable GITHUB_TOKEN.").Envar("GITHUB_TOKEN").String() + githubIncludeForks = githubScan.Flag("include-forks", "Include forks in scan.").Bool() + githubIncludeMembers = githubScan.Flag("include-members", "Include organization member repositories in scan.").Bool() + githubIncludeRepos = githubScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings() + githubExcludeRepos = githubScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Github repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings() + githubScanIncludePaths = githubScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() + githubScanExcludePaths = githubScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() + githubScanIssueComments = githubScan.Flag("issue-comments", "Include issue comments in scan.").Bool() + githubScanPRComments = githubScan.Flag("pr-comments", "Include pull request comments in scan.").Bool() + githubScanGistComments = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool() gitlabScan = cli.Command("gitlab", "Find credentials in GitLab repositories.") // TODO: Add more GitLab options @@ -99,10 +105,11 @@ var ( filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String() filesystemScanExcludePaths = filesystemScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String() - s3Scan = cli.Command("s3", "Find credentials in S3 buckets.") - s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String() - s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String() - s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String() + s3Scan = cli.Command("s3", "Find credentials in S3 buckets.") + s3ScanKey = s3Scan.Flag("key", "S3 key used to authenticate. Can be provided with environment variable AWS_ACCESS_KEY_ID.").Envar("AWS_ACCESS_KEY_ID").String() + s3ScanRoleArns = s3Scan.Flag("role-arn", "Specify the ARN of an IAM role to assume for scanning. You can repeat this flag.").Strings() + s3ScanSecret = s3Scan.Flag("secret", "S3 secret used to authenticate. Can be provided with environment variable AWS_SECRET_ACCESS_KEY.").Envar("AWS_SECRET_ACCESS_KEY").String() + s3ScanSessionToken = s3Scan.Flag("session-token", "S3 session token used to authenticate temporary credentials. Can be provided with environment variable AWS_SESSION_TOKEN.").Envar("AWS_SESSION_TOKEN").String() s3ScanCloudEnv = s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool() s3ScanBuckets = s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings() s3ScanMaxObjectSize = s3Scan.Flag("max-object-size", "Maximum size of objects to scan. Objects larger than this will be skipped. (Byte units eg. 512B, 2KB, 4MB)").Default("250MB").Bytes() @@ -143,6 +150,18 @@ func init() { } cli.Version("trufflehog " + version.BuildVersion) + + if len(os.Args) <= 1 && isatty.IsTerminal(os.Stdout.Fd()) { + args := tui.Run() + if len(args) == 0 { + os.Exit(0) + } + + // Overwrite the Args slice so overseer works properly. + os.Args = os.Args[:1] + os.Args = append(os.Args, args...) + } + cmd = kingpin.MustParse(cli.Parse(os.Args[1:])) switch { @@ -377,6 +396,7 @@ func run(state overseer.State) { HeadRef: *gitScanBranch, BaseRef: *gitScanSinceCommit, MaxDepth: *gitScanMaxDepth, + Bare: *gitScanBare, Filter: filter, ExcludeGlobs: excludedGlobs, } @@ -393,16 +413,19 @@ func run(state overseer.State) { } cfg := sources.GithubConfig{ - Endpoint: *githubScanEndpoint, - Token: *githubScanToken, - IncludeForks: *githubIncludeForks, - IncludeMembers: *githubIncludeMembers, - Concurrency: *concurrency, - ExcludeRepos: *githubExcludeRepos, - IncludeRepos: *githubIncludeRepos, - Repos: *githubScanRepos, - Orgs: *githubScanOrgs, - Filter: filter, + Endpoint: *githubScanEndpoint, + Token: *githubScanToken, + IncludeForks: *githubIncludeForks, + IncludeMembers: *githubIncludeMembers, + Concurrency: *concurrency, + ExcludeRepos: *githubExcludeRepos, + IncludeRepos: *githubIncludeRepos, + Repos: *githubScanRepos, + Orgs: *githubScanOrgs, + IncludeIssueComments: *githubScanIssueComments, + IncludePullRequestComments: *githubScanPRComments, + IncludeGistComments: *githubScanGistComments, + Filter: filter, } if err := e.ScanGitHub(ctx, cfg); err != nil { logFatal(err, "Failed to scan Github.") @@ -446,6 +469,7 @@ func run(state overseer.State) { Secret: *s3ScanSecret, SessionToken: *s3ScanSessionToken, Buckets: *s3ScanBuckets, + Roles: *s3ScanRoleArns, CloudCred: *s3ScanCloudEnv, MaxObjectSize: int64(*s3ScanMaxObjectSize), } @@ -517,6 +541,7 @@ func run(state overseer.State) { "bytes", metrics.BytesScanned, "verified_secrets", metrics.VerifiedSecretsFound, "unverified_secrets", metrics.UnverifiedSecretsFound, + "scan_duration", metrics.ScanDuration.String(), ) if *printAvgDetectorTime { diff --git a/newest.txt b/newest.txt new file mode 100644 index 00000000..0e96048b --- /dev/null +++ b/newest.txt @@ -0,0 +1,39 @@ +goos: darwin +goarch: arm64 +pkg: github.com/trufflesecurity/trufflehog/v3/pkg/detectors/twilio +BenchmarkFromData +BenchmarkFromData/small +BenchmarkFromData/small-8 6834206 174.6 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 6899049 174.6 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 6832052 175.1 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 6907274 176.5 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 5807835 185.5 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 6880714 174.6 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 6863913 176.3 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 6900600 174.4 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 6869823 174.6 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/small-8 6820802 174.0 ns/op 16 B/op 1 allocs/op +BenchmarkFromData/medium +BenchmarkFromData/medium-8 59660 20205 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 59601 21194 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 58342 20113 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 59606 22125 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 59839 20212 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 59560 20201 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 59577 20179 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 59361 20211 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 59622 20167 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/medium-8 59832 20160 ns/op 5392 B/op 2 allocs/op +BenchmarkFromData/big +BenchmarkFromData/big-8 2403 494538 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2389 496629 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2450 495102 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2427 495284 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2433 495032 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2427 494511 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2426 495035 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2416 493870 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2451 493632 ns/op 131088 B/op 2 allocs/op +BenchmarkFromData/big-8 2425 500507 ns/op 131088 B/op 2 allocs/op +PASS +ok github.com/trufflesecurity/trufflehog/v3/pkg/detectors/twilio 40.806s diff --git a/pkg/common/context.go b/pkg/common/context.go index 5a6d005b..77549440 100644 --- a/pkg/common/context.go +++ b/pkg/common/context.go @@ -16,9 +16,14 @@ func IsDone(ctx context.Context) bool { // write would succeed, either operation will be performed randomly. func CancellableWrite[T any](ctx context.Context, ch chan<- T, item T) error { select { - case ch <- item: - return nil - case <-ctx.Done(): + case <-ctx.Done(): // priority to context cancellation return ctx.Err() + default: + select { + case <-ctx.Done(): + return ctx.Err() + case ch <- item: + return nil + } } } diff --git a/pkg/detectors/alchemy/alchemy.go b/pkg/detectors/alchemy/alchemy.go index d5ca2355..1a551180 100644 --- a/pkg/detectors/alchemy/alchemy.go +++ b/pkg/detectors/alchemy/alchemy.go @@ -66,7 +66,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } else if res.StatusCode == 401 { // The secret is determinately not verified (nothing to do) } else { - s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode) + s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } } else { s1.VerificationError = err diff --git a/pkg/detectors/alchemy/alchemy_test.go b/pkg/detectors/alchemy/alchemy_test.go index 0e14eff6..729c7416 100644 --- a/pkg/detectors/alchemy/alchemy_test.go +++ b/pkg/detectors/alchemy/alchemy_test.go @@ -6,10 +6,11 @@ package alchemy import ( "context" "fmt" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "testing" "time" - "github.com/kylelemons/godebug/pretty" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -131,13 +132,12 @@ func TestAlchemy_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil if (got[i].VerificationError != nil) != tt.wantVerificationErr { - t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationErr) + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError) } - got[i].VerificationError = nil } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Alchemy.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/datadogtoken/datadogtoken.go b/pkg/detectors/datadogtoken/datadogtoken.go index 766053b1..d7fe5215 100644 --- a/pkg/detectors/datadogtoken/datadogtoken.go +++ b/pkg/detectors/datadogtoken/datadogtoken.go @@ -21,7 +21,7 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`) + appPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{40})\b`) apiPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datadog", "dd"}) + `\b([a-zA-Z-0-9]{32})\b`) ) @@ -35,24 +35,28 @@ func (s Scanner) Keywords() []string { func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { dataStr := string(data) - matches := keyPat.FindAllStringSubmatch(dataStr, -1) + appMatches := appPat.FindAllStringSubmatch(dataStr, -1) apiMatches := apiPat.FindAllStringSubmatch(dataStr, -1) - for _, match := range matches { - if len(match) != 2 { + for _, apiMatch := range apiMatches { + if len(apiMatch) != 2 { continue } - resMatch := strings.TrimSpace(match[1]) - for _, apimatch := range apiMatches { - if len(apimatch) != 2 { + resApiMatch := strings.TrimSpace(apiMatch[1]) + appIncluded := false + for _, appMatch := range appMatches { + if len(appMatch) != 2 { continue } - resApiMatch := strings.TrimSpace(apimatch[1]) + resAppMatch := strings.TrimSpace(appMatch[1]) s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_DatadogToken, - Raw: []byte(resMatch), - RawV2: []byte(resMatch + resApiMatch), + Raw: []byte(resAppMatch), + RawV2: []byte(resAppMatch + resApiMatch), + ExtraData: map[string]string{ + "Type": "Application+APIKey", + }, } if verify { @@ -63,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } req.Header.Add("Content-Type", "application/json") req.Header.Add("DD-API-KEY", resApiMatch) - req.Header.Add("DD-APPLICATION-KEY", resMatch) + req.Header.Add("DD-APPLICATION-KEY", resAppMatch) res, err := client.Do(req) if err == nil { defer res.Body.Close() @@ -71,16 +75,49 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = true } else { // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) { continue } } } } - + appIncluded = true results = append(results, s1) } + if !appIncluded { + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_DatadogToken, + Raw: []byte(resApiMatch), + RawV2: []byte(resApiMatch), + ExtraData: map[string]string{ + "Type": "APIKeyOnly", + }, + } + + if verify { + + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datadoghq.com/api/v1/validate", nil) + if err != nil { + continue + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("DD-API-KEY", resApiMatch) + res, err := client.Do(req) + if err == nil { + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { + s1.Verified = true + } else { + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if detectors.IsKnownFalsePositive(resApiMatch, detectors.DefaultFalsePositives, true) { + continue + } + } + } + } + results = append(results, s1) + } } return results, nil diff --git a/pkg/detectors/datadogtoken/datadogtoken_test.go b/pkg/detectors/datadogtoken/datadogtoken_test.go index bc540e26..52c75292 100644 --- a/pkg/detectors/datadogtoken/datadogtoken_test.go +++ b/pkg/detectors/datadogtoken/datadogtoken_test.go @@ -19,11 +19,11 @@ import ( func TestDatadogToken_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } - secret := testSecrets.MustGetField("DATADOGTOKEN_TOKEN") + apiKey := testSecrets.MustGetField("DATADOGTOKEN_TOKEN") appKey := testSecrets.MustGetField("DATADOGTOKEN_APPKEY") inactiveAppKey := testSecrets.MustGetField("DATADOGTOKEN_INACTIVE") @@ -44,13 +44,16 @@ func TestDatadogToken_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, secret)), + data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within datadog %s", appKey, apiKey)), verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_DatadogToken, Verified: true, + ExtraData: map[string]string{ + "Type": "Application+APIKey", + }, }, }, wantErr: false, @@ -60,13 +63,35 @@ func TestDatadogToken_FromChunk(t *testing.T) { s: Scanner{}, args: args{ ctx: context.Background(), - data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, secret)), // the secret would satisfy the regex but not pass validation + data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s within but datadog %s not valid", inactiveAppKey, apiKey)), // the secret would satisfy the regex but not pass validation verify: true, }, want: []detectors.Result{ { DetectorType: detectorspb.DetectorType_DatadogToken, Verified: false, + ExtraData: map[string]string{ + "Type": "Application+APIKey", + }, + }, + }, + wantErr: false, + }, + { + name: "api key found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a datadogtoken secret %s", apiKey)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_DatadogToken, + Verified: true, + ExtraData: map[string]string{ + "Type": "APIKeyOnly", + }, }, }, wantErr: false, @@ -96,6 +121,7 @@ func TestDatadogToken_FromChunk(t *testing.T) { t.Fatalf("no raw secret present: \n %+v", got[i]) } got[i].Raw = nil + got[i].RawV2 = nil } if diff := pretty.Compare(got, tt.want); diff != "" { t.Errorf("DatadogToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff) diff --git a/pkg/detectors/ftp/ftp.go b/pkg/detectors/ftp/ftp.go index 251127f9..5d178a5b 100644 --- a/pkg/detectors/ftp/ftp.go +++ b/pkg/detectors/ftp/ftp.go @@ -2,6 +2,8 @@ package ftp import ( "context" + "errors" + "net/textproto" "net/url" "strings" "time" @@ -14,7 +16,17 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +const ( + // https://datatracker.ietf.org/doc/html/rfc959 + ftpNotLoggedIn = 530 + + defaultVerificationTimeout = 5 * time.Second +) + +type Scanner struct { + // Verification timeout. Defaults to 5 seconds if unset. + verificationTimeout time.Duration +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) @@ -59,48 +71,59 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result rawURL.Path = "" redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1)) - s := detectors.Result{ + r := detectors.Result{ DetectorType: detectorspb.DetectorType_FTP, Raw: []byte(rawURL.String()), Redacted: redact, } if verify { - s.Verified = verifyFTP(ctx, parsedURL) + timeout := s.verificationTimeout + if timeout == 0 { + timeout = defaultVerificationTimeout + } + verificationErr := verifyFTP(timeout, parsedURL) + r.Verified = verificationErr == nil + if !isErrDeterminate(verificationErr) { + r.VerificationError = verificationErr + } } - if !s.Verified { + if !r.Verified { // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable. if strings.HasPrefix(password, "$") { continue } } - if detectors.IsKnownFalsePositive(string(s.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) { + if detectors.IsKnownFalsePositive(string(r.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) { continue } - results = append(results, s) + results = append(results, r) } return results, nil } -func verifyFTP(ctx context.Context, u *url.URL) bool { +func isErrDeterminate(e error) bool { + ftpErr := &textproto.Error{} + return errors.As(e, &ftpErr) && ftpErr.Code == ftpNotLoggedIn +} + +func verifyFTP(timeout time.Duration, u *url.URL) error { host := u.Host if !strings.Contains(host, ":") { host = host + ":21" } - c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second)) + c, err := ftp.Dial(host, ftp.DialWithTimeout(timeout)) if err != nil { - return false + return err } password, _ := u.User.Password() - err = c.Login(u.User.Username(), password) - - return err == nil + return c.Login(u.User.Username(), password) } func (s Scanner) Type() detectorspb.DetectorType { diff --git a/pkg/detectors/ftp/ftp_test.go b/pkg/detectors/ftp/ftp_test.go index c56d87c3..cc72e269 100644 --- a/pkg/detectors/ftp/ftp_test.go +++ b/pkg/detectors/ftp/ftp_test.go @@ -5,9 +5,11 @@ package ftp import ( "context" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "testing" + "time" - "github.com/kylelemons/godebug/pretty" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) @@ -19,11 +21,12 @@ func TestFTP_FromChunk(t *testing.T) { verify bool } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool }{ { name: "bad scheme", @@ -48,7 +51,7 @@ func TestFTP_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_FTP, Verified: true, - Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com", + Redacted: "ftp://dlpuser:********@ftp.dlptest.com", }, }, wantErr: false, @@ -66,11 +69,49 @@ func TestFTP_FromChunk(t *testing.T) { { DetectorType: detectorspb.DetectorType_FTP, Verified: false, - Redacted: "ftp://dlpuser:*******@ftp.dlptest.com", + Redacted: "ftp://dlpuser:********@ftp.dlptest.com", }, }, wantErr: false, }, + { + name: "bad host", + s: Scanner{}, + args: args{ + ctx: context.Background(), + // https://dlptest.com/ftp-test/ + data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com.badhost"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_FTP, + Verified: false, + Redacted: "ftp://dlpuser:********@ftp.dlptest.com.badhost", + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "timeout", + s: Scanner{verificationTimeout: 1 * time.Microsecond}, + args: args{ + ctx: context.Background(), + // https://dlptest.com/ftp-test/ + data: []byte("ftp://dlpuser:rNrKYTX9g7z3RgJRmxWuGHbeu@ftp.dlptest.com"), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_FTP, + Verified: false, + Redacted: "ftp://dlpuser:********@ftp.dlptest.com", + }, + }, + wantErr: false, + wantVerificationErr: true, + }, { name: "blocked FP", s: Scanner{}, @@ -84,8 +125,7 @@ func TestFTP_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -95,9 +135,14 @@ func TestFTP_FromChunk(t *testing.T) { // } for i := range got { got[i].Raw = nil + + if (got[i].VerificationError != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { - t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + opts := cmpopts.IgnoreFields(detectors.Result{}, "VerificationError") + if diff := cmp.Diff(got, tt.want, opts); diff != "" { + t.Errorf("FTP.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) } diff --git a/pkg/detectors/github_oauth2/github_oauth2.go b/pkg/detectors/github_oauth2/github_oauth2.go new file mode 100644 index 00000000..6cd6f27e --- /dev/null +++ b/pkg/detectors/github_oauth2/github_oauth2.go @@ -0,0 +1,81 @@ +package github_oauth2 + +import ( + "context" + "regexp" + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" + "golang.org/x/oauth2/clientcredentials" + "golang.org/x/oauth2/github" +) + +type Scanner struct{ detectors.EndpointSetter } + +// Ensure the Scanner satisfies the interfaces at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + // Oauth2 client ID and secret + oauth2ClientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{20})\b`) + oauth2ClientSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github"}) + `\b([a-f0-9]{40})\b`) +) + +const ( + githubBadVerificationCodeError = "bad_verification_code" +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"github"} +} + +// FromData will find and optionally verify GitHub secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + // Oauth2 client ID and secret + oauth2ClientIDMatches := oauth2ClientIDPat.FindAllStringSubmatch(dataStr, -1) + oauth2ClientSecretMatches := oauth2ClientSecretPat.FindAllStringSubmatch(dataStr, -1) + + for _, idMatch := range oauth2ClientIDMatches { + if len(idMatch) != 2 { + continue + } + for _, secretMatch := range oauth2ClientSecretMatches { + if len(secretMatch) != 2 { + continue + } + + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_GitHubOauth2, + Raw: []byte(idMatch[1]), + RawV2: []byte(idMatch[1] + secretMatch[1]), + } + + config := &clientcredentials.Config{ + ClientID: idMatch[1], + ClientSecret: secretMatch[1], + TokenURL: github.Endpoint.TokenURL, + } + _, err := config.Token(ctx) + if err != nil && strings.Contains(err.Error(), githubBadVerificationCodeError) { + s1.Verified = true + } + + if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) { + continue + } + + results = append(results, s1) + } + } + + return +} + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_GitHubOauth2 +} diff --git a/pkg/detectors/gitlab/gitlab.go b/pkg/detectors/gitlab/gitlab.go index 9c0c2891..b1b87266 100644 --- a/pkg/detectors/gitlab/gitlab.go +++ b/pkg/detectors/gitlab/gitlab.go @@ -13,7 +13,11 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{ detectors.EndpointSetter } +type Scanner struct { + client *http.Client + + detectors.EndpointSetter +} // Ensure the Scanner satisfies the interfaces at compile time. var _ detectors.Detector = (*Scanner)(nil) @@ -24,7 +28,8 @@ func (Scanner) Version() int { return 1 } func (Scanner) DefaultEndpoint() string { return "https://gitlab.com" } var ( - keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`) + defaultClient = common.SaneHttpClient() + keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -60,7 +65,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // one of these scopes has access to, so we just check an example endpoint for each scope. If any // of them contain data, we know we have a valid key, but if they all fail, we don't - client := common.SaneHttpClient() + client := s.client + if client == nil { + client = defaultClient + } for _, baseURL := range s.Endpoints(s.DefaultEndpoint()) { // test `read_user` scope req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil) @@ -75,9 +83,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result // 200 means good key and has `read_user` scope // 403 means good key but not the right scope // 401 is bad key - if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden { + switch res.StatusCode { + case http.StatusOK: secret.Verified = true + case http.StatusForbidden: + // Good key but not the right scope + secret.Verified = true + case http.StatusUnauthorized: + // Nothing to do; zero values are the ones we want + default: + secret.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } + } else { + secret.VerificationError = err } } } diff --git a/pkg/detectors/gitlab/gitlab_test.go b/pkg/detectors/gitlab/gitlab_test.go index 0bf1a9b8..0783fdc1 100644 --- a/pkg/detectors/gitlab/gitlab_test.go +++ b/pkg/detectors/gitlab/gitlab_test.go @@ -6,11 +6,11 @@ package gitlab import ( "context" "fmt" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "testing" "time" - "github.com/kylelemons/godebug/pretty" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -31,11 +31,12 @@ func TestGitlab_FromChunk(t *testing.T) { verify bool } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool }{ { name: "found", @@ -85,6 +86,56 @@ func TestGitlab_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, would be verified but for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Gitlab, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found and valid but unexpected api response", + s: Scanner{client: common.ConstantResponseHttpClient(500, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Gitlab, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, good key but wrong scope", + s: Scanner{client: common.ConstantResponseHttpClient(403, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a gitlab super secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Gitlab, + Verified: true, + }, + }, + wantErr: false, + }, { name: "not found", s: Scanner{}, @@ -99,8 +150,7 @@ func TestGitlab_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("Gitlab.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -109,9 +159,12 @@ func TestGitlab_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatal("no raw secret present") } - got[i].Raw = nil + if (got[i].VerificationError != nil) != tt.wantVerificationErr { + t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError") + if diff := cmp.Diff(got, tt.want, opts); diff != "" { t.Errorf("Gitlab.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/huggingface/huggingface.go b/pkg/detectors/huggingface/huggingface.go new file mode 100644 index 00000000..694da79f --- /dev/null +++ b/pkg/detectors/huggingface/huggingface.go @@ -0,0 +1,91 @@ +package huggingface + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. + keyPat = regexp.MustCompile(`\bhf_[a-zA-Z]{34}\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface" +} + +// FromData will find and optionally verify Huggingface secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + matches := keyPat.FindAllStringSubmatch(dataStr, -1) + + for _, match := range matches { + if len(match) != 1 { + continue + } + resMatch := strings.TrimSpace(match[0]) + + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_HuggingFace, + Raw: []byte(resMatch), + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/whoami-v2", nil) + if err != nil { + continue + } + + req.Header.Set("Authorization", "Bearer "+resMatch) + + res, err := client.Do(req) + if err == nil { + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 { + s1.Verified = true + } else if res.StatusCode == 401 { + // The secret is determinately not verified (nothing to do) + } else { + s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) + } + } else { + s1.VerificationError = err + } + } + + // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue + } + + results = append(results, s1) + } + + return results, nil +} + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_HuggingFace +} diff --git a/pkg/detectors/huggingface/huggingface_test.go b/pkg/detectors/huggingface/huggingface_test.go new file mode 100644 index 00000000..d2eee52a --- /dev/null +++ b/pkg/detectors/huggingface/huggingface_test.go @@ -0,0 +1,160 @@ +//go:build detectors +// +build detectors + +package huggingface + +import ( + "context" + "fmt" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "testing" + "time" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestHuggingface_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + secret := testSecrets.MustGetField("HUGGINGFACE") + inactiveSecret := testSecrets.MustGetField("HUGGINGFACE_INACTIVE") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_HuggingFace, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a huggingface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_HuggingFace, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_HuggingFace, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_HuggingFace, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Huggingface.FromData() error = %v, wantErr %v", err, tt.wantErr) + return + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + if (got[i].VerificationError != nil) != tt.wantVerificationErr { + t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Huggingface.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/detectors/ldap/ldap.go b/pkg/detectors/ldap/ldap.go index fb347164..a706f63f 100644 --- a/pkg/detectors/ldap/ldap.go +++ b/pkg/detectors/ldap/ldap.go @@ -3,6 +3,8 @@ package ldap import ( "context" "crypto/tls" + "fmt" + "net" "net/url" "strings" "time" @@ -65,7 +67,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - s1.Verified = verifyLDAP(ctx, username[1], password[1], ldapURL) + verificationErr := verifyLDAP(username[1], password[1], ldapURL) + s1.Verified = verificationErr == nil + if !isErrDeterminate(verificationErr) { + s1.VerificationError = verificationErr + } } results = append(results, s1) @@ -91,7 +97,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - s1.Verified = verifyLDAP(ctx, username, password, ldapURL) + verificationError := verifyLDAP(username, password, ldapURL) + + s1.Verified = verificationError == nil + if !isErrDeterminate(verificationError) { + s1.VerificationError = verificationError + } } results = append(results, s1) @@ -100,7 +111,19 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result return results, nil } -func verifyLDAP(ctx context.Context, username, password string, ldapURL *url.URL) bool { +func isErrDeterminate(err error) bool { + switch e := err.(type) { + case *ldap.Error: + switch e.Err.(type) { + case *net.OpError: + return false + } + } + + return true +} + +func verifyLDAP(username, password string, ldapURL *url.URL) error { // Tests with non-TLS, TLS, and STARTTLS ldap.DefaultTimeout = 5 * time.Second @@ -111,38 +134,35 @@ func verifyLDAP(ctx context.Context, username, password string, ldapURL *url.URL case "ldap": // Non-TLS dial l, err := ldap.DialURL(uri) - if err == nil { - defer l.Close() - // Non-TLS verify - err = l.Bind(username, password) - if err == nil { - return true - } - - // STARTTLS - err = l.StartTLS(&tls.Config{InsecureSkipVerify: true}) - if err == nil { - // STARTTLS verify - err = l.Bind(username, password) - if err == nil { - return true - } - } + if err != nil { + return err } + defer l.Close() + // Non-TLS verify + err = l.Bind(username, password) + if err == nil { + return nil + } + + // STARTTLS + err = l.StartTLS(&tls.Config{InsecureSkipVerify: true}) + if err != nil { + return err + } + // STARTTLS verify + return l.Bind(username, password) case "ldaps": // TLS dial l, err := ldap.DialTLS("tcp", uri, &tls.Config{InsecureSkipVerify: true}) - if err == nil { - defer l.Close() - // TLS verify - err = l.Bind(username, password) - if err == nil { - return true - } + if err != nil { + return err } + defer l.Close() + // TLS verify + return l.Bind(username, password) } - return false + return fmt.Errorf("unknown ldap scheme %q", ldapURL.Scheme) } func (s Scanner) Type() detectorspb.DetectorType { diff --git a/pkg/detectors/ldap/ldap_integration_test.go b/pkg/detectors/ldap/ldap_integration_test.go index 3c7a2779..8b85d382 100644 --- a/pkg/detectors/ldap/ldap_integration_test.go +++ b/pkg/detectors/ldap/ldap_integration_test.go @@ -7,13 +7,14 @@ import ( "bytes" "context" "errors" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "os" "os/exec" "strings" "testing" "time" - "github.com/kylelemons/godebug/pretty" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" @@ -60,11 +61,12 @@ func TestLdap_Integration_FromChunk(t *testing.T) { verify bool } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool }{ { name: "found with URI and separate user+password usage, verified", @@ -152,6 +154,26 @@ func TestLdap_Integration_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "inaccessible host", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(` + ldap://badhost:1389 + binddn="cn=admin,dc=example,dc=org" + pass="P@55w0rd"`), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_LDAP, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, { name: "not found", s: Scanner{}, @@ -176,9 +198,12 @@ func TestLdap_Integration_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + if (got[i].VerificationError != nil) != tt.wantVerificationErr { + t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { t.Errorf("Ldap.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) @@ -205,7 +230,7 @@ func startOpenLDAP() error { return nil case <-time.After(30 * time.Second): stopOpenLDAP() - return errors.New("timeout waiting for postgres database to be ready") + return errors.New("timeout waiting for ldap service to be ready") } } diff --git a/pkg/detectors/opsgenie/opsgenie.go b/pkg/detectors/opsgenie/opsgenie.go index f8a579e9..e84f5bda 100644 --- a/pkg/detectors/opsgenie/opsgenie.go +++ b/pkg/detectors/opsgenie/opsgenie.go @@ -53,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result } if verify { - req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil) + req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/alerts", nil) if err != nil { continue } diff --git a/pkg/detectors/paypaloauth/paypaloauth.go b/pkg/detectors/paypaloauth/paypaloauth.go index 81694801..4466f03d 100644 --- a/pkg/detectors/paypaloauth/paypaloauth.go +++ b/pkg/detectors/paypaloauth/paypaloauth.go @@ -23,8 +23,8 @@ var ( client = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. - idPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{7}-[A-Za-z0-9_\.]{72})\b`) - keyPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{69}-[A-Za-z0-9_\.]{10})\b`) + idPat = regexp.MustCompile(`\b([A-Za-z0-9_\.]{7}-[A-Za-z0-9_\.]{72}|[A-Za-z0-9_\.]{5}-[A-Za-z0-9_\.]{38})\b`) + keyPat = regexp.MustCompile(`\b([A-Za-z0-9_\.\-]{44,80})\b`) ) // Keywords are used for efficiently pre-filtering chunks. @@ -40,24 +40,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result matches := keyPat.FindAllStringSubmatch(dataStr, -1) idmatches := idPat.FindAllStringSubmatch(dataStr, -1) - for _, match := range matches { - if len(match) != 2 { + for _, idMatch := range idmatches { + if len(idMatch) != 2 { continue } - resMatch := strings.TrimSpace(match[1]) - for _, idMatch := range idmatches { - if len(idMatch) != 2 { + resIDMatch := strings.TrimSpace(idMatch[1]) + for _, secretMatch := range matches { + if len(secretMatch) != 2 { continue } - residMatch := strings.TrimSpace(idMatch[1]) + resSecretMatch := strings.TrimSpace(secretMatch[1]) s1 := detectors.Result{ DetectorType: detectorspb.DetectorType_PaypalOauth, - Raw: []byte(resMatch), + Raw: []byte(resSecretMatch), } if verify { - data := fmt.Sprintf("%s:%s", residMatch, resMatch) + data := fmt.Sprintf("%s:%s", resIDMatch, resSecretMatch) encoded := b64.StdEncoding.EncodeToString([]byte(data)) payload := strings.NewReader("grant_type=client_credentials") req, err := http.NewRequestWithContext(ctx, "POST", "https://api-m.sandbox.paypal.com/v1/oauth2/token", payload) @@ -75,7 +75,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result s1.Verified = true } else { // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + if detectors.IsKnownFalsePositive(resIDMatch, detectors.DefaultFalsePositives, true) { continue } } diff --git a/pkg/detectors/paypaloauth/paypaloauth_test.go b/pkg/detectors/paypaloauth/paypaloauth_test.go index 55ed1e08..b94ff993 100644 --- a/pkg/detectors/paypaloauth/paypaloauth_test.go +++ b/pkg/detectors/paypaloauth/paypaloauth_test.go @@ -19,7 +19,7 @@ import ( func TestPaypalOauth_FromChunk(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3") + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") if err != nil { t.Fatalf("could not get test secrets from GCP: %s", err) } @@ -27,6 +27,9 @@ func TestPaypalOauth_FromChunk(t *testing.T) { inactiveSecret := testSecrets.MustGetField("PAYPALOAUTH_SECRET_INACTIVE") id := testSecrets.MustGetField("PAYPALOAUTH_CLIENTID") + newId := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_CLIENTID") + newSecret := testSecrets.MustGetField("PAYPALOAUTH_NEW_INACTIVE_SECRET") + type args struct { ctx context.Context data []byte @@ -52,6 +55,10 @@ func TestPaypalOauth_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_PaypalOauth, Verified: true, }, + { + DetectorType: detectorspb.DetectorType_PaypalOauth, + Verified: false, + }, }, wantErr: false, }, @@ -68,6 +75,30 @@ func TestPaypalOauth_FromChunk(t *testing.T) { DetectorType: detectorspb.DetectorType_PaypalOauth, Verified: false, }, + { + DetectorType: detectorspb.DetectorType_PaypalOauth, + Verified: false, + }, + }, + wantErr: false, + }, + { + name: "new format, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a paypaloauth secret %s within %s but not valid", newSecret, newId)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_PaypalOauth, + Verified: false, + }, + { + DetectorType: detectorspb.DetectorType_PaypalOauth, + Verified: false, + }, }, wantErr: false, }, diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go index 90a55fae..97c35012 100644 --- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go +++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey.go @@ -2,6 +2,7 @@ package pubnubpublishkey import ( "context" + "fmt" "net/http" "strings" @@ -12,13 +13,15 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" ) -type Scanner struct{} +type Scanner struct { + client *http.Client +} // Ensure the Scanner satisfies the interface at compile time. var _ detectors.Detector = (*Scanner)(nil) var ( - client = common.SaneHttpClient() + defaultClient = common.SaneHttpClient() // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. pubPat = regexp.MustCompile(`\b(pub-c-[0-9a-z]{8}-[0-9a-z]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`) @@ -61,19 +64,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result if err != nil { continue } + client := s.client + if client == nil { + client = defaultClient + } res, err := client.Do(req) if err == nil { defer res.Body.Close() if res.StatusCode >= 200 && res.StatusCode < 300 { s1.Verified = true + } else if res.StatusCode == 400 || res.StatusCode == 403 { + // 403 is suggested by the API docs (https://www.pubnub.com/docs/sdks/rest-api/send-signal-to-channel) + // 400 is what actually seems to be coming back for invalid credentials + // There's nothing to do here, because zero values of Result are what we want } else { - // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key. - if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { - continue - } + s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode) } + } else { + s1.VerificationError = err } } + // This function will check false positives for common test words, but also it will make sure the key + // appears 'random' enough to be a real key. + if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) { + continue + } results = append(results, s1) } diff --git a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go index 174f8421..312c5eb9 100644 --- a/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go +++ b/pkg/detectors/pubnubpublishkey/pubnubpublishkey_test.go @@ -6,10 +6,11 @@ package pubnubpublishkey import ( "context" "fmt" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "testing" "time" - "github.com/kylelemons/godebug/pretty" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -33,11 +34,12 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) { verify bool } tests := []struct { - name string - s Scanner - args args - want []detectors.Result - wantErr bool + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool }{ { name: "found, verified", @@ -71,6 +73,40 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) { }, wantErr: false, }, + { + name: "found, would be verified but for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_PubNubPublishKey, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, valid but unexpected api response", + s: Scanner{client: common.ConstantResponseHttpClient(500, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a pubnub secret %s within pubnub %s", pub, sub)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_PubNubPublishKey, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, { name: "not found", s: Scanner{}, @@ -85,8 +121,7 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := Scanner{} - got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) if (err != nil) != tt.wantErr { t.Errorf("PubNubPublishKey.FromData() error = %v, wantErr %v", err, tt.wantErr) return @@ -95,9 +130,12 @@ func TestPubNubPublishKey_FromChunk(t *testing.T) { if len(got[i].Raw) == 0 { t.Fatalf("no raw secret present: \n %+v", got[i]) } - got[i].Raw = nil + if (got[i].VerificationError != nil) != tt.wantVerificationErr { + t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError) + } } - if diff := pretty.Compare(got, tt.want); diff != "" { + opts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError") + if diff := cmp.Diff(got, tt.want, opts); diff != "" { t.Errorf("PubNubPublishKey.FromData() %s diff: (-got +want)\n%s", tt.name, diff) } }) diff --git a/pkg/detectors/salesforce/salesforce.go b/pkg/detectors/salesforce/salesforce.go new file mode 100644 index 00000000..872e3c83 --- /dev/null +++ b/pkg/detectors/salesforce/salesforce.go @@ -0,0 +1,113 @@ +package salesforce + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +const ( + currentVersion = "58.0" // current Salesforce version +) + +type Scanner struct { + client *http.Client +} + +// Ensure the Scanner satisfies the interface at compile time. +var _ detectors.Detector = (*Scanner)(nil) + +var ( + defaultClient = common.SaneHttpClient() + // Make sure that your group is surrounded in boundary characters such as below to reduce false positives. + accessTokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9]{13}![a-zA-Z0-9_.]{96}\b`) + instancePat = regexp.MustCompile(`\bhttps://[0-9a-zA-Z-\.]{1,100}\.my\.salesforce\.com\b`) +) + +// Keywords are used for efficiently pre-filtering chunks. +// Use identifiers in the secret preferably, or the provider name. +func (s Scanner) Keywords() []string { + return []string{"salesforce"} +} + +// FromData will find and optionally verify Salesforce secrets in a given set of bytes. +func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) { + dataStr := string(data) + + instanceMatches := instancePat.FindAllStringSubmatch(dataStr, -1) + tokenMatches := accessTokenPat.FindAllStringSubmatch(dataStr, -1) + + fmt.Printf("instanceMatches: %v\n", instanceMatches) + + for _, instance := range instanceMatches { + if len(instance) != 1 { + continue + } + + instanceMatch := strings.TrimSpace(instance[0]) + + for _, token := range tokenMatches { + if len(token) != 1 { + continue + } + + tokenMatch := strings.TrimSpace(token[0]) + + s1 := detectors.Result{ + DetectorType: detectorspb.DetectorType_Salesforce, + Raw: []byte(tokenMatch), + } + + if verify { + client := s.client + if client == nil { + client = defaultClient + } + + req, err := http.NewRequestWithContext(ctx, "GET", instanceMatch+"/services/data/v"+currentVersion+"/query?q=SELECT+name+from+Account", nil) + if err != nil { + continue + } + req.Header.Set("Authorization", "Bearer "+tokenMatch) + + res, err := client.Do(req) + + if err != nil { + // End execution, append Detector Result if request fails to prevent panic on response body checks + s1.VerificationError = err + results = append(results, s1) + continue + } + + verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "records") + + defer res.Body.Close() + if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse { + s1.Verified = true + } else if res.StatusCode >= 200 && res.StatusCode < 300 && !verifiedBodyResponse { + s1.Verified = false + } else if res.StatusCode == 401 { + s1.Verified = false + } else { + s1.VerificationError = fmt.Errorf("request to %v returned status %d with error %+v", res.Request.URL, res.StatusCode, err) + } + + } + + results = append(results, s1) + + } + } + + return results, nil +} + +func (s Scanner) Type() detectorspb.DetectorType { + return detectorspb.DetectorType_Salesforce +} diff --git a/pkg/detectors/salesforce/salesforce_test.go b/pkg/detectors/salesforce/salesforce_test.go new file mode 100644 index 00000000..7604c8a8 --- /dev/null +++ b/pkg/detectors/salesforce/salesforce_test.go @@ -0,0 +1,167 @@ +//go:build detectors +// +build detectors + +package salesforce + +import ( + "context" + "fmt" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "testing" + "time" + + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" +) + +func TestSalesforce_FromChunk(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5") + if err != nil { + t.Fatalf("could not get test secrets from GCP: %s", err) + } + instance := testSecrets.MustGetField("SALESFORCE_INSTANCE") + // Salesforce secrets are not valid for long, so we may need to regenerate them. + // Steps to regenerate: + // 1. Install Salesforce CLI + // 2. Authenticate with `sfdx org web login`. This will open a browser window. Use the credentials in the detection test accounts 1Pass vault. + // 3. Run `sfdx org display --targetusername zubairkhan@trufflesec.com`to obtain new token. + // Source: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm + token := testSecrets.MustGetField("SALESFORCE_TOKEN") + inactiveToken := testSecrets.MustGetField("SALESFORCE_INACTIVE_TOKEN") + + type args struct { + ctx context.Context + data []byte + verify bool + } + tests := []struct { + name string + s Scanner + args args + want []detectors.Result + wantErr bool + wantVerificationErr bool + }{ + { + name: "found, verified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Salesforce, + Verified: true, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, unverified", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a salesforce secret %s within but not valid for %s", inactiveToken, instance)), // the secret would satisfy the regex but not pass validation + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Salesforce, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "not found", + s: Scanner{}, + args: args{ + ctx: context.Background(), + data: []byte("You cannot find the secret within"), + verify: true, + }, + want: nil, + wantErr: false, + wantVerificationErr: false, + }, + { + name: "found, would be verified if not for timeout", + s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Salesforce, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + { + name: "found, verified but unexpected api surface", + s: Scanner{client: common.ConstantResponseHttpClient(404, "")}, + args: args{ + ctx: context.Background(), + data: []byte(fmt.Sprintf("You can find a salesforce secret within %s for %s", token, instance)), + verify: true, + }, + want: []detectors.Result{ + { + DetectorType: detectorspb.DetectorType_Salesforce, + Verified: false, + }, + }, + wantErr: false, + wantVerificationErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Salesforce.FromData() error = %v, wantErr %v", err, tt.wantErr) + } + for i := range got { + if len(got[i].Raw) == 0 { + t.Fatalf("no raw secret present: \n %+v", got[i]) + } + + if (got[i].VerificationError != nil) != tt.wantVerificationErr { + t.Fatalf(" wantVerificationError = %v, verification error = %v,", tt.wantVerificationErr, got[i].VerificationError) + } + } + ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError") + if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" { + t.Errorf("Salesforce.FromData() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +} + +func BenchmarkFromData(benchmark *testing.B) { + ctx := context.Background() + s := Scanner{} + for name, data := range detectors.MustGetBenchmarkData() { + benchmark.Run(name, func(b *testing.B) { + for n := 0; n < b.N; n++ { + _, err := s.FromData(ctx, false, data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.go index fb665eda..e1113b8e 100644 --- a/pkg/engine/circleci.go +++ b/pkg/engine/circleci.go @@ -1,16 +1,14 @@ package engine import ( - "fmt" "runtime" - "github.com/go-errors/errors" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci" ) @@ -29,23 +27,17 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error { return err } - circleSource := circleci.Source{} - ctx = context.WithValues(ctx, - "source_type", circleSource.Type().String(), - "source_name", "Circle CI", - ) - err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU()) + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + circleSource := circleci.Source{} + if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil { + return nil, err + } + return &circleSource, nil + }) if err != nil { - return errors.WrapPrefix(err, "failed to init Circle CI source", 0) + return err } - - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - err := circleSource.Chunks(ctx, e.ChunksChan()) - if err != nil { - return fmt.Errorf("error scanning CircleCI: %w", err) - } - return nil - }) - return nil + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go index 6bd24b41..89a8f076 100644 --- a/pkg/engine/defaults.go +++ b/pkg/engine/defaults.go @@ -556,6 +556,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/satismeterwritekey" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/saucelabs" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalewaykey" + "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github_oauth2" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scalr" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scrapeowl" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/scraperapi" @@ -1530,6 +1531,7 @@ func DefaultDetectors() []detectors.Detector { dockerhub.Scanner{}, couchbase.Scanner{}, envoyapikey.Scanner{}, + github_oauth2.Scanner{}, } } diff --git a/pkg/engine/docker.go b/pkg/engine/docker.go index 5cc18f6c..97840f98 100644 --- a/pkg/engine/docker.go +++ b/pkg/engine/docker.go @@ -1,38 +1,28 @@ package engine import ( - "fmt" "runtime" - "github.com/go-errors/errors" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker" ) // ScanDocker scans a given docker connection. func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error { - dockerSource := docker.Source{} - - ctx = context.WithValues(ctx, - "source_type", dockerSource.Type().String(), - "source_name", "docker", - ) - err := dockerSource.Init(ctx, "trufflehog - docker", 0, int64(sourcespb.SourceType_SOURCE_TYPE_DOCKER), true, conn, runtime.NumCPU()) + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + dockerSource := docker.Source{} + if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil { + return nil, err + } + return &dockerSource, nil + }) if err != nil { - return errors.WrapPrefix(err, "could not init docker source", 0) + return err } - - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - err := dockerSource.Chunks(ctx, e.ChunksChan()) - if err != nil { - return fmt.Errorf("error scanning docker image: %w", err) - } - return nil - }) - return nil + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 68c912d1..3489d980 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -12,7 +12,6 @@ import ( ahocorasick "github.com/BobuSumisu/aho-corasick" lru "github.com/hashicorp/golang-lru" - "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -34,6 +33,9 @@ type Metrics struct { VerifiedSecretsFound uint64 UnverifiedSecretsFound uint64 AvgDetectorTime map[string]time.Duration + + scanStartTime time.Time + ScanDuration time.Duration } // runtimeMetrics for the scan engine for internal use by the engine. @@ -44,18 +46,16 @@ type runtimeMetrics struct { } // Printer is used to format found results and output them to the user. Ex JSON, plain text, etc. +// Please note printer implementations SHOULD BE thread safe. type Printer interface { Print(ctx context.Context, r *detectors.ResultWithMetadata) error } type Engine struct { + // CLI flags. concurrency uint8 - chunks chan *sources.Chunk - results chan detectors.ResultWithMetadata decoders []decoders.Decoder detectors map[bool][]detectors.Detector - sourcesWg *errgroup.Group - workersWg sync.WaitGroup // filterUnverified is used to reduce the number of unverified results. // If there are multiple unverified results for the same chunk for the same detector, // only the first one will be kept. @@ -67,13 +67,16 @@ type Engine struct { // matching given a set of words (keywords from the rules in the config) prefilter ahocorasick.Trie + // Engine synchronization primitives. + sourceManager *sources.SourceManager + results chan detectors.ResultWithMetadata detectableChunksChan chan detectableChunk + workersWg sync.WaitGroup wgDetectorWorkers sync.WaitGroup WgNotifier sync.WaitGroup - // Runtime metrics. + // Runtime information. metrics runtimeMetrics - // numFoundResults is used to keep track of the number of results found. numFoundResults uint32 @@ -199,6 +202,8 @@ func (e *Engine) GetMetrics() Metrics { result.AvgDetectorTime[detectorName] = avgDuration } + result.ScanDuration = e.metrics.getScanDuration() + return result } @@ -220,6 +225,16 @@ func (e *Engine) GetDetectorsMetrics() map[string]time.Duration { return result } +// getScanDuration returns the duration of the scan. +// If the scan is still running, it returns the time since the scan started. +func (m *Metrics) getScanDuration() time.Duration { + if m.ScanDuration == 0 { + return time.Since(m.scanStartTime) + } + + return m.ScanDuration +} + // DetectorAvgTime returns the average time taken by each detector. func (e *Engine) DetectorAvgTime() map[string][]time.Duration { logger := context.Background().Logger() @@ -256,12 +271,11 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) { } e := &Engine{ - chunks: make(chan *sources.Chunk, defaultChannelBuffer), detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer), results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer), - sourcesWg: &errgroup.Group{}, dedupeCache: cache, printer: new(output.PlainPrinter), // default printer + metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}}, } for _, option := range options { @@ -276,8 +290,11 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) { } ctx.Logger().V(3).Info("engine started", "workers", e.concurrency) - // Limit number of concurrent goroutines dedicated to chunking a source. - e.sourcesWg.SetLimit(int(e.concurrency)) + // Create SourceManager. + e.sourceManager = sources.NewManager( + sources.WithConcurrentSources(int(e.concurrency)), + sources.WithConcurrentUnits(int(e.concurrency)), + ) if len(e.decoders) == 0 { e.decoders = decoders.DefaultDecoders() @@ -376,9 +393,7 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) { func (e *Engine) Finish(ctx context.Context) error { defer common.RecoverWithExit(ctx) // Wait for the sources to finish putting chunks onto the chunks channel. - err := e.sourcesWg.Wait() - - close(e.chunks) // Source workers are done. + err := e.sourceManager.Wait() e.workersWg.Wait() // Wait for the workers to finish scanning chunks. close(e.detectableChunksChan) @@ -387,17 +402,26 @@ func (e *Engine) Finish(ctx context.Context) error { close(e.results) // Detector workers are done, close the results channel and call it a day. e.WgNotifier.Wait() // Wait for the notifier workers to finish notifying results. + e.metrics.ScanDuration = time.Since(e.metrics.scanStartTime) + return err } -func (e *Engine) ChunksChan() chan *sources.Chunk { - return e.chunks +func (e *Engine) ChunksChan() <-chan *sources.Chunk { + return e.sourceManager.Chunks() } func (e *Engine) ResultsChan() chan detectors.ResultWithMetadata { return e.results } +// ScanChunk injects a chunk into the output stream of chunks to be scanned. +// This method should rarely be used. TODO: Remove when dependencies no longer +// rely on this functionality. +func (e *Engine) ScanChunk(chunk *sources.Chunk) { + e.sourceManager.ScanChunk(chunk) +} + // detectableChunk is a decoded chunk that is ready to be scanned by its detector. type detectableChunk struct { detector detectors.Detector @@ -409,7 +433,7 @@ type detectableChunk struct { func (e *Engine) detectorWorker(ctx context.Context) { var wgDetect sync.WaitGroup - for originalChunk := range e.chunks { + for originalChunk := range e.ChunksChan() { for chunk := range sources.Chunker(originalChunk) { matchedKeywords := make(map[string]struct{}) atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data))) @@ -578,17 +602,20 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool { // FragmentLineOffset sets the line number for a provided source chunk with a given detector result. func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) { - lines := bytes.Split(chunk.Data, []byte("\n")) - for i, line := range lines { - if bytes.Contains(line, result.Raw) { - // if the line contains the ignore tag, we should ignore the result - if bytes.Contains(line, []byte(ignoreTag)) { - return int64(i), true - } - return int64(i), false - } + before, after, found := bytes.Cut(chunk.Data, result.Raw) + if !found { + return 0, false } - return 0, false + lineNumber := int64(bytes.Count(before, []byte("\n"))) + // If the line contains the ignore tag, we should ignore the result. + endLine := bytes.Index(after, []byte("\n")) + if endLine == -1 { + endLine = len(after) + } + if bytes.Contains(after[:endLine], []byte(ignoreTag)) { + return lineNumber, true + } + return lineNumber, false } // FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go index e6a0a5b8..18668ddc 100644 --- a/pkg/engine/engine_test.go +++ b/pkg/engine/engine_test.go @@ -1,6 +1,7 @@ package engine import ( + "fmt" "testing" "github.com/trufflesecurity/trufflehog/v3/pkg/decoders" @@ -50,6 +51,39 @@ func TestFragmentLineOffset(t *testing.T) { expectedLine: 4, ignore: false, }, + { + name: "match on consecutive lines", + chunk: &sources.Chunk{ + Data: []byte("line1\nline2\ntrufflehog:ignore\nline4\nsecret\nhere\nline6"), + }, + result: &detectors.Result{ + Raw: []byte("secret\nhere"), + }, + expectedLine: 4, + ignore: false, + }, + { + name: "ignore on last consecutive lines", + chunk: &sources.Chunk{ + Data: []byte("line1\nline2\nline3\nsecret\nhere // trufflehog:ignore\nline5"), + }, + result: &detectors.Result{ + Raw: []byte("secret\nhere"), + }, + expectedLine: 3, + ignore: true, + }, + { + name: "ignore on last line", + chunk: &sources.Chunk{ + Data: []byte("line1\nline2\nline3\nsecret here // trufflehog:ignore"), + }, + result: &detectors.Result{ + Raw: []byte("secret here"), + }, + expectedLine: 3, + ignore: true, + }, } for _, tt := range tests { @@ -65,6 +99,43 @@ func TestFragmentLineOffset(t *testing.T) { } } +func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) { + data := make([]byte, 0, 4096) + needle := []byte("needle") + for i := 0; i < totalLines; i++ { + if i != needleLine { + data = append(data, []byte(fmt.Sprintf("line%d\n", i))...) + continue + } + data = append(data, needle...) + data = append(data, '\n') + } + chunk := &sources.Chunk{Data: data} + result := &detectors.Result{Raw: needle} + return chunk, result +} + +func BenchmarkFragmentLineOffsetStart(b *testing.B) { + chunk, result := setupFragmentLineOffsetBench(512, 2) + for i := 0; i < b.N; i++ { + _, _ = FragmentLineOffset(chunk, result) + } +} + +func BenchmarkFragmentLineOffsetMiddle(b *testing.B) { + chunk, result := setupFragmentLineOffsetBench(512, 256) + for i := 0; i < b.N; i++ { + _, _ = FragmentLineOffset(chunk, result) + } +} + +func BenchmarkFragmentLineOffsetEnd(b *testing.B) { + chunk, result := setupFragmentLineOffsetBench(512, 510) + for i := 0; i < b.N; i++ { + _, _ = FragmentLineOffset(chunk, result) + } +} + // Test to make sure that DefaultDecoders always returns the UTF8 decoder first. // Technically a decoder test but we want this to run and fail in CI func TestDefaultDecoders(t *testing.T) { diff --git a/pkg/engine/filesystem.go b/pkg/engine/filesystem.go index 98293e01..9e26c75b 100644 --- a/pkg/engine/filesystem.go +++ b/pkg/engine/filesystem.go @@ -1,14 +1,11 @@ package engine import ( - "fmt" "runtime" - "github.com/go-errors/errors" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -27,39 +24,18 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) return err } - fileSystemSource := filesystem.Source{} - fileSystemSource.WithFilter(c.Filter) - - // TODO: This is how it would be used; everything after these comments would be removed. - // handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", fileSystemSource.Type(), - // func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { - // fileSystemSource := filesystem.Source{} - // fileSystemSource.WithFilter(c.Filter) - // if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil { - // return nil, err - // } - // return &fileSystemSource, nil - // }) - // if err != nil { - // return err - // } - // return e.sourceManager.ScheduleRun(ctx, handle) - - ctx = context.WithValues(ctx, - "source_type", fileSystemSource.Type().String(), - "source_name", "filesystem", - ) - err = fileSystemSource.Init(ctx, "trufflehog - filesystem", 0, int64(sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM), true, &conn, runtime.NumCPU()) + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + fileSystemSource := filesystem.Source{} + fileSystemSource.WithFilter(c.Filter) + if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil { + return nil, err + } + return &fileSystemSource, nil + }) if err != nil { - return errors.WrapPrefix(err, "could not init filesystem source", 0) + return err } - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - err := fileSystemSource.Chunks(ctx, e.ChunksChan()) - if err != nil { - return fmt.Errorf("error scanning filesystem: %w", err) - } - return nil - }) - return nil + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } diff --git a/pkg/engine/gcs.go b/pkg/engine/gcs.go index 1ae24f90..32d977f7 100644 --- a/pkg/engine/gcs.go +++ b/pkg/engine/gcs.go @@ -6,7 +6,6 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -45,23 +44,19 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error { return fmt.Errorf("failed to marshal GCS connection: %w", err) } - source := gcs.Source{} - ctx = context.WithValues(ctx, - "source_type", source.Type().String(), - "source_name", "gcs", - ) - if err = source.Init(ctx, "trufflehog - GCS", 0, 0, true, &conn, int(c.Concurrency)); err != nil { - return fmt.Errorf("failed to initialize GCS source: %w", err) + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + gcsSource := gcs.Source{} + if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil { + return nil, err + } + return &gcsSource, nil + }) + if err != nil { + return err } - - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - if err := source.Chunks(ctx, e.ChunksChan()); err != nil { - return fmt.Errorf("could not scan GCS: %w", err) - } - return nil - }) - return nil + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } func isAuthValid(ctx context.Context, c sources.GCSConfig, connection *sourcespb.GCS) bool { diff --git a/pkg/engine/gcs_test.go b/pkg/engine/gcs_test.go index b6278340..4a7d2cae 100644 --- a/pkg/engine/gcs_test.go +++ b/pkg/engine/gcs_test.go @@ -80,7 +80,10 @@ func TestScanGCS(t *testing.T) { t.Errorf("ScanGCS() got: %v, want: %v", err, nil) return } - assert.Nil(t, e.Finish(ctx)) + if err := e.Finish(ctx); err != nil && !test.wantErr && !strings.Contains(err.Error(), "googleapi: Error 400: Bad Request") { + t.Errorf("Finish() got: %v, want: %v", err, nil) + return + } if err == nil && test.wantErr { t.Errorf("ScanGCS() got: %v, want: %v", err, "error") diff --git a/pkg/engine/git.go b/pkg/engine/git.go index d6786e19..d7785494 100644 --- a/pkg/engine/git.go +++ b/pkg/engine/git.go @@ -1,15 +1,12 @@ package engine import ( - "fmt" "runtime" - "github.com/go-errors/errors" gogit "github.com/go-git/go-git/v5" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -36,6 +33,9 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error { if c.ExcludeGlobs != nil { opts = append(opts, git.ScanOptionExcludeGlobs(c.ExcludeGlobs)) } + if c.Bare { + opts = append(opts, git.ScanOptionBare(c.Bare)) + } scanOptions := git.NewScanOptions(opts...) connection := &sourcespb.Git{ @@ -46,32 +46,26 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error { Directories: []string{c.RepoPath}, } var conn anypb.Any - err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}) - if err != nil { + if err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}); err != nil { ctx.Logger().Error(err, "failed to marshal git connection") return err } - gitSource := git.Source{} - if err := gitSource.Init(ctx, "trufflehog - git", 0, 0, true, &conn, runtime.NumCPU()); err != nil { - return errors.WrapPrefix(err, "could not init git source", 0) + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + gitSource := git.Source{} + if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil { + return nil, err + } + gitSource.WithScanOptions(scanOptions) + // Don't try to clean up the provided directory. That's handled by the + // caller of ScanGit. + gitSource.WithPreserveTempDirs(true) + return &gitSource, nil + }) + if err != nil { + return err } - gitSource.WithScanOptions(scanOptions) - // Don't try to clean up the provided directory. That's handled by the - // caller of ScanGit. - gitSource.WithPreserveTempDirs(true) - - ctx = context.WithValues(ctx, - "source_type", sourcespb.SourceType_SOURCE_TYPE_GIT.String(), - "source_name", "git", - ) - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - err := gitSource.Chunks(ctx, e.ChunksChan()) - if err != nil { - return fmt.Errorf("could not scan repo: %w", err) - } - return nil - }) - return nil + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } diff --git a/pkg/engine/github.go b/pkg/engine/github.go index 79d34d76..4e83acbd 100644 --- a/pkg/engine/github.go +++ b/pkg/engine/github.go @@ -1,13 +1,10 @@ package engine import ( - "fmt" - gogit "github.com/go-git/go-git/v5" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -17,15 +14,16 @@ import ( // ScanGitHub scans Github with the provided options. func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error { - source := github.Source{} - connection := sourcespb.GitHub{ - Endpoint: c.Endpoint, - Organizations: c.Orgs, - Repositories: c.Repos, - ScanUsers: c.IncludeMembers, - IgnoreRepos: c.ExcludeRepos, - IncludeRepos: c.IncludeRepos, + Endpoint: c.Endpoint, + Organizations: c.Orgs, + Repositories: c.Repos, + ScanUsers: c.IncludeMembers, + IgnoreRepos: c.ExcludeRepos, + IncludeRepos: c.IncludeRepos, + IncludeIssueComments: c.IncludeIssueComments, + IncludePullRequestComments: c.IncludePullRequestComments, + IncludeGistComments: c.IncludeGistComments, } if len(c.Token) > 0 { connection.Credential = &sourcespb.GitHub_Token{ @@ -42,31 +40,25 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error { return err } - ctx = context.WithValues(ctx, - "source_type", source.Type().String(), - "source_name", "github", - ) - err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency) - if err != nil { - ctx.Logger().Error(err, "failed to initialize github source") - return err - } - logOptions := &gogit.LogOptions{} opts := []git.ScanOption{ git.ScanOptionFilter(c.Filter), git.ScanOptionLogOptions(logOptions), } scanOptions := git.NewScanOptions(opts...) - source.WithScanOptions(scanOptions) - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - err := source.Chunks(ctx, e.ChunksChan()) - if err != nil { - return fmt.Errorf("could not scan github: %w", err) - } - return nil - }) - return nil + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + githubSource := github.Source{} + if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil { + return nil, err + } + githubSource.WithScanOptions(scanOptions) + return &githubSource, nil + }) + if err != nil { + return err + } + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } diff --git a/pkg/engine/gitlab.go b/pkg/engine/gitlab.go index ab0f58d2..7179b238 100644 --- a/pkg/engine/gitlab.go +++ b/pkg/engine/gitlab.go @@ -4,12 +4,10 @@ import ( "fmt" "runtime" - "github.com/go-errors/errors" gogit "github.com/go-git/go-git/v5" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -52,24 +50,18 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error { return err } - gitlabSource := gitlab.Source{} - ctx = context.WithValues(ctx, - "source_type", gitlabSource.Type().String(), - "source_name", "gitlab", - ) - err = gitlabSource.Init(ctx, "trufflehog - gitlab", 0, int64(sourcespb.SourceType_SOURCE_TYPE_GITLAB), true, &conn, runtime.NumCPU()) + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + gitlabSource := gitlab.Source{} + if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil { + return nil, err + } + gitlabSource.WithScanOptions(scanOptions) + return &gitlabSource, nil + }) if err != nil { - return errors.WrapPrefix(err, "could not init GitLab source", 0) + return err } - gitlabSource.WithScanOptions(scanOptions) - - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - err := gitlabSource.Chunks(ctx, e.ChunksChan()) - if err != nil { - return fmt.Errorf("error scanning GitLab: %w", err) - } - return nil - }) - return nil + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go index a2056010..db9b75be 100644 --- a/pkg/engine/s3.go +++ b/pkg/engine/s3.go @@ -4,11 +4,9 @@ import ( "fmt" "runtime" - "github.com/go-errors/errors" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" @@ -48,6 +46,11 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error { if len(c.Buckets) > 0 { connection.Buckets = c.Buckets } + + if len(c.Roles) > 0 { + connection.Roles = c.Roles + } + var conn anypb.Any err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}) if err != nil { @@ -55,23 +58,17 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error { return err } - s3Source := s3.Source{} - ctx = context.WithValues(ctx, - "source_type", s3Source.Type().String(), - "source_name", "s3", - ) - err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU()) + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + s3Source := s3.Source{} + if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil { + return nil, err + } + return &s3Source, nil + }) if err != nil { - return errors.WrapPrefix(err, "failed to init S3 source", 0) + return err } - - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - err := s3Source.Chunks(ctx, e.ChunksChan()) - if err != nil { - return fmt.Errorf("error scanning S3: %w", err) - } - return nil - }) - return nil + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } diff --git a/pkg/engine/syslog.go b/pkg/engine/syslog.go index d93cfa97..7bf413ff 100644 --- a/pkg/engine/syslog.go +++ b/pkg/engine/syslog.go @@ -1,14 +1,12 @@ package engine import ( - "fmt" "os" "github.com/go-errors/errors" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" @@ -42,25 +40,19 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error { if err != nil { return errors.WrapPrefix(err, "error unmarshalling connection", 0) } - source := syslog.Source{} - ctx = context.WithValues(ctx, - "source_type", source.Type().String(), - "source_name", "syslog", - ) - err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency) - source.InjectConnection(connection) + + handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(), + func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) { + syslogSource := syslog.Source{} + if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil { + return nil, err + } + syslogSource.InjectConnection(connection) + return &syslogSource, nil + }) if err != nil { - ctx.Logger().Error(err, "failed to initialize syslog source") return err } - - e.sourcesWg.Go(func() error { - defer common.RecoverWithExit(ctx) - err := source.Chunks(ctx, e.ChunksChan()) - if err != nil { - return fmt.Errorf("could not scan syslog: %w", err) - } - return nil - }) - return nil + _, err = e.sourceManager.ScheduleRun(ctx, handle) + return err } diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go index 2f5262cf..2813178a 100644 --- a/pkg/gitparse/gitparse.go +++ b/pkg/gitparse/gitparse.go @@ -5,6 +5,7 @@ import ( "bytes" "fmt" "io" + "os" "os/exec" "path/filepath" "strconv" @@ -160,7 +161,7 @@ func (c1 *Commit) Equal(c2 *Commit) bool { } // RepoPath parses the output of the `git log` command for the `source` path. -func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string) (chan Commit, error) { +func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string, isBare bool) (chan Commit, error) { args := []string{"-C", source, "log", "-p", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"} if abbreviatedLog { args = append(args, "--diff-filter=AM") @@ -177,7 +178,21 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre cmd := exec.Command("git", args...) absPath, err := filepath.Abs(source) if err == nil { - cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git"))) + if !isBare { + cmd.Env = append(cmd.Env, "GIT_DIR="+filepath.Join(absPath, ".git")) + } else { + cmd.Env = append(cmd.Env, + "GIT_DIR="+absPath, + ) + // We need those variables to handle incoming commits + // while using trufflehog in pre-receive hooks + if dir := os.Getenv("GIT_OBJECT_DIRECTORY"); dir != "" { + cmd.Env = append(cmd.Env, "GIT_OBJECT_DIRECTORY="+dir) + } + if dir := os.Getenv("GIT_ALTERNATE_OBJECT_DIRECTORIES"); dir != "" { + cmd.Env = append(cmd.Env, "GIT_ALTERNATE_OBJECT_DIRECTORIES="+dir) + } + } } return c.executeCommand(ctx, cmd, false) diff --git a/pkg/handlers/archive.go b/pkg/handlers/archive.go index 8ad25cb4..f743c2c9 100644 --- a/pkg/handlers/archive.go +++ b/pkg/handlers/archive.go @@ -6,8 +6,13 @@ import ( "errors" "fmt" "io" + "os" + "os/exec" + "path/filepath" + "strings" "time" + "github.com/h2non/filetype" "github.com/mholt/archiver/v4" "github.com/trufflesecurity/trufflehog/v3/pkg/common" @@ -26,14 +31,18 @@ var ( maxTimeout = time.Duration(30) * time.Second ) +// Ensure the Archive satisfies the interfaces at compile time. +var _ SpecializedHandler = (*Archive)(nil) + // Archive is a handler for extracting and decompressing archives. type Archive struct { - size int + size int + currentDepth int } // New sets a default maximum size and current size counter. -func (d *Archive) New() { - d.size = 0 +func (a *Archive) New() { + a.size = 0 } // SetArchiveMaxSize sets the maximum size of the archive. @@ -52,14 +61,14 @@ func SetArchiveMaxTimeout(timeout time.Duration) { } // FromFile extracts the files from an archive. -func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]byte) { - archiveChan := make(chan ([]byte), 512) +func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte { + archiveChan := make(chan []byte, 512) go func() { ctx, cancel := context.WithTimeout(originalCtx, maxTimeout) logger := logContext.AddLogger(ctx).Logger() defer cancel() defer close(archiveChan) - err := d.openArchive(ctx, 0, data, archiveChan) + err := a.openArchive(ctx, 0, data, archiveChan) if err != nil { if errors.Is(err, archiver.ErrNoMatch) { return @@ -71,7 +80,7 @@ func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([] } // openArchive takes a reader and extracts the contents up to the maximum depth. -func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan ([]byte)) error { +func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error { if depth >= maxDepth { return fmt.Errorf("max archive depth reached") } @@ -97,14 +106,14 @@ func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, if err != nil { return err } - fileBytes, err := d.ReadToMax(ctx, compReader) + fileBytes, err := a.ReadToMax(ctx, compReader) if err != nil { return err } newReader := bytes.NewReader(fileBytes) - return d.openArchive(ctx, depth+1, newReader, archiveChan) + return a.openArchive(ctx, depth+1, newReader, archiveChan) case archiver.Extractor: - err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan)) + err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan)) if err != nil { return err } @@ -114,7 +123,7 @@ func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, } // IsFiletype returns true if the provided reader is an archive. -func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) { +func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) { format, readerB, err := archiver.Identify("", reader) if err != nil { return readerB, false @@ -129,7 +138,7 @@ func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, } // extractorHandler is applied to each file in an archiver.Extractor file. -func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error { +func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context, archiver.File) error { return func(ctx context.Context, f archiver.File) error { logger := logContext.AddLogger(ctx).Logger() logger.V(5).Info("Handling extracted file.", "filename", f.Name()) @@ -142,13 +151,13 @@ func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Conte if err != nil { return err } - fileBytes, err := d.ReadToMax(ctx, fReader) + fileBytes, err := a.ReadToMax(ctx, fReader) if err != nil { return err } fileContent := bytes.NewReader(fileBytes) - err = d.openArchive(ctx, depth, fileContent, archiveChan) + err = a.openArchive(ctx, depth, fileContent, archiveChan) if err != nil { return err } @@ -157,7 +166,7 @@ func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Conte } // ReadToMax reads up to the max size. -func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) { +func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) { // Archiver v4 is in alpha and using an experimental version of // rardecode. There is a bug somewhere with rar decoder format 29 // that can lead to a panic. An issue is open in rardecode repo @@ -175,7 +184,7 @@ func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, } }() fileContent := bytes.Buffer{} - logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-d.size) + logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size) for i := 0; i <= maxSize/512; i++ { if common.IsDone(ctx) { return nil, ctx.Err() @@ -185,17 +194,285 @@ func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, if err != nil && !errors.Is(err, io.EOF) { return []byte{}, err } - d.size += bRead + a.size += bRead if len(fileChunk) > 0 { fileContent.Write(fileChunk[0:bRead]) } if bRead < 512 { return fileContent.Bytes(), nil } - if d.size >= maxSize && bRead == 512 { + if a.size >= maxSize && bRead == 512 { logger.V(2).Info("Max archive size reached.") return fileContent.Bytes(), nil } } return fileContent.Bytes(), nil } + +const ( + arMimeType = "application/x-unix-archive" + rpmMimeType = "application/x-rpm" +) + +// Define a map of mime types to corresponding command-line tools +var mimeTools = map[string][]string{ + arMimeType: {"ar"}, + rpmMimeType: {"rpm2cpio", "cpio"}, +} + +// Check if the command-line tool is installed. +func isToolInstalled(tool string) bool { + _, err := exec.LookPath(tool) + return err == nil +} + +// Ensure all tools are available for given mime type. +func ensureToolsForMimeType(mimeType string) error { + tools, exists := mimeTools[mimeType] + if !exists { + return fmt.Errorf("unsupported mime type") + } + + for _, tool := range tools { + if !isToolInstalled(tool) { + return fmt.Errorf("Required tool " + tool + " is not installed") + } + } + + return nil +} + +// HandleSpecialized takes a file path and an io.Reader representing the input file, +// and processes it based on its extension, such as handling Debian (.deb) and RPM (.rpm) packages. +// It returns an io.Reader that can be used to read the processed content of the file, +// and an error if any issues occurred during processing. +// If the file is specialized, the returned boolean is true with no error. +// The caller is responsible for closing the returned reader. +func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (io.Reader, bool, error) { + mimeType, reader, err := determineMimeType(reader) + if err != nil { + return nil, false, err + } + + switch mimeType { + case arMimeType: // includes .deb files + if err := ensureToolsForMimeType(mimeType); err != nil { + return nil, false, err + } + reader, err = a.extractDebContent(ctx, reader) + case rpmMimeType: + if err := ensureToolsForMimeType(mimeType); err != nil { + return nil, false, err + } + reader, err = a.extractRpmContent(ctx, reader) + default: + return reader, false, nil + } + + if err != nil { + return nil, false, fmt.Errorf("unable to extract file with MIME type %s: %w", mimeType, err) + } + return reader, true, nil +} + +// extractDebContent takes a .deb file as an io.Reader, extracts its contents +// into a temporary directory, and returns a Reader for the extracted data archive. +// It handles the extraction process by using the 'ar' command and manages temporary +// files and directories for the operation. +// The caller is responsible for closing the returned reader. +func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) { + if a.currentDepth >= maxDepth { + return nil, fmt.Errorf("max archive depth reached") + } + + tmpEnv, err := a.createTempEnv(ctx, file) + if err != nil { + return nil, err + } + defer os.Remove(tmpEnv.tempFileName) + defer os.RemoveAll(tmpEnv.extractPath) + + cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name()) + cmd.Dir = tmpEnv.extractPath + if err := executeCommand(cmd); err != nil { + return nil, err + } + + handler := func(ctx logContext.Context, env tempEnv, file string) (string, error) { + if strings.HasPrefix(file, "data.tar.") { + return file, nil + } + return a.handleNestedFileMIME(ctx, env, file) + } + + dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler) + if err != nil { + return nil, err + } + + return openDataArchive(tmpEnv.extractPath, dataArchiveName) +} + +// extractRpmContent takes an .rpm file as an io.Reader, extracts its contents +// into a temporary directory, and returns a Reader for the extracted data archive. +// It handles the extraction process by using the 'rpm2cpio' and 'cpio' commands and manages temporary +// files and directories for the operation. +// The caller is responsible for closing the returned reader. +func (a *Archive) extractRpmContent(ctx logContext.Context, file io.Reader) (io.ReadCloser, error) { + if a.currentDepth >= maxDepth { + return nil, fmt.Errorf("max archive depth reached") + } + + tmpEnv, err := a.createTempEnv(ctx, file) + if err != nil { + return nil, err + } + defer os.Remove(tmpEnv.tempFileName) + defer os.RemoveAll(tmpEnv.extractPath) + + // Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command. + cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id") + cmd.Dir = tmpEnv.extractPath + if err := executeCommand(cmd); err != nil { + return nil, err + } + + handler := func(ctx logContext.Context, env tempEnv, file string) (string, error) { + if strings.HasSuffix(file, ".tar.gz") { + return file, nil + } + return a.handleNestedFileMIME(ctx, env, file) + } + + dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler) + if err != nil { + return nil, err + } + + return openDataArchive(tmpEnv.extractPath, dataArchiveName) +} + +func (a *Archive) handleNestedFileMIME(ctx logContext.Context, tempEnv tempEnv, fileName string) (string, error) { + nestedFile, err := os.Open(filepath.Join(tempEnv.extractPath, fileName)) + if err != nil { + return "", err + } + defer nestedFile.Close() + + mimeType, reader, err := determineMimeType(nestedFile) + if err != nil { + return "", fmt.Errorf("unable to determine MIME type of nested filename: %s, %w", nestedFile.Name(), err) + } + + switch mimeType { + case arMimeType: + _, _, err = a.HandleSpecialized(ctx, reader) + case rpmMimeType: + _, _, err = a.HandleSpecialized(ctx, reader) + default: + return "", nil + } + + if err != nil { + return "", fmt.Errorf("unable to extract file with MIME type %s: %w", mimeType, err) + } + + return fileName, nil +} + +// determineMimeType reads from the provided reader to detect the MIME type. +// It returns the detected MIME type and a new reader that includes the read portion. +func determineMimeType(reader io.Reader) (string, io.Reader, error) { + buffer := make([]byte, 512) + n, err := reader.Read(buffer) + if err != nil { + return "", nil, fmt.Errorf("unable to read file for MIME type detection: %w", err) + } + + // Create a new reader that starts with the buffer we just read + // and continues with the rest of the original reader. + reader = io.MultiReader(bytes.NewReader(buffer[:n]), reader) + + kind, err := filetype.Match(buffer) + if err != nil { + return "", nil, fmt.Errorf("unable to determine file type: %w", err) + } + + return kind.MIME.Value, reader, nil +} + +// handleExtractedFiles processes each file in the extracted directory using a provided handler function. +// The function iterates through the files, applying the handleFile function to each, and returns the name +// of the data archive it finds. This centralizes the logic for handling specialized files such as .deb and .rpm +// by using the appropriate handling function passed as an argument. This design allows for flexibility and reuse +// of this function across various extraction processes in the package. +func (a *Archive) handleExtractedFiles(ctx logContext.Context, env tempEnv, handleFile func(logContext.Context, tempEnv, string) (string, error)) (string, error) { + extractedFiles, err := os.ReadDir(env.extractPath) + if err != nil { + return "", fmt.Errorf("unable to read extracted directory: %w", err) + } + + var dataArchiveName string + for _, file := range extractedFiles { + name, err := handleFile(ctx, env, file.Name()) + if err != nil { + return "", err + } + if name != "" { + dataArchiveName = name + break + } + } + + return dataArchiveName, nil +} + +type tempEnv struct { + tempFile *os.File + tempFileName string + extractPath string +} + +// createTempEnv creates a temporary file and a temporary directory for extracting archives. +// The caller is responsible for removing these temporary resources +// (both the file and directory) when they are no longer needed. +func (a *Archive) createTempEnv(ctx context.Context, file io.Reader) (tempEnv, error) { + tempFile, err := os.CreateTemp("", "tmp") + if err != nil { + return tempEnv{}, fmt.Errorf("unable to create temporary file: %w", err) + } + + extractPath, err := os.MkdirTemp("", "tmp_archive") + if err != nil { + return tempEnv{}, fmt.Errorf("unable to create temporary directory: %w", err) + } + + b, err := a.ReadToMax(ctx, file) + if err != nil { + return tempEnv{}, err + } + + if _, err = tempFile.Write(b); err != nil { + return tempEnv{}, fmt.Errorf("unable to write to temporary file: %w", err) + } + + return tempEnv{tempFile: tempFile, tempFileName: tempFile.Name(), extractPath: extractPath}, nil +} + +func executeCommand(cmd *exec.Cmd) error { + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("unable to execute command %v: %w; error: %s", cmd.String(), err, stderr.String()) + } + return nil +} + +func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser, error) { + dataArchivePath := filepath.Join(extractPath, dataArchiveName) + dataFile, err := os.Open(dataArchivePath) + if err != nil { + return nil, fmt.Errorf("unable to open file: %w", err) + } + return dataFile, nil +} diff --git a/pkg/handlers/archive_test.go b/pkg/handlers/archive_test.go index a7dca30e..d0b0ed7c 100644 --- a/pkg/handlers/archive_test.go +++ b/pkg/handlers/archive_test.go @@ -2,7 +2,9 @@ package handlers import ( "context" + "io" "net/http" + "os" "regexp" "strings" "testing" @@ -10,6 +12,7 @@ import ( diskbufferreader "github.com/bill-rich/disk-buffer-reader" "github.com/stretchr/testify/assert" + logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) @@ -122,3 +125,61 @@ func TestHandleFile(t *testing.T) { assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch)) assert.Equal(t, 1, len(ch)) } + +func TestExtractDebContent(t *testing.T) { + // Open the sample .deb file from the testdata folder. + file, err := os.Open("testdata/test.deb") + assert.Nil(t, err) + defer file.Close() + + ctx := logContext.AddLogger(context.Background()) + a := &Archive{} + + reader, err := a.extractDebContent(ctx, file) + assert.Nil(t, err) + + content, err := io.ReadAll(reader) + assert.Nil(t, err) + expectedLength := 1015582 + assert.Equal(t, expectedLength, len(string(content))) +} + +func TestExtractTarContent(t *testing.T) { + file, err := os.Open("testdata/test.tgz") + assert.Nil(t, err) + defer file.Close() + + ctx := context.Background() + + chunkCh := make(chan *sources.Chunk) + go func() { + defer close(chunkCh) + ok := HandleFile(ctx, file, &sources.Chunk{}, chunkCh) + assert.True(t, ok) + }() + + wantCount := 4 + count := 0 + for range chunkCh { + count++ + } + assert.Equal(t, wantCount, count) +} + +func TestExtractRPMContent(t *testing.T) { + // Open the sample .rpm file from the testdata folder. + file, err := os.Open("testdata/test.rpm") + assert.Nil(t, err) + defer file.Close() + + ctx := logContext.AddLogger(context.Background()) + a := &Archive{} + + reader, err := a.extractRpmContent(ctx, file) + assert.Nil(t, err) + + content, err := io.ReadAll(reader) + assert.Nil(t, err) + expectedLength := 1822720 + assert.Equal(t, expectedLength, len(string(content))) +} diff --git a/pkg/handlers/handlers.go b/pkg/handlers/handlers.go index 84eeadcd..cce0fe1b 100644 --- a/pkg/handlers/handlers.go +++ b/pkg/handlers/handlers.go @@ -4,6 +4,9 @@ import ( "context" "io" + diskbufferreader "github.com/bill-rich/disk-buffer-reader" + + logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) @@ -13,39 +16,82 @@ func DefaultHandlers() []Handler { } } +// SpecializedHandler defines the interface for handlers that can process specialized archives. +// It includes a method to handle specialized archives and determine if the file is of a special type. +type SpecializedHandler interface { + // HandleSpecialized examines the provided file reader within the context and determines if it is a specialized archive. + // It returns a reader with any necessary modifications, a boolean indicating if the file was specialized, + // and an error if something went wrong during processing. + HandleSpecialized(logContext.Context, io.Reader) (io.Reader, bool, error) +} + type Handler interface { - FromFile(context.Context, io.Reader) chan ([]byte) + FromFile(context.Context, io.Reader) chan []byte IsFiletype(context.Context, io.Reader) (io.Reader, bool) New() } -func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan (*sources.Chunk)) bool { - // Find a handler for this file. - var handler Handler +// HandleFile processes a given file by selecting an appropriate handler from DefaultHandlers. +// It first checks if the handler implements SpecializedHandler for any special processing, +// then falls back to regular file type handling. If successful, it reads the file in chunks, +// packages them in the provided chunk skeleton, and sends them to chunksChan. +// The function returns true if processing was successful and false otherwise. +// Context is used for cancellation, and the caller is responsible for canceling it if needed. +func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool { + aCtx := logContext.AddLogger(ctx) for _, h := range DefaultHandlers() { h.New() - var isType bool - if file, isType = h.IsFiletype(ctx, file); isType { - handler = h - break + + // The re-reader is used to reset the file reader after checking if the handler implements SpecializedHandler. + // This is necessary because the archive pkg doesn't correctly determine the file type when using + // an io.MultiReader, which is used by the SpecializedHandler. + reReader, err := diskbufferreader.New(file) + if err != nil { + aCtx.Logger().Error(err, "error creating re-reader reader") + return false } - } - if handler == nil { - return false + defer reReader.Close() + + // Check if the handler implements SpecializedHandler and process accordingly. + if specialHandler, ok := h.(SpecializedHandler); ok { + file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader) + if isSpecial { + return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan) + } + + if err != nil { + aCtx.Logger().Error(err, "error handling file") + } + } + + if err := reReader.Reset(); err != nil { + aCtx.Logger().Error(err, "error resetting re-reader") + return false + } + if _, isType := h.IsFiletype(aCtx, reReader); !isType { + continue + } + + if err := reReader.Reset(); err != nil { + aCtx.Logger().Error(err, "error resetting re-reader") + return false + } + reReader.Stop() + return handleChunks(aCtx, h.FromFile(ctx, reReader), chunkSkel, chunksChan) } - // Process the file and read all []byte chunks from handlerChan. - handlerChan := handler.FromFile(ctx, file) + return false +} + +func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool { for { select { case data, open := <-handlerChan: if !open { - // We finished reading everything from handlerChan. return true } chunk := *chunkSkel chunk.Data = data - // Send data on chunksChan. select { case chunksChan <- &chunk: case <-ctx.Done(): diff --git a/pkg/handlers/testdata/test.deb b/pkg/handlers/testdata/test.deb new file mode 100644 index 00000000..1b078b95 Binary files /dev/null and b/pkg/handlers/testdata/test.deb differ diff --git a/pkg/handlers/testdata/test.rpm b/pkg/handlers/testdata/test.rpm new file mode 100644 index 00000000..40373635 Binary files /dev/null and b/pkg/handlers/testdata/test.rpm differ diff --git a/pkg/handlers/testdata/test.tgz b/pkg/handlers/testdata/test.tgz new file mode 100644 index 00000000..61717fb8 Binary files /dev/null and b/pkg/handlers/testdata/test.tgz differ diff --git a/pkg/output/github_actions.go b/pkg/output/github_actions.go index 077a4da7..d8d99eaf 100644 --- a/pkg/output/github_actions.go +++ b/pkg/output/github_actions.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "sync" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -13,7 +14,7 @@ import ( var dedupeCache = make(map[string]struct{}) // GitHubActionsPrinter is a printer that prints results in GitHub Actions format. -type GitHubActionsPrinter struct{} +type GitHubActionsPrinter struct{ mu sync.Mutex } func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error { out := gitHubActionsOutputFormat{ @@ -56,6 +57,8 @@ func (p *GitHubActionsPrinter) Print(_ context.Context, r *detectors.ResultWithM } dedupeCache[key] = struct{}{} + p.mu.Lock() + defer p.mu.Unlock() message := fmt.Sprintf("Found %s %s result šŸ·šŸ”‘\n", verifiedStatus, out.DetectorType) if r.Result.DecoderType != detectorspb.DecoderType_PLAIN { message = fmt.Sprintf("Found %s %s result with %s encoding šŸ·šŸ”‘\n", verifiedStatus, out.DetectorType, out.DecoderType) diff --git a/pkg/output/json.go b/pkg/output/json.go index af15be63..2510fdaf 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -3,6 +3,7 @@ package output import ( "encoding/json" "fmt" + "sync" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/detectors" @@ -12,7 +13,7 @@ import ( ) // JSONPrinter is a printer that prints results in JSON format. -type JSONPrinter struct{} +type JSONPrinter struct{ mu sync.Mutex } func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error { v := &struct { @@ -60,6 +61,9 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) if err != nil { return fmt.Errorf("could not marshal result: %w", err) } + + p.mu.Lock() fmt.Println(string(out)) + p.mu.Unlock() return nil } diff --git a/pkg/output/legacy_json.go b/pkg/output/legacy_json.go index bbeea627..5ea3f1d3 100644 --- a/pkg/output/legacy_json.go +++ b/pkg/output/legacy_json.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "strings" + "sync" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" @@ -20,7 +21,7 @@ import ( ) // LegacyJSONPrinter is a printer that prints results in legacy JSON format for backwards compatibility. -type LegacyJSONPrinter struct{} +type LegacyJSONPrinter struct{ mu sync.Mutex } func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMetadata) error { var repo string @@ -52,7 +53,10 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe if err != nil { return fmt.Errorf("could not marshal result: %w", err) } + + p.mu.Lock() fmt.Println(string(out)) + p.mu.Unlock() return nil } diff --git a/pkg/output/plain.go b/pkg/output/plain.go index ec70f8ae..9629e439 100644 --- a/pkg/output/plain.go +++ b/pkg/output/plain.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "strings" + "sync" "github.com/fatih/color" "golang.org/x/text/cases" @@ -22,7 +23,7 @@ var ( ) // PlainPrinter is a printer that prints results in plain text format. -type PlainPrinter struct{} +type PlainPrinter struct{ mu sync.Mutex } func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) error { out := outputFormat{ @@ -39,6 +40,8 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata) } printer := greenPrinter + p.mu.Lock() + defer p.mu.Unlock() if out.Verified { yellowPrinter.Print("Found verified result šŸ·šŸ”‘\n") diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.go index 04f00928..1e2117bf 100644 --- a/pkg/pb/detectorspb/detectors.pb.go +++ b/pkg/pb/detectorspb/detectors.pb.go @@ -995,6 +995,9 @@ const ( DetectorType_Dockerhub DetectorType = 921 DetectorType_TrufflehogEnterprise DetectorType = 922 DetectorType_EnvoyApiKey DetectorType = 923 + DetectorType_GitHubOauth2 DetectorType = 924 + DetectorType_Salesforce DetectorType = 925 + DetectorType_HuggingFace DetectorType = 926 ) // Enum value maps for DetectorType. @@ -1920,6 +1923,9 @@ var ( 921: "Dockerhub", 922: "TrufflehogEnterprise", 923: "EnvoyApiKey", + 924: "GitHubOauth2", + 925: "Salesforce", + 926: "HuggingFace", } DetectorType_value = map[string]int32{ "Alibaba": 0, @@ -2842,6 +2848,9 @@ var ( "Dockerhub": 921, "TrufflehogEnterprise": 922, "EnvoyApiKey": 923, + "GitHubOauth2": 924, + "Salesforce": 925, + "HuggingFace": 926, } ) @@ -3220,7 +3229,7 @@ var file_detectors_proto_rawDesc = []byte{ 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12, - 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xc6, 0x73, 0x0a, 0x0c, 0x44, + 0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xfc, 0x73, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, @@ -4145,11 +4154,14 @@ var file_detectors_proto_rawDesc = []byte{ 0x68, 0x75, 0x62, 0x10, 0x99, 0x07, 0x12, 0x19, 0x0a, 0x14, 0x54, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79, - 0x10, 0x9b, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, - 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, - 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x10, 0x9b, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x4f, 0x61, 0x75, + 0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x0f, 0x0a, 0x0a, 0x53, 0x61, 0x6c, 0x65, 0x73, 0x66, + 0x6f, 0x72, 0x63, 0x65, 0x10, 0x9d, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69, + 0x6e, 0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, + 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, + 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, + 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/pb/sourcespb/sources.pb.go b/pkg/pb/sourcespb/sources.pb.go index cd629971..378047fe 100644 --- a/pkg/pb/sourcespb/sources.pb.go +++ b/pkg/pb/sourcespb/sources.pb.go @@ -1953,6 +1953,7 @@ type S3 struct { Credential isS3_Credential `protobuf_oneof:"credential"` Buckets []string `protobuf:"bytes,3,rep,name=buckets,proto3" json:"buckets,omitempty"` MaxObjectSize int64 `protobuf:"varint,6,opt,name=max_object_size,json=maxObjectSize,proto3" json:"max_object_size,omitempty"` + Roles []string `protobuf:"bytes,7,rep,name=roles,proto3" json:"roles,omitempty"` } func (x *S3) Reset() { @@ -2036,6 +2037,13 @@ func (x *S3) GetMaxObjectSize() int64 { return 0 } +func (x *S3) GetRoles() []string { + if x != nil { + return x.Roles + } + return nil +} + type isS3_Credential interface { isS3_Credential() } @@ -3419,7 +3427,7 @@ var file_sources_proto_rawDesc = []byte{ 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xf0, 0x02, 0x0a, 0x02, 0x53, 0x33, 0x12, + 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x86, 0x03, 0x0a, 0x02, 0x53, 0x33, 0x12, 0x37, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4b, 0x65, 0x79, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x48, 0x00, 0x52, 0x09, 0x61, @@ -3441,201 +3449,202 @@ var file_sources_proto_rawDesc = []byte{ 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, - 0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x0c, 0x0a, - 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05, - 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, - 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, - 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, - 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, - 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, - 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, - 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, - 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, - 0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, - 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, - 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a, - 0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, - 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, - 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, - 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, - 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, - 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, - 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, - 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a, - 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, - 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, - 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, - 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, - 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, - 0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, - 0x61, 0x64, 0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, - 0x61, 0x6c, 0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08, + 0x6d, 0x61, 0x78, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x14, 0x0a, + 0x05, 0x72, 0x6f, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x72, 0x6f, + 0x6c, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x05, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x12, 0x24, 0x0a, 0x08, 0x65, + 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, + 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, + 0x65, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, + 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x0a, + 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, + 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x06, 0x0a, 0x04, 0x54, 0x65, 0x73, 0x74, 0x22, + 0x31, 0x0a, 0x09, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x6b, 0x69, 0x74, 0x65, 0x12, 0x16, 0x0a, 0x05, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x22, 0xdb, 0x01, 0x0a, 0x06, 0x47, 0x65, 0x72, 0x72, 0x69, 0x74, 0x12, 0x24, 0x0a, + 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, + 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, + 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, + 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x48, 0x0a, 0x0f, + 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, + 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, + 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x22, 0xa5, 0x01, 0x0a, 0x07, 0x4a, 0x65, 0x6e, 0x6b, 0x69, 0x6e, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75, - 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, - 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, - 0x73, 0x48, 0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, - 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, - 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, - 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, - 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, - 0x65, 0x61, 0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x6f, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, - 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, - 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, - 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, - 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, - 0x75, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, - 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, - 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53, - 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x12, 0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, - 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, - 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, - 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, - 0x74, 0x22, 0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75, - 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, - 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a, - 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, - 0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, - 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, - 0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, - 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, - 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, - 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a, - 0x0a, 0x53, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f, - 0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, - 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, - 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, - 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, - 0x55, 0x72, 0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, - 0x6c, 0x22, 0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, - 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, - 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, - 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, - 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, - 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, - 0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, - 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, - 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, - 0x65, 0x70, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, - 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, - 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, - 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69, - 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, - 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, - 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a, - 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, - 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, - 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, - 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, - 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, - 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, - 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, + 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, + 0x52, 0x09, 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x2d, 0x0a, 0x06, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, + 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x48, 0x00, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, + 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0x9f, 0x02, 0x0a, 0x05, 0x54, 0x65, 0x61, + 0x6d, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, + 0x12, 0x46, 0x0a, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x72, 0x65, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x48, 0x00, 0x52, 0x0d, 0x61, 0x75, 0x74, 0x68, 0x65, + 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, + 0x68, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, + 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, + 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, + 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x4c, 0x69, 0x73, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x65, 0x61, 0x6d, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x61, 0x6d, 0x49, 0x64, 0x73, 0x42, 0x0c, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xc3, 0x01, 0x0a, 0x0b, 0x41, + 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, + 0x05, 0x72, 0x03, 0x90, 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x12, 0x37, 0x0a, 0x0a, 0x62, 0x61, 0x73, 0x69, 0x63, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x73, 0x2e, 0x42, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x48, 0x00, 0x52, 0x09, + 0x62, 0x61, 0x73, 0x69, 0x63, 0x41, 0x75, 0x74, 0x68, 0x12, 0x23, 0x0a, 0x0c, 0x61, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, + 0x00, 0x52, 0x0b, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x22, + 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, + 0x65, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x22, 0x94, 0x01, 0x0a, 0x06, 0x53, 0x79, 0x73, 0x6c, 0x6f, 0x67, 0x12, 0x1a, 0x0a, 0x08, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x12, 0x24, 0x0a, 0x0d, 0x6c, 0x69, 0x73, 0x74, 0x65, + 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, + 0x6c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x41, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x18, 0x0a, + 0x07, 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x74, 0x6c, 0x73, 0x43, 0x65, 0x72, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x6c, 0x73, 0x4b, 0x65, 0x79, 0x12, + 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd6, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x62, 0x6c, + 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x6e, + 0x67, 0x12, 0x48, 0x0a, 0x0f, 0x75, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x55, 0x6e, 0x61, 0x75, 0x74, 0x68, 0x65, + 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x48, 0x00, 0x52, 0x0f, 0x75, 0x6e, 0x61, 0x75, + 0x74, 0x68, 0x65, 0x6e, 0x74, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x78, 0x5f, 0x64, 0x65, 0x70, 0x74, 0x68, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x44, 0x65, 0x70, 0x74, 0x68, + 0x12, 0x30, 0x0a, 0x05, 0x73, 0x69, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x05, 0x73, 0x69, 0x6e, + 0x63, 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x22, 0x51, 0x0a, 0x0d, 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x61, 0x6c, 0x74, 0x69, 0x6d, + 0x65, 0x12, 0x32, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, + 0x53, 0x6c, 0x61, 0x63, 0x6b, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x48, 0x00, 0x52, 0x06, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, + 0x69, 0x61, 0x6c, 0x22, 0x62, 0x0a, 0x0a, 0x53, 0x68, 0x61, 0x72, 0x65, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x2e, 0x4f, + 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x12, 0x19, + 0x0a, 0x08, 0x73, 0x69, 0x74, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x73, 0x69, 0x74, 0x65, 0x55, 0x72, 0x6c, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, + 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x22, 0xa7, 0x03, 0x0a, 0x0a, 0x41, 0x7a, 0x75, 0x72, + 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x24, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x72, 0x03, 0x90, + 0x01, 0x01, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, + 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x2b, 0x0a, 0x05, 0x6f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, + 0x73, 0x2e, 0x4f, 0x61, 0x75, 0x74, 0x68, 0x32, 0x48, 0x00, 0x52, 0x05, 0x6f, 0x61, 0x75, 0x74, + 0x68, 0x12, 0x22, 0x0a, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x69, 0x65, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, + 0x6f, 0x72, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x6f, 0x72, 0x67, 0x61, 0x6e, 0x69, 0x7a, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x72, + 0x67, 0x61, 0x6e, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, + 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, + 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x46, 0x6f, 0x72, 0x6b, 0x73, 0x12, 0x20, 0x0a, 0x0b, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0b, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x12, 0x22, 0x0a, + 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, 0x73, 0x18, 0x09, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x52, 0x65, 0x70, 0x6f, + 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, + 0x65, 0x63, 0x74, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x6e, 0x63, 0x6c, + 0x75, 0x64, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x26, 0x0a, 0x0e, 0x69, + 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, + 0x6c, 0x2a, 0x84, 0x07, 0x0a, 0x0a, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x1d, 0x0a, 0x19, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x53, 0x54, 0x4f, 0x52, 0x41, 0x47, 0x45, 0x10, 0x00, 0x12, + 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, + 0x49, 0x54, 0x42, 0x55, 0x43, 0x4b, 0x45, 0x54, 0x10, 0x01, 0x12, 0x18, 0x0a, 0x14, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x49, 0x52, 0x43, 0x4c, 0x45, + 0x43, 0x49, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4f, 0x4e, 0x46, 0x4c, 0x55, 0x45, 0x4e, 0x43, 0x45, 0x10, 0x03, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, - 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, - 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, - 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, - 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, - 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, - 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, - 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, - 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, - 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, - 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, - 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, - 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, - 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, - 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, - 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, - 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, - 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, - 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, - 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, - 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, - 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, - 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, - 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, - 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, - 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, - 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, - 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, - 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, - 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, - 0x10, 0x1f, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, - 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, - 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x44, 0x4f, 0x43, 0x4b, 0x45, 0x52, 0x10, 0x04, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x43, 0x52, 0x10, 0x05, 0x12, 0x13, 0x0a, + 0x0f, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, + 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x10, 0x07, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, + 0x5f, 0x47, 0x49, 0x54, 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x4c, 0x41, 0x42, 0x10, 0x09, 0x12, 0x14, + 0x0a, 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x49, + 0x52, 0x41, 0x10, 0x0a, 0x12, 0x24, 0x0a, 0x20, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x4e, 0x50, 0x4d, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, + 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, 0x0b, 0x12, 0x25, 0x0a, 0x21, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x59, 0x50, 0x49, 0x5f, 0x55, + 0x4e, 0x41, 0x55, 0x54, 0x48, 0x44, 0x5f, 0x50, 0x41, 0x43, 0x4b, 0x41, 0x47, 0x45, 0x53, 0x10, + 0x0c, 0x12, 0x12, 0x0a, 0x0e, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x53, 0x33, 0x10, 0x0d, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x10, 0x0e, 0x12, 0x1a, 0x0a, 0x16, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, + 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x10, 0x0f, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x49, 0x54, 0x10, 0x10, 0x12, 0x14, 0x0a, + 0x10, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x53, + 0x54, 0x10, 0x11, 0x12, 0x1b, 0x0a, 0x17, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x53, 0x33, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x12, + 0x12, 0x2a, 0x0a, 0x26, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x47, 0x49, 0x54, 0x48, 0x55, 0x42, 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x4e, 0x54, + 0x49, 0x43, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x4f, 0x52, 0x47, 0x10, 0x13, 0x12, 0x19, 0x0a, 0x15, + 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, + 0x44, 0x4b, 0x49, 0x54, 0x45, 0x10, 0x14, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x52, 0x49, 0x54, 0x10, 0x15, 0x12, + 0x17, 0x0a, 0x13, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, + 0x45, 0x4e, 0x4b, 0x49, 0x4e, 0x53, 0x10, 0x16, 0x12, 0x15, 0x0a, 0x11, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x45, 0x41, 0x4d, 0x53, 0x10, 0x17, 0x12, + 0x21, 0x0a, 0x1d, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, + 0x46, 0x52, 0x4f, 0x47, 0x5f, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x4f, 0x52, 0x59, + 0x10, 0x18, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x53, 0x59, 0x53, 0x4c, 0x4f, 0x47, 0x10, 0x19, 0x12, 0x27, 0x0a, 0x23, 0x53, 0x4f, + 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, + 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x5f, 0x4d, 0x4f, 0x4e, 0x49, 0x54, 0x4f, 0x52, 0x49, 0x4e, + 0x47, 0x10, 0x1a, 0x12, 0x1e, 0x0a, 0x1a, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x53, 0x4c, 0x41, 0x43, 0x4b, 0x5f, 0x52, 0x45, 0x41, 0x4c, 0x54, 0x49, 0x4d, + 0x45, 0x10, 0x1b, 0x12, 0x1c, 0x0a, 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x47, 0x4f, 0x4f, 0x47, 0x4c, 0x45, 0x5f, 0x44, 0x52, 0x49, 0x56, 0x45, 0x10, + 0x1c, 0x12, 0x1a, 0x0a, 0x16, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x1d, 0x12, 0x1c, 0x0a, + 0x18, 0x53, 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x43, 0x53, + 0x5f, 0x55, 0x4e, 0x41, 0x55, 0x54, 0x48, 0x45, 0x44, 0x10, 0x1e, 0x12, 0x1b, 0x0a, 0x17, 0x53, + 0x4f, 0x55, 0x52, 0x43, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x41, 0x5a, 0x55, 0x52, 0x45, + 0x5f, 0x52, 0x45, 0x50, 0x4f, 0x53, 0x10, 0x1f, 0x42, 0x3b, 0x5a, 0x39, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, + 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, + 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/sources/chunker.go b/pkg/sources/chunker.go index 6cb39bb2..78a26aed 100644 --- a/pkg/sources/chunker.go +++ b/pkg/sources/chunker.go @@ -5,6 +5,8 @@ import ( "bytes" "errors" "io" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) const ( @@ -25,6 +27,7 @@ func Chunker(originalChunk *Chunk) chan *Chunk { chunkChan <- originalChunk return } + r := bytes.NewReader(originalChunk.Data) reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize) for { @@ -32,21 +35,126 @@ func Chunker(originalChunk *Chunk) chan *Chunk { chunk := *originalChunk chunkBytes = chunkBytes[:ChunkSize] n, err := reader.Read(chunkBytes) - if err != nil && !errors.Is(err, io.EOF) { + if n > 0 { + peekData, _ := reader.Peek(TotalChunkSize - n) + chunkBytes = append(chunkBytes[:n], peekData...) + chunk.Data = chunkBytes + chunkChan <- &chunk + } + if err != nil { break } - if n == 0 { - if errors.Is(err, io.EOF) { - break - } - continue - } - peekData, _ := reader.Peek(PeekSize) - copy(chunkBytes[n:], peekData) - chunk.Data = chunkBytes[:n+len(peekData)] - - chunkChan <- &chunk } }() return chunkChan } + +type chunkReaderConfig struct { + chunkSize int + totalSize int + peekSize int +} + +// ConfigOption is a function that configures a chunker. +type ConfigOption func(*chunkReaderConfig) + +// WithChunkSize sets the chunk size. +func WithChunkSize(size int) ConfigOption { + return func(c *chunkReaderConfig) { + c.chunkSize = size + } +} + +// WithPeekSize sets the peek size. +func WithPeekSize(size int) ConfigOption { + return func(c *chunkReaderConfig) { + c.peekSize = size + } +} + +// ChunkResult is the output unit of a ChunkReader, +// it contains the data and error of a chunk. +type ChunkResult struct { + data []byte + err error +} + +// Bytes for a ChunkResult. +func (cr ChunkResult) Bytes() []byte { + return cr.data +} + +// Error for a ChunkResult. +func (cr ChunkResult) Error() error { + return cr.err +} + +// ChunkReader reads chunks from a reader and returns a channel of chunks and a channel of errors. +// The channel of chunks is closed when the reader is closed. +// This should be used whenever a large amount of data is read from a reader. +// Ex: reading attachments, archives, etc. +type ChunkReader func(ctx context.Context, reader io.Reader) <-chan ChunkResult + +// NewChunkReader returns a ChunkReader with the given options. +func NewChunkReader(opts ...ConfigOption) ChunkReader { + config := applyOptions(opts) + return createReaderFn(config) +} + +func applyOptions(opts []ConfigOption) *chunkReaderConfig { + // Set defaults. + config := &chunkReaderConfig{ + chunkSize: ChunkSize, // default + peekSize: PeekSize, // default + } + + for _, opt := range opts { + opt(config) + } + + config.totalSize = config.chunkSize + config.peekSize + + return config +} + +func createReaderFn(config *chunkReaderConfig) ChunkReader { + return func(ctx context.Context, reader io.Reader) <-chan ChunkResult { + return readInChunks(ctx, reader, config) + } +} + +func readInChunks(ctx context.Context, reader io.Reader, config *chunkReaderConfig) <-chan ChunkResult { + const channelSize = 1 + chunkReader := bufio.NewReaderSize(reader, config.chunkSize) + chunkResultChan := make(chan ChunkResult, channelSize) + + go func() { + defer close(chunkResultChan) + + for { + chunkRes := ChunkResult{} + chunkBytes := make([]byte, config.totalSize) + chunkBytes = chunkBytes[:config.chunkSize] + n, err := chunkReader.Read(chunkBytes) + if n > 0 { + peekData, _ := chunkReader.Peek(config.totalSize - n) + chunkBytes = append(chunkBytes[:n], peekData...) + chunkRes.data = chunkBytes + } + + // If there is an error other than EOF, or if we have read some bytes, send the chunk. + if err != nil && !errors.Is(err, io.EOF) || n > 0 { + if err != nil && !errors.Is(err, io.EOF) { + ctx.Logger().Error(err, "error reading chunk") + chunkRes.err = err + } + chunkResultChan <- chunkRes + } + + if err != nil { + return + } + } + }() + return chunkResultChan +} diff --git a/pkg/sources/chunker_test.go b/pkg/sources/chunker_test.go index 7ec81119..921aefa4 100644 --- a/pkg/sources/chunker_test.go +++ b/pkg/sources/chunker_test.go @@ -1,13 +1,15 @@ package sources import ( - "bufio" "bytes" - "errors" "io" + "strings" "testing" diskbufferreader "github.com/bill-rich/disk-buffer-reader" + "github.com/stretchr/testify/assert" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) func TestChunker(t *testing.T) { @@ -18,59 +20,39 @@ func TestChunker(t *testing.T) { } defer reReader.Close() - baseChunkCount := 0 + baseChunks := make([]*Chunk, 0, 9) - // Count chunks from looping using chunk size. - for { - tmpChunk := make([]byte, ChunkSize) - _, err := reReader.Read(tmpChunk) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - t.Fatal(err) - } - baseChunkCount++ - } - _ = reReader.Reset() - - // Get the first two chunks for comparing later. - baseChunkOne := make([]byte, ChunkSize) - baseChunkTwo := make([]byte, ChunkSize) - - baseReader := bufio.NewReaderSize(reReader, ChunkSize) - _, _ = baseReader.Read(baseChunkOne) - peek, _ := baseReader.Peek(PeekSize) - baseChunkOne = append(baseChunkOne, peek...) - _, _ = baseReader.Read(baseChunkTwo) - peek, _ = baseReader.Peek(PeekSize) - baseChunkTwo = append(baseChunkTwo, peek...) - - // Reset the reader to the beginning and use ChunkReader. - _ = reReader.Reset() - - testChunkCount := 0 chunkData, _ := io.ReadAll(reReader) originalChunk := &Chunk{ Data: chunkData, } + for chunk := range Chunker(originalChunk) { - testChunkCount++ - switch testChunkCount { - case 1: - if !bytes.Equal(baseChunkOne, chunk.Data) { - t.Errorf("First chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk.Data), len(baseChunkOne)) - } - case 2: - if !bytes.Equal(baseChunkTwo, chunk.Data) { - t.Errorf("Second chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk.Data), len(baseChunkTwo)) - } - } - } - if testChunkCount != baseChunkCount { - t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", testChunkCount, baseChunkCount) + baseChunks = append(baseChunks, chunk) } + _ = reReader.Reset() + + testChunks := make([]*Chunk, 0, 9) + + testData, _ := io.ReadAll(reReader) + testOriginalChunk := &Chunk{ + Data: testData, + } + + for chunk := range Chunker(testOriginalChunk) { + testChunks = append(testChunks, chunk) + } + + if len(testChunks) != len(baseChunks) { + t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", len(testChunks), len(baseChunks)) + } + + for i, baseChunk := range baseChunks { + if !bytes.Equal(baseChunk.Data, testChunks[i].Data) { + t.Errorf("Chunk %d did not match expected. Got: %d bytes, expected: %d bytes", i+1, len(testChunks[i].Data), len(baseChunk.Data)) + } + } } func BenchmarkChunker(b *testing.B) { @@ -84,3 +66,133 @@ func BenchmarkChunker(b *testing.B) { } } } + +func TestNewChunkedReader(t *testing.T) { + tests := []struct { + name string + input string + chunkSize int + peekSize int + wantChunks []string + wantErr bool + }{ + { + name: "Smaller data than default chunkSize and peekSize", + input: "example input", + chunkSize: ChunkSize, + peekSize: PeekSize, + wantChunks: []string{"example input"}, + wantErr: false, + }, + { + name: "Reader with no data", + input: "", + chunkSize: 1024, + peekSize: 512, + wantChunks: []string{}, + wantErr: false, + }, + { + name: "Smaller data than chunkSize and peekSize", + input: "small data", + chunkSize: 1024, + peekSize: 512, + wantChunks: []string{"small data"}, + wantErr: false, + }, + { + name: "Equal to chunkSize", + input: strings.Repeat("a", 1024), + chunkSize: 1024, + peekSize: 512, + wantChunks: []string{strings.Repeat("a", 1024)}, + wantErr: false, + }, + { + name: "Equal to chunkSize + peekSize", + input: strings.Repeat("a", 1536), + chunkSize: 1024, + peekSize: 512, + wantChunks: []string{strings.Repeat("a", 1536), strings.Repeat("a", 512)}, + wantErr: false, + }, + { + name: "EOF during peeking", + input: strings.Repeat("a", 1300), + chunkSize: 1024, + peekSize: 512, + wantChunks: []string{strings.Repeat("a", 1300), strings.Repeat("a", 276)}, + wantErr: false, + }, + { + name: "EOF during reading", + input: strings.Repeat("a", 512), + chunkSize: 1024, + peekSize: 512, + wantChunks: []string{strings.Repeat("a", 512)}, + wantErr: false, + }, + { + name: "Equal to totalSize", + input: strings.Repeat("a", 2048), + chunkSize: 1024, + peekSize: 1024, + wantChunks: []string{strings.Repeat("a", 2048), strings.Repeat("a", 1024)}, + wantErr: false, + }, + { + name: "Larger than totalSize", + input: strings.Repeat("a", 4096), + chunkSize: 1024, + peekSize: 1024, + wantChunks: []string{strings.Repeat("a", 2048), strings.Repeat("a", 2048), strings.Repeat("a", 2048), strings.Repeat("a", 1024)}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + readerFunc := NewChunkReader(WithChunkSize(tt.chunkSize), WithPeekSize(tt.peekSize)) + reader := strings.NewReader(tt.input) + ctx := context.Background() + chunkResChan := readerFunc(ctx, reader) + + var err error + chunks := make([]string, 0) + for data := range chunkResChan { + chunks = append(chunks, string(data.Bytes())) + err = data.Error() + } + + assert.Equal(t, tt.wantChunks, chunks, "Chunks do not match") + if tt.wantErr { + assert.Error(t, err, "Expected an error") + } else { + assert.NoError(t, err, "Unexpected error") + } + }) + } +} + +func BenchmarkChunkReader(b *testing.B) { + var bigChunk = make([]byte, 1<<24) // 16MB + + reader := bytes.NewReader(bigChunk) + chunkReader := NewChunkReader(WithChunkSize(ChunkSize), WithPeekSize(PeekSize)) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + b.StartTimer() + chunkResChan := chunkReader(context.Background(), reader) + + // Drain the channel. + for range chunkResChan { + } + + b.StopTimer() + _, err := reader.Seek(0, 0) + assert.Nil(b, err) + } +} diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.go index ebf7b6cf..1fb94c17 100644 --- a/pkg/sources/circleci/circleci.go +++ b/pkg/sources/circleci/circleci.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "io" "net/http" "sync/atomic" @@ -219,7 +218,7 @@ func (s *Source) stepsForBuild(_ context.Context, proj project, bld build) ([]bu return bldRes.Steps, nil } -func (s *Source) chunkAction(_ context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error { +func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error { req, err := http.NewRequest("GET", act.OutputURL, nil) if err != nil { return err @@ -229,35 +228,40 @@ func (s *Source) chunkAction(_ context.Context, proj project, bld build, act act return err } defer res.Body.Close() - logOutput, err := io.ReadAll(res.Body) - if err != nil { - return err - } linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum) - chunk := &sources.Chunk{ - SourceType: s.Type(), - SourceName: s.name, - SourceID: s.SourceID(), - Data: removeCircleSha1Line(logOutput), - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Circleci{ - Circleci: &source_metadatapb.CircleCI{ - VcsType: proj.VCS, - Username: proj.Username, - Repository: proj.RepoName, - BuildNumber: int64(bld.BuildNum), - BuildStep: stepName, - Link: linkURL, + chunkReader := sources.NewChunkReader() + chunkResChan := chunkReader(ctx, res.Body) + for data := range chunkResChan { + chunk := &sources.Chunk{ + SourceType: s.Type(), + SourceName: s.name, + SourceID: s.SourceID(), + Data: removeCircleSha1Line(data.Bytes()), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Circleci{ + Circleci: &source_metadatapb.CircleCI{ + VcsType: proj.VCS, + Username: proj.Username, + Repository: proj.RepoName, + BuildNumber: int64(bld.BuildNum), + BuildStep: stepName, + Link: linkURL, + }, }, }, - }, - Verify: s.verify, + Verify: s.verify, + } + chunk.Data = data.Bytes() + if err := data.Error(); err != nil { + return err + } + if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil { + return err + } } - chunksChan <- chunk - return nil } diff --git a/pkg/sources/docker/docker.go b/pkg/sources/docker/docker.go index 102fd55f..71b57e6e 100644 --- a/pkg/sources/docker/docker.go +++ b/pkg/sources/docker/docker.go @@ -84,6 +84,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err var img v1.Image var err error var base, tag string + var hasDigest bool + var imageName name.Reference if strings.HasPrefix(image, "file://") { image = strings.TrimPrefix(image, "file://") @@ -93,10 +95,18 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err return err } } else { - base, tag = baseAndTagFromImage(image) - imageName, err := name.NewTag(image) - if err != nil { - return err + base, tag, hasDigest = baseAndTagFromImage(image) + + if hasDigest { + imageName, err = name.NewDigest(image) + if err != nil { + return err + } + } else { + imageName, err = name.NewTag(image) + if err != nil { + return err + } } img, err = remote.Image(imageName, remoteOpts...) @@ -194,10 +204,20 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err return nil } -func baseAndTagFromImage(image string) (base, tag string) { +func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) { regRepoDelimiter := "/" tagDelim := ":" - parts := strings.Split(image, tagDelim) + digestDelim := "@" + + // Split on digest first, if present. + parts := strings.SplitN(image, digestDelim, 2) + if len(parts) > 1 { + base = parts[0] + tag = parts[1] + hasDigest = true + return + } + parts = strings.Split(image, tagDelim) // Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation. if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], regRepoDelimiter) { base = strings.Join(parts[:len(parts)-1], tagDelim) diff --git a/pkg/sources/docker/docker_test.go b/pkg/sources/docker/docker_test.go index e022b8a1..aa290c34 100644 --- a/pkg/sources/docker/docker_test.go +++ b/pkg/sources/docker/docker_test.go @@ -1,5 +1,3 @@ -//go:build integration - package docker import ( @@ -51,3 +49,64 @@ func TestDockerImageScan(t *testing.T) { assert.Equal(t, 1, chunkCounter) } + +func TestDockerImageScanWithDigest(t *testing.T) { + dockerConn := &sourcespb.Docker{ + Credential: &sourcespb.Docker_Unauthenticated{ + Unauthenticated: &credentialspb.Unauthenticated{}, + }, + Images: []string{"trufflesecurity/secrets@sha256:864f6d41209462d8e37fc302ba1532656e265f7c361f11e29fed6ca1f4208e11"}, + } + + conn := &anypb.Any{} + err := conn.MarshalFrom(dockerConn) + assert.NoError(t, err) + + s := &Source{} + err = s.Init(context.TODO(), "test source", 0, 0, false, conn, 1) + assert.NoError(t, err) + + var wg sync.WaitGroup + chunksChan := make(chan *sources.Chunk, 1) + chunkCounter := 0 + wg.Add(1) + go func() { + defer wg.Done() + for chunk := range chunksChan { + assert.NotEmpty(t, chunk) + chunkCounter++ + } + }() + + err = s.Chunks(context.TODO(), chunksChan) + assert.NoError(t, err) + + close(chunksChan) + wg.Wait() + + assert.Equal(t, 1, chunkCounter) +} + +func TestBaseAndTagFromImage(t *testing.T) { + tests := []struct { + image string + wantBase string + wantTag string + wantDigest bool + }{ + {"golang:1.16", "golang", "1.16", false}, + {"golang@sha256:abcdef", "golang", "sha256:abcdef", true}, + {"ghcr.io/golang:1.16", "ghcr.io/golang", "1.16", false}, + {"ghcr.io/golang:nightly", "ghcr.io/golang", "nightly", false}, + {"ghcr.io/golang", "ghcr.io/golang", "latest", false}, + {"ghcr.io/trufflesecurity/secrets", "ghcr.io/trufflesecurity/secrets", "latest", false}, + } + + for _, tt := range tests { + gotBase, gotTag, gotDigest := baseAndTagFromImage(tt.image) + if gotBase != tt.wantBase || gotTag != tt.wantTag || gotDigest != tt.wantDigest { + t.Errorf("baseAndTagFromImage(%q) = (%q, %q, %v), want (%q, %q, %v)", + tt.image, gotBase, gotTag, gotDigest, tt.wantBase, tt.wantTag, tt.wantDigest) + } + } +} diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go index 41673a4e..89d47348 100644 --- a/pkg/sources/filesystem/filesystem.go +++ b/pkg/sources/filesystem/filesystem.go @@ -1,7 +1,6 @@ package filesystem import ( - "bufio" "fmt" "io" "io/fs" @@ -23,13 +22,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) -const ( - // These buffer sizes are mainly driven by our largest credential size, which is GCP @ ~2.25KB. - // Having a peek size larger than that ensures that we have complete credential coverage in our chunks. - BufferSize = 10 * 1024 // 10KB - PeekSize = 3 * 1024 // 3KB -) - type Source struct { name string sourceId int64 @@ -187,34 +179,33 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou } reReader.Stop() - for { - chunkBytes := make([]byte, BufferSize) - reader := bufio.NewReaderSize(reReader, BufferSize) - n, err := reader.Read(chunkBytes) - if err != nil && !errors.Is(err, io.EOF) { - break + chunkReader := sources.NewChunkReader() + chunkResChan := chunkReader(ctx, reReader) + for data := range chunkResChan { + if err := data.Error(); err != nil { + s.log.Error(err, "error reading chunk.") + continue } - peekData, _ := reader.Peek(PeekSize) - if n > 0 { - chunksChan <- &sources.Chunk{ - SourceType: s.Type(), - SourceName: s.name, - SourceID: s.SourceID(), - Data: append(chunkBytes[:n], peekData...), - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Filesystem{ - Filesystem: &source_metadatapb.Filesystem{ - File: sanitizer.UTF8(path), - }, + + chunk := &sources.Chunk{ + SourceType: s.Type(), + SourceName: s.name, + SourceID: s.SourceID(), + Data: data.Bytes(), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Filesystem{ + Filesystem: &source_metadatapb.Filesystem{ + File: sanitizer.UTF8(path), }, }, - Verify: s.verify, - } + }, + Verify: s.verify, } - if errors.Is(err, io.EOF) { - break + if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil { + return err } } + return nil } diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.go index 7d56bc9d..ef5dfbdd 100644 --- a/pkg/sources/filesystem/filesystem_test.go +++ b/pkg/sources/filesystem/filesystem_test.go @@ -1,10 +1,14 @@ package filesystem import ( + "os" + "strings" "testing" "time" + "github.com/go-logr/logr" "github.com/kylelemons/godebug/pretty" + "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/anypb" "github.com/trufflesecurity/trufflehog/v3/pkg/context" @@ -78,3 +82,41 @@ func TestSource_Scan(t *testing.T) { }) } } + +func TestScanFile(t *testing.T) { + tmpfile, err := os.CreateTemp("", "example.txt") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + chunkSize := sources.ChunkSize + secretPart1 := "SECRET" + secretPart2 := "SPLIT" + // Split the secret into two parts and pad the rest of the chunk with A's. + data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2)) + + _, err = tmpfile.Write([]byte(data)) + assert.Nil(t, err) + + err = tmpfile.Close() + assert.Nil(t, err) + + source := &Source{} + chunksChan := make(chan *sources.Chunk, 2) + + ctx := context.WithLogger(context.Background(), logr.Discard()) + go func() { + defer close(chunksChan) + err = source.scanFile(ctx, tmpfile.Name(), chunksChan) + assert.Nil(t, err) + }() + + // Read from the channel and validate the secrets. + foundSecret := "" + for chunkCh := range chunksChan { + foundSecret += string(chunkCh.Data) + } + + assert.Contains(t, foundSecret, secretPart1+secretPart2) +} diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.go index d3edb87b..16a2e64d 100644 --- a/pkg/sources/git/git.go +++ b/pkg/sources/git/git.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "io" "net/url" "os" "os/exec" @@ -26,6 +25,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/gitparse" "github.com/trufflesecurity/trufflehog/v3/pkg/handlers" @@ -115,6 +115,9 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, s.sourceId = sourceId s.jobId = jobId s.verify = verify + if s.scanOptions == nil { + s.scanOptions = &ScanOptions{} + } var conn sourcespb.Git if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil { @@ -251,12 +254,12 @@ func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) e if len(gitDir) == 0 { continue } - if strings.HasSuffix(gitDir, "git") { + if !s.scanOptions.Bare && strings.HasSuffix(gitDir, "git") { // TODO: Figure out why we skip directories ending in "git". continue } // try paths instead of url - repo, err := RepoFromPath(gitDir) + repo, err := RepoFromPath(gitDir, s.scanOptions.Bare) if err != nil { ctx.Logger().Info("error scanning repository", "repo", gitDir, "error", err) continue @@ -278,10 +281,11 @@ func (s *Source) scanDirs(ctx context.Context, chunksChan chan *sources.Chunk) e return nil } -func RepoFromPath(path string) (*git.Repository, error) { - options := &git.PlainOpenOptions{ - DetectDotGit: true, - EnableDotGitCommonDir: true, +func RepoFromPath(path string, isBare bool) (*git.Repository, error) { + options := &git.PlainOpenOptions{} + if !isBare { + options.DetectDotGit = true + options.EnableDotGitCommonDir = true } return git.PlainOpenWithOptions(path, options) } @@ -393,7 +397,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string return err } - commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs) + commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "", scanOptions.ExcludeGlobs, scanOptions.Bare) if err != nil { return err } @@ -621,8 +625,10 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil { return err } - if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil { - ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err) + if !scanOptions.Bare { + if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil { + ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err) + } } // We're logging time, but the repoPath is usually a dynamically generated folder in /tmp @@ -634,7 +640,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin } scanTime := time.Now().Unix() - start - ctx.Logger().V(1).Info("scanning git repo complete", "repo", repoUrl, "path", repoPath, "time_seconds", scanTime) + ctx.Logger().V(1).Info("scanning git repo complete", "repo", repoUrl, "path", repoPath, "time_seconds", scanTime, "commits_scanned", atomic.LoadUint64(&s.metrics.commitsScanned)) return nil } @@ -924,15 +930,19 @@ func handleBinary(ctx context.Context, repo *git.Repository, chunksChan chan *so } reader.Stop() - chunkData, err := io.ReadAll(reader) - if err != nil { - return err + chunkReader := sources.NewChunkReader() + chunkResChan := chunkReader(ctx, reader) + for data := range chunkResChan { + chunk := *chunkSkel + chunk.Data = data.Bytes() + if err := data.Error(); err != nil { + return err + } + if err := common.CancellableWrite(ctx, chunksChan, &chunk); err != nil { + return err + } } - chunk := *chunkSkel - chunk.Data = chunkData - chunksChan <- &chunk - return nil } diff --git a/pkg/sources/git/scan_options.go b/pkg/sources/git/scan_options.go index 09572a6f..d5591419 100644 --- a/pkg/sources/git/scan_options.go +++ b/pkg/sources/git/scan_options.go @@ -10,6 +10,7 @@ type ScanOptions struct { BaseHash string // When scanning a git.Log, this is the oldest/first commit. HeadHash string MaxDepth int64 + Bare bool ExcludeGlobs []string LogOptions *git.LogOptions } @@ -52,6 +53,12 @@ func ScanOptionLogOptions(logOptions *git.LogOptions) ScanOption { } } +func ScanOptionBare(bare bool) ScanOption { + return func(scanOptions *ScanOptions) { + scanOptions.Bare = bare + } +} + func NewScanOptions(options ...ScanOption) *ScanOptions { scanOptions := &ScanOptions{ Filter: common.FilterEmpty(), diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index c4b0df85..bf7157e6 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -977,9 +977,9 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo) } -func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error { - s.log.Info("scanning comments", "repository", repoPath) +const initialPage = 1 // page to start listing from +func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan chan *sources.Chunk) error { // Support ssh and https URLs repoURL, err := git.GitURLParse(repoPath) if err != nil { @@ -988,6 +988,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c trimmedURL := removeURLAndSplit(repoURL.String()) if repoURL.Host == "gist.github.com" && s.includeGistComments { + s.log.Info("scanning github gist comments", "repository", repoPath) // GitHub Gist URL. var gistId string if len(trimmedURL) == 2 { @@ -1002,7 +1003,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c options := &github.ListOptions{ PerPage: defaultPagination, - Page: 1, + Page: initialPage, } for { comments, resp, err := s.apiClient.Gists.ListComments(ctx, gistId, options) @@ -1023,90 +1024,124 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c break } } - } else { - // Normal repository URL (https://github.com//). - owner := trimmedURL[1] - repo := trimmedURL[2] + } + return s.processRepoComments(ctx, repoPath, trimmedURL, repoURL, chunksChan) +} - var ( - sortType = "created" - directionType = "desc" - allComments = 0 - ) +// Note: these can't be consts because the address is needed when using with the GitHub library. +var ( + // sortType defines the criteria for sorting comments. + // By default comments are sorted by their creation date. + sortType = "created" + // directionType defines the direction of sorting. + // "desc" means comments will be sorted in descending order, showing the latest comments first. + directionType = "desc" + // allComments is a placeholder for specifying the comment ID to start listing from. + // A value of 0 means that all comments will be listed. + allComments = 0 +) - if s.includeIssueComments { +type repoInfo struct { + owner string + repo string + repoPath string +} - issueOpts := &github.IssueListCommentsOptions{ - Sort: &sortType, - Direction: &directionType, - ListOptions: github.ListOptions{ - PerPage: defaultPagination, - Page: 1, - }, - } +func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error { + // Normal repository URL (https://github.com//). + if len(trimmedURL) < 3 { + return fmt.Errorf("url missing owner and/or repo: '%s'", repoURL.String()) + } + owner := trimmedURL[1] + repo := trimmedURL[2] - for { - issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, owner, repo, allComments, issueOpts) - if s.handleRateLimit(err, resp) { - break - } - - if err != nil { - return err - } - - err = s.chunkIssueComments(ctx, repo, issueComments, chunksChan, repoPath) - if err != nil { - return err - } - - issueOpts.ListOptions.Page++ - - if len(issueComments) < defaultPagination { - break - } - } + repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath} + if s.includeIssueComments { + if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil { + return err } - if s.includePRComments { - prOpts := &github.PullRequestListCommentsOptions{ - Sort: sortType, - Direction: directionType, - ListOptions: github.ListOptions{ - PerPage: defaultPagination, - Page: 1, - }, - } - - for { - prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, owner, repo, allComments, prOpts) - if s.handleRateLimit(err, resp) { - break - } - - if err != nil { - return err - } - - err = s.chunkPullRequestComments(ctx, repo, prComments, chunksChan, repoPath) - if err != nil { - return err - } - - prOpts.ListOptions.Page++ - - if len(prComments) < defaultPagination { - break - } - } - } } + if s.includePRComments { + return s.processPRComments(ctx, repoInfo, chunksChan) + } + return nil + +} + +func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error { + s.log.Info("scanning github issue comments", "repository", info.repoPath) + + issueOpts := &github.IssueListCommentsOptions{ + Sort: &sortType, + Direction: &directionType, + ListOptions: github.ListOptions{ + PerPage: defaultPagination, + Page: initialPage, + }, + } + + for { + issueComments, resp, err := s.apiClient.Issues.ListComments(ctx, info.owner, info.repo, allComments, issueOpts) + if s.handleRateLimit(err, resp) { + break + } + + if err != nil { + return err + } + + if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil { + return err + } + + issueOpts.ListOptions.Page++ + + if len(issueComments) < defaultPagination { + break + } + } return nil } -func (s *Source) chunkIssueComments(ctx context.Context, repo string, comments []*github.IssueComment, chunksChan chan *sources.Chunk, repoPath string) error { +func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error { + s.log.Info("scanning github pull request comments", "repository", info.repoPath) + + prOpts := &github.PullRequestListCommentsOptions{ + Sort: sortType, + Direction: directionType, + ListOptions: github.ListOptions{ + PerPage: defaultPagination, + Page: initialPage, + }, + } + + for { + prComments, resp, err := s.apiClient.PullRequests.ListComments(ctx, info.owner, info.repo, allComments, prOpts) + if s.handleRateLimit(err, resp) { + break + } + + if err != nil { + return err + } + + if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil { + return err + } + + prOpts.ListOptions.Page++ + + if len(prComments) < defaultPagination { + break + } + } + return nil +} + +func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error { for _, comment := range comments { // Create chunk and send it to the channel. chunk := &sources.Chunk{ @@ -1138,7 +1173,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repo string, comments [ return nil } -func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk, repoPath string) error { +func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error { for _, comment := range comments { // Create chunk and send it to the channel. chunk := &sources.Chunk{ @@ -1186,7 +1221,7 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()), // Fetching this information requires making an additional API call. // We may want to include this in the future. - //Visibility: s.visibilityOf(ctx, repoPath), + // Visibility: s.visibilityOf(ctx, repoPath), }, }, }, diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 552e5cd9..e30f47d8 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -81,7 +81,7 @@ func TestSource_Token(t *testing.T) { } func TestSource_ScanComments(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() secret, err := common.GetTestSecret(ctx) @@ -190,7 +190,7 @@ func TestSource_ScanComments(t *testing.T) { return } - chunksCh := make(chan *sources.Chunk, 5) + chunksCh := make(chan *sources.Chunk, 1) go func() { // Close the channel defer close(chunksCh) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index ee106638..73890bb0 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -8,8 +8,10 @@ import ( "encoding/pem" "fmt" "net/http" + "net/url" "reflect" "strconv" + "strings" "testing" "time" @@ -25,6 +27,7 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) { @@ -709,3 +712,34 @@ func Test_scan_SetProgressComplete(t *testing.T) { }) } } + +func TestProcessRepoComments(t *testing.T) { + tests := []struct { + name string + trimmedURL []string + wantErr bool + }{ + { + name: "URL with missing owner and/or repo", + trimmedURL: []string{"https://github.com/"}, + wantErr: true, + }, + { + name: "URL with complete owner and repo", + trimmedURL: []string{"https://github.com/", "owner", "repo"}, + wantErr: false, + }, + // TODO: Add more test cases to cover other scenarios. + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Source{} + repoURL, _ := url.Parse(strings.Join(tt.trimmedURL, "/")) + chunksChan := make(chan *sources.Chunk) + + err := s.processRepoComments(context.Background(), "repoPath", tt.trimmedURL, repoURL, chunksChan) + assert.Equal(t, tt.wantErr, err != nil) + }) + } +} diff --git a/pkg/sources/job_progress.go b/pkg/sources/job_progress.go index 50605485..0ae4b847 100644 --- a/pkg/sources/job_progress.go +++ b/pkg/sources/job_progress.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "sync" - "sync/atomic" "time" ) @@ -103,10 +102,14 @@ type JobProgress struct { // JobProgressMetrics tracks the metrics of a job. type JobProgressMetrics struct { - StartTime time.Time - EndTime time.Time - TotalUnits uint64 - FinishedUnits uint64 + StartTime time.Time + EndTime time.Time + // Total number of units found by the Source. + TotalUnits uint64 + // Total number of units that have finished chunking. + FinishedUnits uint64 + // Total number of chunks produced. This metric updates before the + // chunk is sent on the output channel. TotalChunks uint64 Errors []error DoneEnumerating bool @@ -163,11 +166,15 @@ func (jp *JobProgress) Finish() { } func (jp *JobProgress) Done() <-chan struct{} { return jp.ctx.Done() } func (jp *JobProgress) ReportUnit(unit SourceUnit) { - atomic.AddUint64(&jp.metrics.TotalUnits, 1) + jp.metricsLock.Lock() + jp.metrics.TotalUnits++ + jp.metricsLock.Unlock() jp.executeHooks(func(hook JobProgressHook) { hook.ReportUnit(jp.Ref(), unit) }) } func (jp *JobProgress) ReportChunk(unit SourceUnit, chunk *Chunk) { - atomic.AddUint64(&jp.metrics.TotalChunks, 1) + jp.metricsLock.Lock() + jp.metrics.TotalChunks++ + jp.metricsLock.Unlock() jp.executeHooks(func(hook JobProgressHook) { hook.ReportChunk(jp.Ref(), unit, chunk) }) } func (jp *JobProgress) StartUnitChunking(unit SourceUnit, start time.Time) { @@ -176,7 +183,9 @@ func (jp *JobProgress) StartUnitChunking(unit SourceUnit, start time.Time) { } func (jp *JobProgress) EndUnitChunking(unit SourceUnit, end time.Time) { // TODO: Record time. - atomic.AddUint64(&jp.metrics.FinishedUnits, 1) + jp.metricsLock.Lock() + jp.metrics.FinishedUnits++ + jp.metricsLock.Unlock() jp.executeHooks(func(hook JobProgressHook) { hook.EndUnitChunking(jp.Ref(), unit, end) }) } func (jp *JobProgress) StartEnumerating(start time.Time) { @@ -196,7 +205,12 @@ func (jp *JobProgress) EndEnumerating(end time.Time) { func (jp *JobProgress) Snapshot() JobProgressMetrics { jp.metricsLock.Lock() defer jp.metricsLock.Unlock() - return jp.metrics + + metrics := jp.metrics + metrics.Errors = make([]error, len(metrics.Errors)) + copy(metrics.Errors, jp.metrics.Errors) + + return metrics } // ReportError adds a non-nil error to the aggregate of errors diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.go index b7971ae4..b05832ed 100644 --- a/pkg/sources/s3/s3.go +++ b/pkg/sources/s3/s3.go @@ -2,7 +2,6 @@ package s3 import ( "fmt" - "io" "strings" "sync" "sync/atomic" @@ -10,9 +9,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go/service/sts" diskbufferreader "github.com/bill-rich/disk-buffer-reader" "github.com/go-errors/errors" "github.com/go-logr/logr" @@ -132,54 +133,82 @@ func (s *Source) newClient(region string) (*s3.S3, error) { return s3.New(sess), nil } -// Chunks emits chunks of bytes over a channel. -func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error { - const defaultAWSRegion = "us-east-1" +// Separate role assumption functionality into a different newClient function +func (s *Source) newRoleClient(region string, roleArn string) (*s3.S3, error) { - client, err := s.newClient(defaultAWSRegion) + cfg := aws.NewConfig() + cfg.CredentialsChainVerboseErrors = aws.Bool(true) + cfg.Region = aws.String(region) + + var baseCredentials *credentials.Credentials + + sess, err := session.NewSession(cfg) if err != nil { - return errors.WrapPrefix(err, "could not create s3 client", 0) + return nil, err + } + + stsClient := sts.New(sess) + baseCredentials = stscreds.NewCredentialsWithClient(stsClient, roleArn, func(p *stscreds.AssumeRoleProvider) { + p.RoleSessionName = "trufflehog" + }) + + cfg.Credentials = baseCredentials + + sess, err = session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Config: *cfg, + }) + + if err != nil { + return nil, err + } + + // Create and return the S3 client with the final configuration + return s3.New(sess), nil +} + +// IAM identity needs s3:ListBuckets permission +func (s *Source) getBucketsToScan(client *s3.S3) ([]string, error) { + if len(s.conn.Buckets) > 0 { + return s.conn.Buckets, nil + } + + res, err := client.ListBuckets(&s3.ListBucketsInput{}) + if err != nil { + return nil, fmt.Errorf("could not list s3 buckets: %w", err) } var bucketsToScan []string - - switch s.conn.GetCredential().(type) { - case *sourcespb.S3_AccessKey, *sourcespb.S3_SessionToken, *sourcespb.S3_CloudEnvironment: - if len(s.conn.Buckets) == 0 { - res, err := client.ListBuckets(&s3.ListBucketsInput{}) - if err != nil { - return fmt.Errorf("could not list s3 buckets: %w", err) - } - buckets := res.Buckets - for _, bucket := range buckets { - bucketsToScan = append(bucketsToScan, *bucket.Name) - } - } else { - bucketsToScan = s.conn.Buckets - } - case *sourcespb.S3_Unauthenticated: - bucketsToScan = s.conn.Buckets - default: - return errors.Errorf("invalid configuration given for %s source", s.name) + for _, bucket := range res.Buckets { + bucketsToScan = append(bucketsToScan, *bucket.Name) } + return bucketsToScan, nil +} +func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bucketsToScan []string, chunksChan chan *sources.Chunk) error { + const defaultAWSRegion = "us-east-1" objectCount := uint64(0) + for i, bucket := range bucketsToScan { if common.IsDone(ctx) { return nil } s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "") - s.log.Info("Scanning bucket", "bucket", bucket) - region, err := s3manager.GetBucketRegionWithClient(context.Background(), client, bucket) + region, err := s3manager.GetBucketRegionWithClient(ctx, client, bucket) if err != nil { - s.log.Error(err, "could not get s3 region for bucket", "bucket", bucket) + s.log.Error(err, "could not get s3 region for bucket", "bucket: ", bucket) continue } + var regionalClient *s3.S3 if region != defaultAWSRegion { - regionalClient, err = s.newClient(region) + if role != "" { + regionalClient, err = s.newRoleClient(region, role) + } else { + regionalClient, err = s.newClient(region) + } if err != nil { s.log.Error(err, "could not make regional s3 client") } @@ -197,13 +226,49 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err }) if err != nil { - return fmt.Errorf( - "could not list objects in s3 bucket: bucket %s: %w", - bucket, - err) + s.log.Error(err, "could not list objects in s3 bucket", "bucket: ", bucket) + continue } } s.SetProgressComplete(len(bucketsToScan), len(bucketsToScan), fmt.Sprintf("Completed scanning source %s. %d objects scanned.", s.name, objectCount), "") + return nil +} + +// Chunks emits chunks of bytes over a channel. +func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error { + const defaultAWSRegion = "us-east-1" + + if len(s.conn.Roles) > 0 { + for _, role := range s.conn.Roles { + client, err := s.newRoleClient(defaultAWSRegion, role) + if err != nil { + return errors.WrapPrefix(err, "could not create s3 client", 0) + } + + bucketsToScan, err := s.getBucketsToScan(client) + if err != nil { + return err + } + + if err := s.scanBuckets(ctx, client, role, bucketsToScan, chunksChan); err != nil { + return err + } + } + } else { + client, err := s.newClient(defaultAWSRegion) + if err != nil { + return errors.WrapPrefix(err, "could not create s3 client", 0) + } + + bucketsToScan, err := s.getBucketsToScan(client) + if err != nil { + return err + } + + if err := s.scanBuckets(ctx, client, "", bucketsToScan, chunksChan); err != nil { + return err + } + } return nil } @@ -335,17 +400,22 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan } reader.Stop() - chunk := *chunkSkel - chunkData, err := io.ReadAll(reader) - if err != nil { - s.log.Error(err, "Could not read file data.") - return nil + chunkReader := sources.NewChunkReader() + chunkResChan := chunkReader(ctx, reader) + for data := range chunkResChan { + if err := data.Error(); err != nil { + s.log.Error(err, "error reading chunk.") + continue + } + chunk := *chunkSkel + chunk.Data = data.Bytes() + if err := common.CancellableWrite(ctx, chunksChan, &chunk); err != nil { + return err + } } + atomic.AddUint64(objectCount, 1) s.log.V(5).Info("S3 object scanned.", "object_count", objectCount, "page_number", pageNumber) - chunk.Data = chunkData - chunksChan <- &chunk - nErr, ok = errorCount.Load(prefix) if !ok { nErr = 0 diff --git a/pkg/sources/s3/s3_integration_test.go b/pkg/sources/s3/s3_integration_test.go new file mode 100644 index 00000000..bf675c61 --- /dev/null +++ b/pkg/sources/s3/s3_integration_test.go @@ -0,0 +1,47 @@ +//go:build integration +// +build integration + +package s3 + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +func TestSource_ChunksCount(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + s := Source{} + connection := &sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + Buckets: []string{"truffletestbucket"}, + } + conn, err := anypb.New(connection) + if err != nil { + t.Fatal(err) + } + + err = s.Init(ctx, "test name", 0, 0, false, conn, 1) + chunksCh := make(chan *sources.Chunk) + go func() { + defer close(chunksCh) + err = s.Chunks(ctx, chunksCh) + assert.Nil(t, err) + }() + + wantChunkCount := 120 + got := 0 + + for range chunksCh { + got++ + } + assert.Greater(t, got, wantChunkCount) +} diff --git a/pkg/sources/source_manager.go b/pkg/sources/source_manager.go index 89009947..923f0464 100644 --- a/pkg/sources/source_manager.go +++ b/pkg/sources/source_manager.go @@ -21,11 +21,18 @@ type handle int64 // initialized Source. type SourceInitFunc func(ctx context.Context, sourceID int64, jobID int64) (Source, error) +// sourceInfo is an aggregate struct to store source information provided on +// initialization. +type sourceInfo struct { + initFunc SourceInitFunc + name string +} + type SourceManager struct { api apiClient hooks []JobProgressHook // Map of handle to source initializer. - handles map[handle]SourceInitFunc + handles map[handle]sourceInfo handlesLock sync.Mutex // Pool limiting the amount of concurrent sources running. pool errgroup.Group @@ -58,8 +65,8 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) { } } -// WithConcurrency limits the concurrent number of sources a manager can run. -func WithConcurrency(concurrency int) func(*SourceManager) { +// WithConcurrentSources limits the concurrent number of sources a manager can run. +func WithConcurrentSources(concurrency int) func(*SourceManager) { return func(mgr *SourceManager) { mgr.pool.SetLimit(concurrency) } } @@ -85,7 +92,7 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager { mgr := SourceManager{ // Default to the headless API. Can be overwritten by the WithAPI option. api: &headlessAPI{}, - handles: make(map[handle]SourceInitFunc), + handles: make(map[handle]sourceInfo), outputChunks: make(chan *Chunk), } for _, opt := range opts { @@ -110,19 +117,22 @@ func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb. // TODO: smartly handle this? return 0, fmt.Errorf("handle ID '%d' already in use", handleID) } - s.handles[handleID] = f + s.handles[handleID] = sourceInfo{ + initFunc: f, + name: name, + } return handleID, nil } // Run blocks until a resource is available to run the source, then // synchronously runs it. The first fatal error, if any, will be returned. func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) { - report, err := s.asyncRun(ctx, handle) + progress, err := s.asyncRun(ctx, handle) if err != nil { - return report, err + return progress, err } - <-report.Done() - return report, report.Snapshot().FatalError() + <-progress.Done() + return progress, progress.Snapshot().FatalError() } // ScheduleRun blocks until a resource is available to run the source, then @@ -133,26 +143,30 @@ func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProg } // asyncRun is a helper method to asynchronously run the Source. It calls out -// to the API to get a job ID for this run, creates a report, then waits for an -// available goroutine to asynchronously run it. +// to the API to get a job ID for this run, creates a JobProgress object, then +// waits for an available goroutine to asynchronously run it. func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) { // Do preflight checks before waiting on the pool. if err := s.preflightChecks(ctx, handle); err != nil { return JobProgressRef{}, err } // Get a Job ID. + ctx = context.WithValue(ctx, "source_id", int64(handle)) jobID, err := s.api.GetJobID(ctx, int64(handle)) if err != nil { return JobProgressRef{SourceID: int64(handle)}, err } - // Start a report for this job. - report := NewJobProgress(int64(handle), jobID, WithHooks(s.hooks...)) + // Create a JobProgress object for tracking progress. + progress := NewJobProgress(int64(handle), jobID, WithHooks(s.hooks...)) s.pool.Go(func() error { + ctx := context.WithValues(ctx, + "job_id", jobID, + "source_manager_worker_id", common.RandomID(5), + ) defer common.Recover(ctx) - _ = s.run(ctx, handle, jobID, report) - return nil + return s.run(ctx, handle, jobID, progress) }) - return report.Ref(), nil + return progress.Ref(), nil } // Chunks returns the read only channel of all the chunks produced by all of @@ -165,17 +179,23 @@ func (s *SourceManager) Chunks() <-chan *Chunk { // returned by Chunks(). The manager should not be reused after calling this // method. This current implementation is not thread safe and should only be // called by one thread. -func (s *SourceManager) Wait() { +func (s *SourceManager) Wait() error { // Check if the manager has been Waited. if s.done { - return + return s.pool.Wait() } defer close(s.outputChunks) defer func() { s.done = true }() - // We are only using the errgroup for limiting concurrency. - // TODO: Maybe switch to using a semaphore.Weighted. - _ = s.pool.Wait() + // Return the first error returned by run. + return s.pool.Wait() +} + +// ScanChunk injects a chunk into the output stream of chunks to be scanned. +// This method should rarely be used. TODO: Remove when dependencies no longer +// rely on this functionality. +func (s *SourceManager) ScanChunk(chunk *Chunk) { + s.outputChunks <- chunk } // preflightChecks is a helper method to check the Manager or the context isn't @@ -186,7 +206,7 @@ func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) erro return fmt.Errorf("manager is done") } // Check the handle is valid. - if _, ok := s.getInitFunc(handle); !ok { + if _, ok := s.getSourceInfo(handle); !ok { return fmt.Errorf("unrecognized handle") } return ctx.Err() @@ -201,18 +221,22 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep defer func() { report.End(time.Now()) }() // Initialize the source. - initFunc, ok := s.getInitFunc(handle) + sourceInfo, ok := s.getSourceInfo(handle) if !ok { // Shouldn't happen due to preflight checks. err := fmt.Errorf("unrecognized handle") report.ReportError(Fatal{err}) return Fatal{err} } - source, err := initFunc(ctx, int64(handle), jobID) + source, err := sourceInfo.initFunc(ctx, int64(handle), jobID) if err != nil { report.ReportError(Fatal{err}) return Fatal{err} } + ctx = context.WithValues(ctx, + "source_type", source.Type().String(), + "source_name", sourceInfo.name, + ) // Check for the preferred method of tracking source units. if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits { return s.runWithUnits(ctx, handle, enumChunker, report) @@ -224,7 +248,7 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep // job reporting. func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error { // Introspect on the chunks we get from the Chunks method. - ch := make(chan *Chunk) + ch := make(chan *Chunk, 1) var wg sync.WaitGroup // Consume chunks and export chunks. wg.Add(1) @@ -232,7 +256,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour defer wg.Done() for chunk := range ch { report.ReportChunk(nil, chunk) - _ = common.CancellableWrite(ctx, s.outputChunks, chunk) + s.outputChunks <- chunk } }() // Don't return from this function until the goroutine has finished @@ -253,7 +277,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour // scanned and any errors encountered. func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error { unitReporter := &mgrUnitReporter{ - unitCh: make(chan SourceUnit), + unitCh: make(chan SourceUnit, 1), report: report, } // Create a function that will save the first error encountered (if @@ -271,6 +295,7 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source report.StartEnumerating(time.Now()) defer func() { report.EndEnumerating(time.Now()) }() defer close(unitReporter.unitCh) + ctx.Logger().V(2).Info("enumerating source") if err := source.Enumerate(ctx, unitReporter); err != nil { report.ReportError(Fatal{err}) catchFirstFatal(Fatal{err}) @@ -287,7 +312,7 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source unit := unit chunkReporter := &mgrChunkReporter{ unit: unit, - chunkCh: make(chan *Chunk), + chunkCh: make(chan *Chunk, 1), report: report, } // Consume units and produce chunks. @@ -295,6 +320,8 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source report.StartUnitChunking(unit, time.Now()) // TODO: Catch panics and add to report. defer close(chunkReporter.chunkCh) + ctx := context.WithValue(ctx, "unit", unit.SourceUnitID()) + ctx.Logger().V(3).Info("chunking unit") if err := source.ChunkUnit(ctx, unit, chunkReporter); err != nil { report.ReportError(Fatal{err}) catchFirstFatal(Fatal{err}) @@ -307,8 +334,7 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source defer wg.Done() defer func() { report.EndUnitChunking(unit, time.Now()) }() for chunk := range chunkReporter.chunkCh { - report.ReportChunk(chunkReporter.unit, chunk) - _ = common.CancellableWrite(ctx, s.outputChunks, chunk) + s.outputChunks <- chunk } }() } @@ -321,9 +347,9 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source } } -// getInitFunc is a helper method for safe concurrent access to the +// getSourceInfo is a helper method for safe concurrent access to the // map[handle]SourceInitFunc map. -func (s *SourceManager) getInitFunc(handle handle) (SourceInitFunc, bool) { +func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) { s.handlesLock.Lock() defer s.handlesLock.Unlock() f, ok := s.handles[handle] @@ -352,6 +378,7 @@ type mgrUnitReporter struct { } func (s *mgrUnitReporter) UnitOk(ctx context.Context, unit SourceUnit) error { + s.report.ReportUnit(unit) return common.CancellableWrite(ctx, s.unitCh, unit) } @@ -368,6 +395,7 @@ type mgrChunkReporter struct { } func (s *mgrChunkReporter) ChunkOk(ctx context.Context, chunk Chunk) error { + s.report.ReportChunk(s.unit, &chunk) return common.CancellableWrite(ctx, s.chunkCh, &chunk) } diff --git a/pkg/sources/source_manager_test.go b/pkg/sources/source_manager_test.go index 4f7db5f1..26626334 100644 --- a/pkg/sources/source_manager_test.go +++ b/pkg/sources/source_manager_test.go @@ -4,6 +4,8 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/assert" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "google.golang.org/protobuf/types/known/anypb" @@ -39,10 +41,14 @@ type counterChunker struct { count int } -func (c *counterChunker) Chunks(_ context.Context, ch chan *Chunk) error { +func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error { for i := 0; i < c.count; i++ { - ch <- &Chunk{Data: []byte{c.chunkCounter}} - c.chunkCounter++ + select { + case ch <- &Chunk{Data: []byte{c.chunkCounter}}: + c.chunkCounter++ + case <-ctx.Done(): + return ctx.Err() + } } return nil } @@ -98,71 +104,50 @@ func tryRead(ch <-chan *Chunk) (*Chunk, error) { func TestSourceManagerRun(t *testing.T) { mgr := NewManager(WithBufferedOutput(8)) handle, err := enrollDummy(mgr, &counterChunker{count: 1}) - if err != nil { - t.Fatalf("unexpected error enrolling source: %v", err) - } + assert.NoError(t, err) for i := 0; i < 3; i++ { - if _, err := mgr.Run(context.Background(), handle); err != nil { - t.Fatalf("unexpected error running source: %v", err) - } + _, err = mgr.Run(context.Background(), handle) + assert.NoError(t, err) chunk, err := tryRead(mgr.Chunks()) - if err != nil { - t.Fatalf("reading chunk failed: %v", err) - } - if chunk.Data[0] != byte(i) { - t.Fatalf("unexpected chunk value, wanted %v, got: %v", chunk.Data[0], i) - } - + assert.NoError(t, err) + assert.Equal(t, []byte{byte(i)}, chunk.Data) // The Chunks channel should be empty now. - if chunk, err := tryRead(mgr.Chunks()); err == nil { - t.Fatalf("unexpected chunk found: %+v", chunk) - } + _, err = tryRead(mgr.Chunks()) + assert.Error(t, err) } } func TestSourceManagerWait(t *testing.T) { mgr := NewManager() handle, err := enrollDummy(mgr, &counterChunker{count: 1}) - if err != nil { - t.Fatalf("unexpected error enrolling source: %v", err) - } + assert.NoError(t, err) // Asynchronously run the source. - if _, err := mgr.ScheduleRun(context.Background(), handle); err != nil { - t.Fatalf("unexpected error scheduling run: %v", err) - } + _, err = mgr.ScheduleRun(context.Background(), handle) + assert.NoError(t, err) // Read the 1 chunk we're expecting so Waiting completes. <-mgr.Chunks() // Wait for all resources to complete. - mgr.Wait() + assert.NoError(t, mgr.Wait()) // Enroll and run should return an error now. - if _, err := enrollDummy(mgr, &counterChunker{count: 1}); err == nil { - t.Fatalf("expected enroll to fail") - } - if _, err := mgr.ScheduleRun(context.Background(), handle); err == nil { - t.Fatalf("expected scheduling run to fail") - } + _, err = enrollDummy(mgr, &counterChunker{count: 1}) + assert.Error(t, err) + _, err = mgr.ScheduleRun(context.Background(), handle) + assert.Error(t, err) } func TestSourceManagerError(t *testing.T) { mgr := NewManager() handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")}) - if err != nil { - t.Fatalf("unexpected error enrolling source: %v", err) - } + assert.NoError(t, err) // A synchronous run should fail. - if _, err := mgr.Run(context.Background(), handle); err == nil { - t.Fatalf("expected run to fail") - } + _, err = mgr.Run(context.Background(), handle) + assert.Error(t, err) // Scheduling a run should not fail, but the error should surface in // Wait(). ref, err := mgr.ScheduleRun(context.Background(), handle) - if err != nil { - t.Fatalf("unexpected error scheduling run: %v", err) - } - <-ref.Done() - if err := ref.Snapshot().FatalError(); err == nil { - t.Fatalf("expected wait to fail") - } + assert.NoError(t, err) + assert.Error(t, mgr.Wait()) + assert.Error(t, ref.Snapshot().FatalError()) } func TestSourceManagerReport(t *testing.T) { @@ -173,19 +158,90 @@ func TestSourceManagerReport(t *testing.T) { } { mgr := NewManager(opts...) handle, err := enrollDummy(mgr, &counterChunker{count: 4}) - if err != nil { - t.Fatalf("unexpected error enrolling source: %v", err) - } + assert.NoError(t, err) // Synchronously run the source. ref, err := mgr.Run(context.Background(), handle) - if err != nil { - t.Fatalf("unexpected error running source: %v", err) - } - if errs := ref.Snapshot().Errors; len(errs) > 0 { - t.Fatalf("unexpected errors in report: %v", errs) - } - if count := ref.Snapshot().TotalChunks; count != 4 { - t.Fatalf("expected report to have 4 chunks, got: %d", count) - } + assert.NoError(t, err) + assert.Equal(t, 0, len(ref.Snapshot().Errors)) + assert.Equal(t, uint64(4), ref.Snapshot().TotalChunks) } } + +type unitChunk struct { + unit string + output string + err string +} + +type unitChunker struct{ steps []unitChunk } + +func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error { + for _, step := range c.steps { + if step.err != "" { + continue + } + if err := common.CancellableWrite(ctx, ch, &Chunk{Data: []byte(step.output)}); err != nil { + return err + } + } + return nil +} +func (c *unitChunker) Enumerate(ctx context.Context, rep UnitReporter) error { + for _, step := range c.steps { + if err := rep.UnitOk(ctx, CommonSourceUnit{step.unit}); err != nil { + return err + } + } + return nil +} +func (c *unitChunker) ChunkUnit(ctx context.Context, unit SourceUnit, rep ChunkReporter) error { + for _, step := range c.steps { + if unit.SourceUnitID() != step.unit { + continue + } + if step.err != "" { + if err := rep.ChunkErr(ctx, fmt.Errorf(step.err)); err != nil { + return err + } + continue + } + if err := rep.ChunkOk(ctx, Chunk{Data: []byte(step.output)}); err != nil { + return err + } + } + return nil +} + +func TestSourceManagerNonFatalError(t *testing.T) { + input := []unitChunk{ + {unit: "one", output: "bar"}, + {unit: "two", err: "oh no"}, + {unit: "three", err: "not again"}, + } + mgr := NewManager(WithBufferedOutput(8), WithSourceUnits()) + handle, err := enrollDummy(mgr, &unitChunker{input}) + assert.NoError(t, err) + ref, err := mgr.Run(context.Background(), handle) + assert.NoError(t, err) + report := ref.Snapshot() + assert.Equal(t, len(input), int(report.TotalUnits)) + assert.Equal(t, len(input), int(report.FinishedUnits)) + assert.Equal(t, 1, int(report.TotalChunks)) + assert.Equal(t, 2, len(report.Errors)) + assert.True(t, report.DoneEnumerating) +} + +func TestSourceManagerContextCancelled(t *testing.T) { + mgr := NewManager(WithBufferedOutput(8)) + handle, err := enrollDummy(mgr, &counterChunker{count: 100}) + assert.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + ref, err := mgr.ScheduleRun(ctx, handle) + assert.NoError(t, err) + + cancel() + <-ref.Done() + report := ref.Snapshot() + assert.Error(t, report.FatalError()) +} diff --git a/pkg/sources/sources.go b/pkg/sources/sources.go index a09975f4..17818d26 100644 --- a/pkg/sources/sources.go +++ b/pkg/sources/sources.go @@ -141,6 +141,8 @@ type GitConfig struct { BaseRef string // MaxDepth is the maximum depth to scan the source. MaxDepth int + // Bare is an indicator to handle bare repositories properly. + Bare bool // Filter is the filter to use to scan the source. Filter *common.Filter // ExcludeGlobs is a list of globs to exclude from the scan. @@ -170,6 +172,12 @@ type GithubConfig struct { IncludeRepos []string // Filter is the filter to use to scan the source. Filter *common.Filter + // IncludeIssueComments indicates whether to include GitHub issue comments in the scan. + IncludeIssueComments, + // IncludePullRequestComments indicates whether to include GitHub pull request comments in the scan. + IncludePullRequestComments, + // IncludeGistComments indicates whether to include GitHub gist comments in the scan. + IncludeGistComments bool } // GitlabConfig defines the optional configuration for a gitlab source. @@ -205,6 +213,8 @@ type S3Config struct { SessionToken string // Buckets is the list of buckets to scan. Buckets []string + // Roles is the list of Roles to use. + Roles []string // MaxObjectSize is the maximum object size to scan. MaxObjectSize int64 } diff --git a/pkg/tui/common/common.go b/pkg/tui/common/common.go new file mode 100644 index 00000000..9b8cf4a9 --- /dev/null +++ b/pkg/tui/common/common.go @@ -0,0 +1,24 @@ +package common + +import ( + "github.com/aymanbagabas/go-osc52" + zone "github.com/lrstanley/bubblezone" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/keymap" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +// Common is a struct all components should embed. +type Common struct { + Copy *osc52.Output + Styles *styles.Styles + KeyMap *keymap.KeyMap + Width int + Height int + Zone *zone.Manager +} + +// SetSize sets the width and height of the common struct. +func (c *Common) SetSize(width, height int) { + c.Width = width + c.Height = height +} diff --git a/pkg/tui/common/component.go b/pkg/tui/common/component.go new file mode 100644 index 00000000..ed8b9bf0 --- /dev/null +++ b/pkg/tui/common/component.go @@ -0,0 +1,13 @@ +package common + +import ( + "github.com/charmbracelet/bubbles/help" + tea "github.com/charmbracelet/bubbletea" +) + +// Component represents a Bubble Tea model that implements a SetSize function. +type Component interface { + tea.Model + help.KeyMap + SetSize(width, height int) +} diff --git a/pkg/tui/common/error.go b/pkg/tui/common/error.go new file mode 100644 index 00000000..fe972980 --- /dev/null +++ b/pkg/tui/common/error.go @@ -0,0 +1,13 @@ +package common + +import tea "github.com/charmbracelet/bubbletea" + +// ErrorMsg is a Bubble Tea message that represents an error. +type ErrorMsg error + +// ErrorCmd returns an ErrorMsg from error. +func ErrorCmd(err error) tea.Cmd { + return func() tea.Msg { + return ErrorMsg(err) + } +} diff --git a/pkg/tui/common/style.go b/pkg/tui/common/style.go new file mode 100644 index 00000000..8b91d9af --- /dev/null +++ b/pkg/tui/common/style.go @@ -0,0 +1,27 @@ +package common + +import ( + "github.com/charmbracelet/glamour" + gansi "github.com/charmbracelet/glamour/ansi" +) + +func strptr(s string) *string { + return &s +} + +// StyleConfig returns the default Glamour style configuration. +func StyleConfig() gansi.StyleConfig { + noColor := strptr("") + s := glamour.DarkStyleConfig + s.H1.BackgroundColor = noColor + s.H1.Prefix = "# " + s.H1.Suffix = "" + s.H1.Color = strptr("39") + s.Document.StylePrimitive.Color = noColor + s.CodeBlock.Chroma.Text.Color = noColor + s.CodeBlock.Chroma.Name.Color = noColor + // This fixes an issue with the default style config. For example + // highlighting empty spaces with red in Dockerfile type. + s.CodeBlock.Chroma.Error.BackgroundColor = noColor + return s +} diff --git a/pkg/tui/common/utils.go b/pkg/tui/common/utils.go new file mode 100644 index 00000000..a61c052e --- /dev/null +++ b/pkg/tui/common/utils.go @@ -0,0 +1,27 @@ +package common + +import ( + "strings" + + "github.com/muesli/reflow/truncate" +) + +// TruncateString is a convenient wrapper around truncate.TruncateString. +func TruncateString(s string, max int) string { + if max < 0 { + max = 0 + } + return truncate.StringWithTail(s, uint(max), "ā€¦") +} + +func SummarizeSource(keys []string, inputs map[string]string, labels map[string]string) string { + summary := strings.Builder{} + for _, key := range keys { + if inputs[key] != "" { + summary.WriteString("\t" + labels[key] + ": " + inputs[key] + "\n") + } + } + + summary.WriteString("\n") + return summary.String() +} diff --git a/pkg/tui/components/footer/footer.go b/pkg/tui/components/footer/footer.go new file mode 100644 index 00000000..68ca0842 --- /dev/null +++ b/pkg/tui/components/footer/footer.go @@ -0,0 +1,96 @@ +package footer + +import ( + "github.com/charmbracelet/bubbles/help" + "github.com/charmbracelet/bubbles/key" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" +) + +// ToggleFooterMsg is a message sent to show/hide the footer. +type ToggleFooterMsg struct{} + +// Footer is a Bubble Tea model that displays help and other info. +type Footer struct { + common common.Common + help help.Model + keymap help.KeyMap +} + +// New creates a new Footer. +func New(c common.Common, keymap help.KeyMap) *Footer { + h := help.New() + h.Styles.ShortKey = c.Styles.HelpKey + h.Styles.ShortDesc = c.Styles.HelpValue + h.Styles.FullKey = c.Styles.HelpKey + h.Styles.FullDesc = c.Styles.HelpValue + f := &Footer{ + common: c, + help: h, + keymap: keymap, + } + f.SetSize(c.Width, c.Height) + return f +} + +// SetSize implements common.Component. +func (f *Footer) SetSize(width, height int) { + f.common.SetSize(width, height) + f.help.Width = width - + f.common.Styles.Footer.GetHorizontalFrameSize() +} + +// Init implements tea.Model. +func (f *Footer) Init() tea.Cmd { + return nil +} + +// Update implements tea.Model. +func (f *Footer) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + return f, nil +} + +// View implements tea.Model. +func (f *Footer) View() string { + if f.keymap == nil { + return "" + } + s := f.common.Styles.Footer.Copy(). + Width(f.common.Width) + helpView := f.help.View(f.keymap) + return f.common.Zone.Mark( + "footer", + s.Render(helpView), + ) +} + +// ShortHelp returns the short help key bindings. +func (f *Footer) ShortHelp() []key.Binding { + return f.keymap.ShortHelp() +} + +// FullHelp returns the full help key bindings. +func (f *Footer) FullHelp() [][]key.Binding { + return f.keymap.FullHelp() +} + +// ShowAll returns whether the full help is shown. +func (f *Footer) ShowAll() bool { + return f.help.ShowAll +} + +// SetShowAll sets whether the full help is shown. +func (f *Footer) SetShowAll(show bool) { + f.help.ShowAll = show +} + +// Height returns the height of the footer. +func (f *Footer) Height() int { + return lipgloss.Height(f.View()) +} + +// ToggleFooterCmd sends a ToggleFooterMsg to show/hide the help footer. +func ToggleFooterCmd() tea.Msg { + return ToggleFooterMsg{} +} diff --git a/pkg/tui/components/formfield/formfield.go b/pkg/tui/components/formfield/formfield.go new file mode 100644 index 00000000..ab00749f --- /dev/null +++ b/pkg/tui/components/formfield/formfield.go @@ -0,0 +1,38 @@ +package formfield + +import ( + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +type FormField struct { + Label string + Required bool + Help string + Component tea.Model +} + +func NewFormField(common common.Common) *FormField { + return &FormField{} +} + +func (field *FormField) ViewLabel() string { + var label strings.Builder + if field.Required { + label.WriteString(styles.BoldTextStyle.Render(field.Label) + "*\n") + } else { + label.WriteString(styles.BoldTextStyle.Render(field.Label) + "\n") + } + + return label.String() +} + +func (field *FormField) ViewHelp() string { + var help strings.Builder + help.WriteString(styles.HintTextStyle.Render(field.Help) + "\n") + + return help.String() +} diff --git a/pkg/tui/components/header/header.go b/pkg/tui/components/header/header.go new file mode 100644 index 00000000..f83c191a --- /dev/null +++ b/pkg/tui/components/header/header.go @@ -0,0 +1,42 @@ +package header + +import ( + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" +) + +// Header represents a header component. +type Header struct { + common common.Common + text string +} + +// New creates a new header component. +func New(c common.Common, text string) *Header { + return &Header{ + common: c, + text: text, + } +} + +// SetSize implements common.Component. +func (h *Header) SetSize(width, height int) { + h.common.SetSize(width, height) +} + +// Init implements tea.Model. +func (h *Header) Init() tea.Cmd { + return nil +} + +// Update implements tea.Model. +func (h *Header) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + return h, nil +} + +// View implements tea.Model. +func (h *Header) View() string { + return h.common.Styles.ServerName.Render(strings.TrimSpace(h.text)) +} diff --git a/pkg/tui/components/selector/selector.go b/pkg/tui/components/selector/selector.go new file mode 100644 index 00000000..e76ccd91 --- /dev/null +++ b/pkg/tui/components/selector/selector.go @@ -0,0 +1,237 @@ +package selector + +import ( + "github.com/charmbracelet/bubbles/key" + "github.com/charmbracelet/bubbles/list" + tea "github.com/charmbracelet/bubbletea" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" +) + +// Selector is a list of items that can be selected. +type Selector struct { + list.Model + common common.Common + active int + filterState list.FilterState +} + +// IdentifiableItem is an item that can be identified by a string. Implements +// list.DefaultItem. +type IdentifiableItem interface { + list.DefaultItem + ID() string +} + +// ItemDelegate is a wrapper around list.ItemDelegate. +type ItemDelegate interface { + list.ItemDelegate +} + +// SelectMsg is a message that is sent when an item is selected. +type SelectMsg struct{ IdentifiableItem } + +// ActiveMsg is a message that is sent when an item is active but not selected. +type ActiveMsg struct{ IdentifiableItem } + +// New creates a new selector. +func New(common common.Common, items []IdentifiableItem, delegate ItemDelegate) *Selector { + itms := make([]list.Item, len(items)) + for i, item := range items { + itms[i] = item + } + l := list.New(itms, delegate, common.Width, common.Height) + s := &Selector{ + Model: l, + common: common, + } + s.SetSize(common.Width, common.Height) + return s +} + +// PerPage returns the number of items per page. +func (s *Selector) PerPage() int { + return s.Model.Paginator.PerPage +} + +// SetPage sets the current page. +func (s *Selector) SetPage(page int) { + s.Model.Paginator.Page = page +} + +// Page returns the current page. +func (s *Selector) Page() int { + return s.Model.Paginator.Page +} + +// TotalPages returns the total number of pages. +func (s *Selector) TotalPages() int { + return s.Model.Paginator.TotalPages +} + +// Select selects the item at the given index. +func (s *Selector) Select(index int) { + s.Model.Select(index) +} + +// SetShowTitle sets the show title flag. +func (s *Selector) SetShowTitle(show bool) { + s.Model.SetShowTitle(show) +} + +// SetShowHelp sets the show help flag. +func (s *Selector) SetShowHelp(show bool) { + s.Model.SetShowHelp(show) +} + +// SetShowStatusBar sets the show status bar flag. +func (s *Selector) SetShowStatusBar(show bool) { + s.Model.SetShowStatusBar(show) +} + +// DisableQuitKeybindings disables the quit keybindings. +func (s *Selector) DisableQuitKeybindings() { + s.Model.DisableQuitKeybindings() +} + +// SetShowFilter sets the show filter flag. +func (s *Selector) SetShowFilter(show bool) { + s.Model.SetShowFilter(show) +} + +// SetShowPagination sets the show pagination flag. +func (s *Selector) SetShowPagination(show bool) { + s.Model.SetShowPagination(show) +} + +// SetFilteringEnabled sets the filtering enabled flag. +func (s *Selector) SetFilteringEnabled(enabled bool) { + s.Model.SetFilteringEnabled(enabled) +} + +// SetSize implements common.Component. +func (s *Selector) SetSize(width, height int) { + s.common.SetSize(width, height) + s.Model.SetSize(width, height) +} + +// SetItems sets the items in the selector. +func (s *Selector) SetItems(items []IdentifiableItem) tea.Cmd { + its := make([]list.Item, len(items)) + for i, item := range items { + its[i] = item + } + return s.Model.SetItems(its) +} + +// Index returns the index of the selected item. +func (s *Selector) Index() int { + return s.Model.Index() +} + +// Init implements tea.Model. +func (s *Selector) Init() tea.Cmd { + return s.activeCmd +} + +// Update implements tea.Model. +func (s *Selector) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + cmds := make([]tea.Cmd, 0) + switch msg := msg.(type) { + case tea.MouseMsg: + switch msg.Type { + case tea.MouseWheelUp: + s.Model.CursorUp() + case tea.MouseWheelDown: + s.Model.CursorDown() + case tea.MouseLeft: + curIdx := s.Model.Index() + for i, item := range s.Model.Items() { + item, _ := item.(IdentifiableItem) + // Check each item to see if it's in bounds. + if item != nil && s.common.Zone.Get(item.ID()).InBounds(msg) { + if i == curIdx { + cmds = append(cmds, s.selectCmd) + } else { + s.Model.Select(i) + } + break + } + } + } + case tea.KeyMsg: + filterState := s.Model.FilterState() + switch { + case key.Matches(msg, s.common.KeyMap.Help): + if filterState == list.Filtering { + return s, tea.Batch(cmds...) + } + case key.Matches(msg, s.common.KeyMap.Select): + if filterState != list.Filtering { + cmds = append(cmds, s.selectCmd) + } + } + case list.FilterMatchesMsg: + cmds = append(cmds, s.activeFilterCmd) + } + m, cmd := s.Model.Update(msg) + s.Model = m + if cmd != nil { + cmds = append(cmds, cmd) + } + // Track filter state and update active item when filter state changes. + filterState := s.Model.FilterState() + if s.filterState != filterState { + cmds = append(cmds, s.activeFilterCmd) + } + s.filterState = filterState + // Send ActiveMsg when index change. + if s.active != s.Model.Index() { + cmds = append(cmds, s.activeCmd) + } + s.active = s.Model.Index() + return s, tea.Batch(cmds...) +} + +// View implements tea.Model. +func (s *Selector) View() string { + return s.Model.View() +} + +// SelectItem is a command that selects the currently active item. +func (s *Selector) SelectItem() tea.Msg { + return s.selectCmd() +} + +func (s *Selector) selectCmd() tea.Msg { + item := s.Model.SelectedItem() + i, ok := item.(IdentifiableItem) + if !ok { + return SelectMsg{} + } + return SelectMsg{i} +} + +func (s *Selector) activeCmd() tea.Msg { + item := s.Model.SelectedItem() + i, ok := item.(IdentifiableItem) + if !ok { + return ActiveMsg{} + } + return ActiveMsg{i} +} + +func (s *Selector) activeFilterCmd() tea.Msg { + // Here we use VisibleItems because when list.FilterMatchesMsg is sent, + // VisibleItems is the only way to get the list of filtered items. The list + // bubble should export something like list.FilterMatchesMsg.Items(). + items := s.Model.VisibleItems() + if len(items) == 0 { + return nil + } + item := items[0] + i, ok := item.(IdentifiableItem) + if !ok { + return nil + } + return ActiveMsg{i} +} diff --git a/pkg/tui/components/statusbar/statusbar.go b/pkg/tui/components/statusbar/statusbar.go new file mode 100644 index 00000000..a486784b --- /dev/null +++ b/pkg/tui/components/statusbar/statusbar.go @@ -0,0 +1,88 @@ +package statusbar + +import ( + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/muesli/reflow/truncate" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" +) + +// StatusBarMsg is a message sent to the status bar. +type StatusBarMsg struct { + Key string + Value string + Info string + Branch string +} + +// StatusBar is a status bar model. +type StatusBar struct { + common common.Common + msg StatusBarMsg +} + +// Model is an interface that supports setting the status bar information. +type Model interface { + StatusBarValue() string + StatusBarInfo() string +} + +// New creates a new status bar component. +func New(c common.Common) *StatusBar { + s := &StatusBar{ + common: c, + } + return s +} + +// SetSize implements common.Component. +func (s *StatusBar) SetSize(width, height int) { + s.common.Width = width + s.common.Height = height +} + +// Init implements tea.Model. +func (s *StatusBar) Init() tea.Cmd { + return nil +} + +// Update implements tea.Model. +func (s *StatusBar) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case StatusBarMsg: + s.msg = msg + } + return s, nil +} + +// View implements tea.Model. +func (s *StatusBar) View() string { + st := s.common.Styles + w := lipgloss.Width + help := s.common.Zone.Mark( + "repo-help", + st.StatusBarHelp.Render("? Help"), + ) + key := st.StatusBarKey.Render(s.msg.Key) + info := "" + if s.msg.Info != "" { + info = st.StatusBarInfo.Render(s.msg.Info) + } + branch := st.StatusBarBranch.Render(s.msg.Branch) + maxWidth := s.common.Width - w(key) - w(info) - w(branch) - w(help) + v := truncate.StringWithTail(s.msg.Value, uint(maxWidth-st.StatusBarValue.GetHorizontalFrameSize()), "ā€¦") + value := st.StatusBarValue. + Width(maxWidth). + Render(v) + + return lipgloss.NewStyle().MaxWidth(s.common.Width). + Render( + lipgloss.JoinHorizontal(lipgloss.Top, + key, + value, + info, + branch, + help, + ), + ) +} diff --git a/pkg/tui/components/tabs/tabs.go b/pkg/tui/components/tabs/tabs.go new file mode 100644 index 00000000..b3ad0ae1 --- /dev/null +++ b/pkg/tui/components/tabs/tabs.go @@ -0,0 +1,122 @@ +package tabs + +import ( + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" +) + +// SelectTabMsg is a message that contains the index of the tab to select. +type SelectTabMsg int + +// ActiveTabMsg is a message that contains the index of the current active tab. +type ActiveTabMsg int + +// Tabs is bubbletea component that displays a list of tabs. +type Tabs struct { + common common.Common + tabs []string + activeTab int + TabSeparator lipgloss.Style + TabInactive lipgloss.Style + TabActive lipgloss.Style + TabDot lipgloss.Style + UseDot bool +} + +// New creates a new Tabs component. +func New(c common.Common, tabs []string) *Tabs { + r := &Tabs{ + common: c, + tabs: tabs, + activeTab: 0, + TabSeparator: c.Styles.TabSeparator, + TabInactive: c.Styles.TabInactive, + TabActive: c.Styles.TabActive, + } + return r +} + +// SetSize implements common.Component. +func (t *Tabs) SetSize(width, height int) { + t.common.SetSize(width, height) +} + +// Init implements tea.Model. +func (t *Tabs) Init() tea.Cmd { + t.activeTab = 0 + return nil +} + +// Update implements tea.Model. +func (t *Tabs) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + cmds := make([]tea.Cmd, 0) + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.String() { + case "tab": + t.activeTab = (t.activeTab + 1) % len(t.tabs) + cmds = append(cmds, t.activeTabCmd) + case "shift+tab": + t.activeTab = (t.activeTab - 1 + len(t.tabs)) % len(t.tabs) + cmds = append(cmds, t.activeTabCmd) + } + case tea.MouseMsg: + if msg.Type == tea.MouseLeft { + for i, tab := range t.tabs { + if t.common.Zone.Get(tab).InBounds(msg) { + t.activeTab = i + cmds = append(cmds, t.activeTabCmd) + } + } + } + case SelectTabMsg: + tab := int(msg) + if tab >= 0 && tab < len(t.tabs) { + t.activeTab = int(msg) + } + } + return t, tea.Batch(cmds...) +} + +// View implements tea.Model. +func (t *Tabs) View() string { + s := strings.Builder{} + sep := t.TabSeparator + for i, tab := range t.tabs { + style := t.TabInactive.Copy() + prefix := " " + if i == t.activeTab { + style = t.TabActive.Copy() + prefix = t.TabDot.Render("ā€¢ ") + } + if t.UseDot { + s.WriteString(prefix) + } + s.WriteString( + t.common.Zone.Mark( + tab, + style.Render(tab), + ), + ) + if i != len(t.tabs)-1 { + s.WriteString(sep.String()) + } + } + return lipgloss.NewStyle(). + MaxWidth(t.common.Width). + Render(s.String()) +} + +func (t *Tabs) activeTabCmd() tea.Msg { + return ActiveTabMsg(t.activeTab) +} + +// SelectTabCmd is a bubbletea command that selects the tab at the given index. +func SelectTabCmd(tab int) tea.Cmd { + return func() tea.Msg { + return SelectTabMsg(tab) + } +} diff --git a/pkg/tui/components/textinput/textinput.go b/pkg/tui/components/textinput/textinput.go new file mode 100644 index 00000000..14a1675c --- /dev/null +++ b/pkg/tui/components/textinput/textinput.go @@ -0,0 +1,51 @@ +package textinput + +import ( + "github.com/charmbracelet/bubbles/textinput" + tea "github.com/charmbracelet/bubbletea" +) + +type ( + errMsg error +) + +type TextInput struct { + textInput textinput.Model + err error +} + +func New(placeholder string) TextInput { + ti := textinput.New() + ti.Placeholder = placeholder + ti.Focus() + ti.CharLimit = 156 + ti.Width = 60 + + return TextInput{ + textInput: ti, + err: nil, + } +} + +func (m TextInput) Init() tea.Cmd { + return textinput.Blink +} + +func (m TextInput) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var cmd tea.Cmd + + switch msg := msg.(type) { + + // We handle errors just like any other message + case errMsg: + m.err = msg + return m, nil + } + + m.textInput, cmd = m.textInput.Update(msg) + return m, cmd +} + +func (m TextInput) View() string { + return m.textInput.View() +} diff --git a/pkg/tui/components/textinputs/textinputs.go b/pkg/tui/components/textinputs/textinputs.go new file mode 100644 index 00000000..d0e68c6b --- /dev/null +++ b/pkg/tui/components/textinputs/textinputs.go @@ -0,0 +1,229 @@ +package textinputs + +// from https://github.com/charmbracelet/bubbletea/blob/master/examples/textinputs/main.go + +import ( + "fmt" + "strings" + + "github.com/charmbracelet/bubbles/textinput" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +var ( + focusedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("205")) + blurredStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("240")) + noStyle = lipgloss.NewStyle() + helpStyle = blurredStyle.Copy() + // cursorStyle = focusedStyle.Copy() + // cursorModeHelpStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("244")) + + focusedButton = focusedStyle.Copy().Render("[ Next ]") + blurredButton = fmt.Sprintf("[ %s ]", blurredStyle.Render("Next")) + focusedSkipButton = lipgloss.NewStyle().Foreground(lipgloss.Color("205")).Render("[ Run with defaults ]") + blurredSkipButton = fmt.Sprintf("[ %s ]", lipgloss.NewStyle().Foreground(lipgloss.Color("240")).Render("Run with defaults")) +) + +// SelectNextMsg used for emitting events when the 'Next' button is selected. +type SelectNextMsg int + +// SelectSkipMsg used for emitting events when the 'Skip' button is selected. +type SelectSkipMsg int + +type Model struct { + focusIndex int + inputs []textinput.Model + configs []InputConfig + // cursorMode cursor.Mode + skipButton bool +} + +type InputConfig struct { + Label string + Key string + Help string + Required bool + Placeholder string +} + +func (m Model) GetInputs() map[string]string { + inputs := make(map[string]string) + + for i, input := range m.inputs { + inputs[m.configs[i].Key] = input.Value() + } + + return inputs +} + +func (m Model) GetLabels() map[string]string { + labels := make(map[string]string) + + for _, config := range m.configs { + labels[config.Key] = config.Label + } + + return labels +} + +func New(config []InputConfig) Model { + m := Model{ + inputs: make([]textinput.Model, len(config)), + } + + for i, conf := range config { + input := textinput.New() + input.Placeholder = conf.Placeholder + + if i == 0 { + input.Focus() + input.TextStyle = focusedStyle + input.PromptStyle = focusedStyle + } + + m.inputs[i] = input + } + + m.configs = config + return m +} + +func (m Model) Init() tea.Cmd { + return textinput.Blink +} + +func (m Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.String() { + // Set focus to next input + case "enter", "up", "down": + s := msg.String() + + // Did the user press enter while the submit or skip button was focused? + // If so, emit the appropriate command. + if s == "enter" && m.focusIndex == len(m.inputs) { + return m, func() tea.Msg { return SelectNextMsg(0) } + } else if s == "enter" && m.focusIndex == -1 { + return m, func() tea.Msg { return SelectSkipMsg(0) } + } + + // Cycle indexes + if s == "up" { + m.focusIndex-- + } else { + m.focusIndex++ + } + + if m.focusIndex > len(m.inputs) { + m.focusIndex = 0 + } else if !m.skipButton && m.focusIndex < 0 { + m.focusIndex = len(m.inputs) + } else if m.skipButton && m.focusIndex < -1 { + m.focusIndex = len(m.inputs) + } + + cmds := make([]tea.Cmd, len(m.inputs)) + for i := 0; i < len(m.inputs); i++ { + if i == m.focusIndex { + // Set focused state + cmds[i] = m.focusInput(i) + continue + } + // Remove focused state + m.unfocusInput(i) + } + + return m, tea.Batch(cmds...) + } + } + + // Handle character input and blinking + cmd := m.updateInputs(msg) + + return m, cmd +} + +func (m *Model) updateInputs(msg tea.Msg) tea.Cmd { + cmds := make([]tea.Cmd, len(m.inputs)) + + // Only text inputs with Focus() set will respond, so it's safe to simply + // update all of them here without any further logic. + for i := range m.inputs { + m.inputs[i], cmds[i] = m.inputs[i].Update(msg) + } + + return tea.Batch(cmds...) +} + +func (m Model) View() string { + var b strings.Builder + + if m.skipButton { + button := &blurredSkipButton + if m.focusIndex == -1 { + button = &focusedSkipButton + } + fmt.Fprintf(&b, "%s\n\n\n", *button) + } + + for i := range m.inputs { + if m.configs[i].Label != "" { + b.WriteString(m.GetLabel(m.configs[i])) + } + + b.WriteString(m.inputs[i].View()) + b.WriteRune('\n') + if i < len(m.inputs)-1 { + b.WriteRune('\n') + } + } + + button := &blurredButton + if m.focusIndex == len(m.inputs) { + button = &focusedButton + } + fmt.Fprintf(&b, "\n\n%s\n\n", *button) + + return b.String() +} + +func (m Model) GetLabel(c InputConfig) string { + var label strings.Builder + + label.WriteString(c.Label) + if c.Required { + label.WriteString("*") + } + + if len(c.Help) > 0 { + label.WriteString("\n" + helpStyle.Render(c.Help)) + } + + label.WriteString("\n") + return label.String() +} + +func (m Model) SetSkip(skip bool) Model { + m.skipButton = skip + if m.skipButton { + if len(m.inputs) > 0 { + m.unfocusInput(0) + } + m.focusIndex = -1 + } + return m +} + +func (m *Model) unfocusInput(index int) { + m.inputs[index].Blur() + m.inputs[index].PromptStyle = noStyle + m.inputs[index].TextStyle = noStyle +} + +func (m *Model) focusInput(index int) tea.Cmd { + m.inputs[index].PromptStyle = focusedStyle + m.inputs[index].TextStyle = focusedStyle + return m.inputs[index].Focus() +} diff --git a/pkg/tui/components/viewport/viewport.go b/pkg/tui/components/viewport/viewport.go new file mode 100644 index 00000000..77fa6182 --- /dev/null +++ b/pkg/tui/components/viewport/viewport.go @@ -0,0 +1,97 @@ +package viewport + +import ( + "github.com/charmbracelet/bubbles/viewport" + tea "github.com/charmbracelet/bubbletea" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" +) + +// Viewport represents a viewport component. +type Viewport struct { + common common.Common + *viewport.Model +} + +// New returns a new Viewport. +func New(c common.Common) *Viewport { + vp := viewport.New(c.Width, c.Height) + vp.MouseWheelEnabled = true + return &Viewport{ + common: c, + Model: &vp, + } +} + +// SetSize implements common.Component. +func (v *Viewport) SetSize(width, height int) { + v.common.SetSize(width, height) + v.Model.Width = width + v.Model.Height = height +} + +// Init implements tea.Model. +func (v *Viewport) Init() tea.Cmd { + return nil +} + +// Update implements tea.Model. +func (v *Viewport) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + vp, cmd := v.Model.Update(msg) + v.Model = &vp + return v, cmd +} + +// View implements tea.Model. +func (v *Viewport) View() string { + return v.Model.View() +} + +// SetContent sets the viewport's content. +func (v *Viewport) SetContent(content string) { + v.Model.SetContent(content) +} + +// GotoTop moves the viewport to the top of the log. +func (v *Viewport) GotoTop() { + v.Model.GotoTop() +} + +// GotoBottom moves the viewport to the bottom of the log. +func (v *Viewport) GotoBottom() { + v.Model.GotoBottom() +} + +// HalfViewDown moves the viewport down by half the viewport height. +func (v *Viewport) HalfViewDown() { + v.Model.HalfViewDown() +} + +// HalfViewUp moves the viewport up by half the viewport height. +func (v *Viewport) HalfViewUp() { + v.Model.HalfViewUp() +} + +// ViewUp moves the viewport up by a page. +func (v *Viewport) ViewUp() []string { + return v.Model.ViewUp() +} + +// ViewDown moves the viewport down by a page. +func (v *Viewport) ViewDown() []string { + return v.Model.ViewDown() +} + +// LineUp moves the viewport up by the given number of lines. +func (v *Viewport) LineUp(n int) []string { + return v.Model.LineUp(n) +} + +// LineDown moves the viewport down by the given number of lines. +func (v *Viewport) LineDown(n int) []string { + return v.Model.LineDown(n) +} + +// ScrollPercent returns the viewport's scroll percentage. +func (v *Viewport) ScrollPercent() float64 { + return v.Model.ScrollPercent() +} diff --git a/pkg/tui/keymap/keymap.go b/pkg/tui/keymap/keymap.go new file mode 100644 index 00000000..bb3cb8c2 --- /dev/null +++ b/pkg/tui/keymap/keymap.go @@ -0,0 +1,217 @@ +package keymap + +import "github.com/charmbracelet/bubbles/key" + +// KeyMap is a map of key bindings for the UI. +type KeyMap struct { + Quit key.Binding + CmdQuit key.Binding + Up key.Binding + Down key.Binding + UpDown key.Binding + LeftRight key.Binding + Arrows key.Binding + Select key.Binding + Section key.Binding + Back key.Binding + PrevPage key.Binding + NextPage key.Binding + Help key.Binding + + SelectItem key.Binding + BackItem key.Binding + + Copy key.Binding +} + +// DefaultKeyMap returns the default key map. +func DefaultKeyMap() *KeyMap { + km := new(KeyMap) + + km.Quit = key.NewBinding( + key.WithKeys( + "ctrl+c", + ), + key.WithHelp( + "ctrl+c", + "quit", + ), + ) + + km.CmdQuit = key.NewBinding( + key.WithKeys( + "q", + "ctrl+c", + ), + key.WithHelp( + "q", + "quit", + ), + ) + + km.Up = key.NewBinding( + key.WithKeys( + "up", + "k", + ), + key.WithHelp( + "ā†‘/k", + "up", + ), + ) + + km.Down = key.NewBinding( + key.WithKeys( + "down", + "j", + ), + key.WithHelp( + "ā†“/j", + "down", + ), + ) + + km.UpDown = key.NewBinding( + key.WithKeys( + "up", + "down", + "k", + "j", + ), + key.WithHelp( + "ā†‘ā†“", + "navigate", + ), + ) + + km.LeftRight = key.NewBinding( + key.WithKeys( + "left", + "h", + "right", + "l", + ), + key.WithHelp( + "ā†ā†’", + "navigate", + ), + ) + + km.Arrows = key.NewBinding( + key.WithKeys( + "up", + "right", + "down", + "left", + "k", + "j", + "h", + "l", + ), + key.WithHelp( + "ā†‘ā†ā†“ā†’", + "navigate", + ), + ) + + km.Select = key.NewBinding( + key.WithKeys( + "enter", + ), + key.WithHelp( + "enter", + "select", + ), + ) + + km.Section = key.NewBinding( + key.WithKeys( + "tab", + "shift+tab", + ), + key.WithHelp( + "tab", + "section", + ), + ) + + km.Back = key.NewBinding( + key.WithKeys( + "esc", + ), + key.WithHelp( + "esc", + "back", + ), + ) + + km.PrevPage = key.NewBinding( + key.WithKeys( + "pgup", + "b", + "u", + ), + key.WithHelp( + "pgup", + "prev page", + ), + ) + + km.NextPage = key.NewBinding( + key.WithKeys( + "pgdown", + "f", + "d", + ), + key.WithHelp( + "pgdn", + "next page", + ), + ) + + km.Help = key.NewBinding( + key.WithKeys( + "?", + ), + key.WithHelp( + "?", + "toggle help", + ), + ) + + km.SelectItem = key.NewBinding( + key.WithKeys( + "l", + "right", + ), + key.WithHelp( + "ā†’", + "select", + ), + ) + + km.BackItem = key.NewBinding( + key.WithKeys( + "h", + "left", + "backspace", + ), + key.WithHelp( + "ā†", + "back", + ), + ) + + km.Copy = key.NewBinding( + key.WithKeys( + "c", + "ctrl+c", + ), + key.WithHelp( + "c", + "copy text", + ), + ) + + return km +} diff --git a/pkg/tui/pages/contact_enterprise/contact_enterprise.go b/pkg/tui/pages/contact_enterprise/contact_enterprise.go new file mode 100644 index 00000000..4f345e0c --- /dev/null +++ b/pkg/tui/pages/contact_enterprise/contact_enterprise.go @@ -0,0 +1,60 @@ +package contact_enterprise + +import ( + "strings" + + "github.com/charmbracelet/bubbles/key" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +type ContactEnterprise struct { + common.Common + viewed bool +} + +var ( + linkStyle = lipgloss.NewStyle().Foreground( + lipgloss.Color("28")) // green +) + +func New(c common.Common) *ContactEnterprise { + return &ContactEnterprise{ + Common: c, + viewed: false, + } +} + +func (m *ContactEnterprise) Init() tea.Cmd { + return nil +} + +func (m *ContactEnterprise) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + if m.viewed { + return m, tea.Quit + } + + return m, func() tea.Msg { return nil } +} + +func (m *ContactEnterprise) View() string { + + s := strings.Builder{} + s.WriteString("Interested in Trufflehog enterprise?\n") + s.WriteString(linkStyle.Render("šŸ”— https://trufflesecurity.com/contact")) + + m.viewed = true + return styles.AppStyle.Render(s.String()) +} + +func (m *ContactEnterprise) ShortHelp() []key.Binding { + // TODO: actually return something + return nil +} + +func (m *ContactEnterprise) FullHelp() [][]key.Binding { + // TODO: actually return something + return nil +} diff --git a/pkg/tui/pages/source_configure/item.go b/pkg/tui/pages/source_configure/item.go new file mode 100644 index 00000000..70110b67 --- /dev/null +++ b/pkg/tui/pages/source_configure/item.go @@ -0,0 +1,23 @@ +package source_configure + +type Item struct { + title string + description string +} + +func (i Item) ID() string { return i.title } + +func (i Item) Title() string { + return i.title +} +func (i Item) Description() string { + return i.description +} + +func (i Item) SetDescription(d string) Item { + i.description = d + return i +} + +// We shouldn't be filtering for these list items. +func (i Item) FilterValue() string { return "" } diff --git a/pkg/tui/pages/source_configure/run_component.go b/pkg/tui/pages/source_configure/run_component.go new file mode 100644 index 00000000..db700259 --- /dev/null +++ b/pkg/tui/pages/source_configure/run_component.go @@ -0,0 +1,118 @@ +package source_configure + +import ( + "strings" + + "github.com/charmbracelet/bubbles/key" + "github.com/charmbracelet/bubbles/list" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +type SetArgsMsg string + +type RunComponent struct { + common.Common + parent *SourceConfigure + reviewList list.Model + reviewListItems []list.Item +} + +func NewRunComponent(common common.Common, parent *SourceConfigure) *RunComponent { + // Make list of SourceItems. + listItems := []list.Item{ + Item{title: "šŸ”Ž Source configuration"}, + Item{title: "šŸ½ TruffleHog configuration"}, + Item{title: "šŸ’ø Sales pitch", description: "\tContinuous monitoring, state tracking, remediations, and more\n\tšŸ”— https://trufflesecurity.com/trufflehog"}, + } + + // Setup list + delegate := list.NewDefaultDelegate() + delegate.Styles.SelectedTitle.Foreground(lipgloss.Color("white")) + delegate.Styles.SelectedDesc.Foreground(lipgloss.Color("white")) + delegate.SetHeight(3) + + reviewList := list.New(listItems, delegate, common.Width, common.Height) + + reviewList.SetShowTitle(false) + reviewList.SetShowStatusBar(false) + reviewList.SetFilteringEnabled(false) + + return &RunComponent{ + Common: common, + parent: parent, + reviewList: reviewList, + reviewListItems: listItems, + } +} + +func (m *RunComponent) Init() tea.Cmd { + return nil +} + +func (m *RunComponent) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + h, v := styles.AppStyle.GetFrameSize() + m.reviewList.SetSize(msg.Width-h, msg.Height/2-v) + case tea.KeyMsg: + if msg.Type == tea.KeyEnter { + command := m.parent.sourceFields.Cmd() + if m.parent.truffleFields.Cmd() != "" { + command += " " + m.parent.truffleFields.Cmd() + } + cmd := func() tea.Msg { return SetArgsMsg(command) } + return m, cmd + } + } + if len(m.reviewListItems) > 0 && m.parent != nil && m.parent.sourceFields != nil { + m.reviewListItems[0] = m.reviewListItems[0].(Item).SetDescription(m.parent.sourceFields.Summary()) + m.reviewListItems[1] = m.reviewListItems[1].(Item).SetDescription(m.parent.truffleFields.Summary()) + } + var cmd tea.Cmd + m.reviewList, cmd = m.reviewList.Update(msg) + return m, tea.Batch(cmd) +} + +func (m *RunComponent) View() string { + var view strings.Builder + + view.WriteString("\nšŸ”Ž Source configuration\n") + view.WriteString(m.parent.sourceFields.Summary()) + + view.WriteString("\nšŸ½ TruffleHog configuration\n") + view.WriteString(m.parent.truffleFields.Summary()) + + view.WriteString("\nšŸ’ø Sales pitch\n") + view.WriteString("\tContinuous monitoring, state tracking, remediations, and more\n") + view.WriteString("\tšŸ”— https://trufflesecurity.com/trufflehog\n\n") + + view.WriteString(styles.BoldTextStyle.Render("\n\nšŸ· Run TruffleHog for "+m.parent.configTabSource) + " šŸ·\n\n") + + view.WriteString("Generated TruffleHog command\n") + view.WriteString(styles.HintTextStyle.Render("Save this if you want to run it again later!") + "\n") + + command := m.parent.sourceFields.Cmd() + if m.parent.truffleFields.Cmd() != "" { + command += " " + m.parent.truffleFields.Cmd() + } + view.WriteString(styles.CodeTextStyle.Render(command)) + + focusedStyle := lipgloss.NewStyle().Foreground(lipgloss.Color("205")) + view.WriteString("\n\n" + focusedStyle.Render("[ Run TruffleHog ]") + "\n\n") + + // view.WriteString(m.reviewList.View()) + return view.String() +} + +func (m *RunComponent) ShortHelp() []key.Binding { + // TODO: actually return something + return nil +} + +func (m *RunComponent) FullHelp() [][]key.Binding { + // TODO: actually return something + return nil +} diff --git a/pkg/tui/pages/source_configure/source_component.go b/pkg/tui/pages/source_configure/source_component.go new file mode 100644 index 00000000..f5ad69e3 --- /dev/null +++ b/pkg/tui/pages/source_configure/source_component.go @@ -0,0 +1,71 @@ +package source_configure + +import ( + "strings" + + "github.com/charmbracelet/bubbles/key" + tea "github.com/charmbracelet/bubbletea" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +type SourceComponent struct { + common.Common + parent *SourceConfigure + form tea.Model +} + +func NewSourceComponent(common common.Common, parent *SourceConfigure) *SourceComponent { + return &SourceComponent{ + Common: common, + parent: parent, + } +} + +func (m *SourceComponent) SetForm(form tea.Model) { + m.form = form +} + +func (m *SourceComponent) Init() tea.Cmd { + return nil +} + +func (m *SourceComponent) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + // TODO: Add a focus variable. + if m.form != nil { + model, cmd := m.form.Update(msg) + m.form = model + return m, cmd + } + return m, nil +} + +func (m *SourceComponent) View() string { + var view strings.Builder + + view.WriteString(styles.BoldTextStyle.Render("\nConfiguring "+styles.PrimaryTextStyle.Render(m.parent.configTabSource)) + "\n") + + view.WriteString(styles.HintTextStyle.Render("* required field") + "\n\n") + + sourceNote := sources.GetSourceNotes(m.parent.configTabSource) + if len(sourceNote) > 0 { + view.WriteString("ā­ " + sourceNote + " ā­\n\n") + } + + if m.form != nil { + view.WriteString(m.form.View()) + view.WriteString("\n") + } + return view.String() +} + +func (m *SourceComponent) ShortHelp() []key.Binding { + // TODO: actually return something + return nil +} + +func (m *SourceComponent) FullHelp() [][]key.Binding { + // TODO: actually return something + return nil +} diff --git a/pkg/tui/pages/source_configure/source_configure.go b/pkg/tui/pages/source_configure/source_configure.go new file mode 100644 index 00000000..d543a3f0 --- /dev/null +++ b/pkg/tui/pages/source_configure/source_configure.go @@ -0,0 +1,139 @@ +package source_configure + +import ( + "github.com/charmbracelet/bubbles/key" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/tabs" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources" +) + +type SetSourceMsg struct { + Source string +} + +type tab int + +const ( + configTab tab = iota + truffleConfigTab + runTab +) + +func (t tab) String() string { + return []string{ + "1. Source Configuration", + "2. TruffleHog Configuration", + "3. Run", + }[t] +} + +type SourceConfigure struct { + common.Common + activeTab tab + tabs *tabs.Tabs + configTabSource string + tabComponents []common.Component + sourceFields sources.CmdModel + truffleFields sources.CmdModel +} + +func (m SourceConfigure) Init() tea.Cmd { + return m.tabs.Init() +} + +func New(c common.Common) *SourceConfigure { + conf := SourceConfigure{Common: c, truffleFields: GetTrufflehogConfiguration()} + conf.tabs = tabs.New(c, []string{configTab.String(), truffleConfigTab.String(), runTab.String()}) + + conf.tabComponents = []common.Component{ + configTab: NewSourceComponent(c, &conf), + truffleConfigTab: NewTrufflehogComponent(c, &conf), + runTab: NewRunComponent(c, &conf), + } + return &conf +} + +func (m *SourceConfigure) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var cmds []tea.Cmd + + switch msg := msg.(type) { + case tea.WindowSizeMsg: + for i := range m.tabComponents { + model, cmd := m.tabComponents[i].Update(msg) + m.tabComponents[i] = model.(common.Component) + cmds = append(cmds, cmd) + } + + case tabs.ActiveTabMsg: + m.activeTab = tab(msg) + t, cmd := m.tabs.Update(msg) + m.tabs = t.(*tabs.Tabs) + + if cmd != nil { + cmds = append(cmds, cmd) + } + case tabs.SelectTabMsg: + m.activeTab = tab(msg) + t, cmd := m.tabs.Update(msg) + m.tabs = t.(*tabs.Tabs) + + if cmd != nil { + cmds = append(cmds, cmd) + } + case tea.KeyMsg: + t, cmd := m.tabs.Update(msg) + m.tabs = t.(*tabs.Tabs) + if cmd != nil { + cmds = append(cmds, cmd) + } + case SetSourceMsg: + m.configTabSource = msg.Source + // TODO: Use actual messages or something? + m.tabComponents[truffleConfigTab].(*TrufflehogComponent).SetForm(m.truffleFields) + fields := sources.GetSourceFields(m.configTabSource) + + if fields != nil { + m.sourceFields = fields + m.tabComponents[configTab].(*SourceComponent).SetForm(fields) + } + + case textinputs.SelectNextMsg, textinputs.SelectSkipMsg: + if m.activeTab < runTab { + m.activeTab++ + } + t, cmd := m.tabs.Update(tabs.SelectTabMsg(int(m.activeTab))) + m.tabs = t.(*tabs.Tabs) + + if cmd != nil { + cmds = append(cmds, cmd) + } + } + + tab, cmd := m.tabComponents[m.activeTab].Update(msg) + m.tabComponents[m.activeTab] = tab.(common.Component) + if cmd != nil { + cmds = append(cmds, cmd) + } + + return m, tea.Batch(cmds...) +} + +func (m *SourceConfigure) View() string { + return lipgloss.JoinVertical(lipgloss.Top, + m.tabs.View(), + m.tabComponents[m.activeTab].View(), + ) +} + +func (m *SourceConfigure) ShortHelp() []key.Binding { + // TODO: actually return something + return nil +} + +func (m *SourceConfigure) FullHelp() [][]key.Binding { + // TODO: actually return something + return nil +} diff --git a/pkg/tui/pages/source_configure/trufflehog_component.go b/pkg/tui/pages/source_configure/trufflehog_component.go new file mode 100644 index 00000000..13c8a56b --- /dev/null +++ b/pkg/tui/pages/source_configure/trufflehog_component.go @@ -0,0 +1,66 @@ +package source_configure + +import ( + "strings" + + "github.com/charmbracelet/bubbles/key" + tea "github.com/charmbracelet/bubbletea" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +type TrufflehogComponent struct { + common.Common + parent *SourceConfigure + form tea.Model +} + +func NewTrufflehogComponent(common common.Common, parent *SourceConfigure) *TrufflehogComponent { + return &TrufflehogComponent{ + Common: common, + parent: parent, + } +} + +func (m *TrufflehogComponent) SetForm(form tea.Model) { + m.form = form +} + +func (m *TrufflehogComponent) Init() tea.Cmd { + return nil +} + +func (m *TrufflehogComponent) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + // TODO: Add a focus variable. + if m.form != nil { + model, cmd := m.form.Update(msg) + m.form = model + + return m, cmd + } + return m, nil +} + +func (m *TrufflehogComponent) View() string { + var view strings.Builder + + view.WriteString(styles.BoldTextStyle.Render("\nConfiguring "+styles.PrimaryTextStyle.Render("TruffleHog")) + "\n") + view.WriteString(styles.HintTextStyle.Render("You can skip this completely and run with defaults") + "\n\n") + + if m.form != nil { + view.WriteString(m.form.View()) + view.WriteString("\n") + } + + return view.String() +} + +func (m *TrufflehogComponent) ShortHelp() []key.Binding { + // TODO: actually return something + return nil +} + +func (m *TrufflehogComponent) FullHelp() [][]key.Binding { + // TODO: actually return something + return nil +} diff --git a/pkg/tui/pages/source_configure/trufflehog_configure.go b/pkg/tui/pages/source_configure/trufflehog_configure.go new file mode 100644 index 00000000..4d17efd8 --- /dev/null +++ b/pkg/tui/pages/source_configure/trufflehog_configure.go @@ -0,0 +1,115 @@ +package source_configure + +import ( + "strconv" + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type truffleCmdModel struct { + textinputs.Model +} + +func GetTrufflehogConfiguration() truffleCmdModel { + verification := textinputs.InputConfig{ + Label: "Skip Verification", + Key: "no-verification", + Required: false, + Help: "Check if a suspected secret is real or not", + Placeholder: "false", + } + + verifiedResults := textinputs.InputConfig{ + Label: "Verified results", + Key: "only-verified", + Required: false, + Help: "Return only verified results", + Placeholder: "false", + } + + jsonOutput := textinputs.InputConfig{ + Label: "JSON output", + Key: "json", + Required: false, + Help: "Output results to JSON", + Placeholder: "false", + } + + excludeDetectors := textinputs.InputConfig{ + Label: "Exclude detectors", + Key: "exclude_detectors", + Required: false, + Help: "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.", + Placeholder: "", + } + + concurrency := textinputs.InputConfig{ + Label: "Concurrency", + Key: "concurrency", + Required: false, + Help: "Number of concurrent workers.", + Placeholder: "1", + } + + return truffleCmdModel{textinputs.New([]textinputs.InputConfig{jsonOutput, verification, verifiedResults, excludeDetectors, concurrency}).SetSkip(true)} +} + +func (m truffleCmdModel) Cmd() string { + var command []string + inputs := m.GetInputs() + + if isTrue(inputs["json"]) { + command = append(command, "--json") + } + + if isTrue(inputs["no-verification"]) { + command = append(command, "--no-verification") + } + + if isTrue(inputs["only-verified"]) { + command = append(command, "--only-verified") + } + + if inputs["exclude_detectors"] != "" { + cmd := "--exclude-detectors=" + strings.ReplaceAll(inputs["exclude_detectors"], " ", "") + command = append(command, cmd) + } + + if inputs["concurrency"] != "" { + command = append(command, "--concurrency="+inputs["concurrency"]) + } + + return strings.Join(command, " ") +} + +func (m truffleCmdModel) Summary() string { + summary := strings.Builder{} + keys := []string{"no-verification", "only-verified", "json", "exclude_detectors", "concurrency"} + + inputs := m.GetInputs() + labels := m.GetLabels() + for _, key := range keys { + if inputs[key] != "" { + summary.WriteString("\t" + labels[key] + ": " + inputs[key] + "\n") + } + } + + if summary.Len() == 0 { + summary.WriteString("\tRunning with defaults\n") + + } + + summary.WriteString("\n") + return summary.String() +} + +func isTrue(val string) bool { + value := strings.ToLower(val) + isTrue, _ := strconv.ParseBool(value) + + if isTrue || value == "yes" || value == "y" { + return true + } + return false +} diff --git a/pkg/tui/pages/source_select/item.go b/pkg/tui/pages/source_select/item.go new file mode 100644 index 00000000..dd57a124 --- /dev/null +++ b/pkg/tui/pages/source_select/item.go @@ -0,0 +1,32 @@ +package source_select + +type SourceItem struct { + title string + description string + enterprise bool +} + +func OssItem(title, description string) SourceItem { + return SourceItem{title, description, false} +} + +func EnterpriseItem(title, description string) SourceItem { + return SourceItem{title, description, true} +} + +func (i SourceItem) ID() string { return i.title } + +func (i SourceItem) Title() string { + if i.enterprise { + return "šŸ’ø " + i.title + } + return i.title +} +func (i SourceItem) Description() string { + if i.enterprise { + return i.description + " (Enterprise only)" + } + return i.description +} + +func (i SourceItem) FilterValue() string { return i.title + i.description } diff --git a/pkg/tui/pages/source_select/source_select.go b/pkg/tui/pages/source_select/source_select.go new file mode 100644 index 00000000..94bd8b28 --- /dev/null +++ b/pkg/tui/pages/source_select/source_select.go @@ -0,0 +1,222 @@ +package source_select + +import ( + "time" + + "github.com/charmbracelet/bubbles/key" + "github.com/charmbracelet/bubbles/list" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/selector" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +// TODO: Review light theme styling +var ( + titleStyle = lipgloss.NewStyle(). + Foreground(lipgloss.Color("#FFFDF5")). + Background(lipgloss.Color(styles.Colors["bronze"])). + Padding(0, 1) + + // FIXME: Hon pls help + errorStatusMessageStyle = lipgloss.NewStyle(). + Foreground(lipgloss.AdaptiveColor{Dark: "#ff0000"}). + Render + + selectedSourceItemStyle = lipgloss.NewStyle(). + Border(lipgloss.NormalBorder(), false, false, false, true). + BorderForeground(lipgloss.AdaptiveColor{Dark: styles.Colors["sprout"], Light: styles.Colors["bronze"]}). + Foreground(lipgloss.AdaptiveColor{Dark: styles.Colors["sprout"], Light: styles.Colors["fern"]}). + Padding(0, 0, 0, 1) + + selectedDescription = selectedSourceItemStyle.Copy(). + Foreground(lipgloss.AdaptiveColor{Dark: styles.Colors["sprout"], Light: styles.Colors["sprout"]}) +) + +type listKeyMap struct { + toggleHelpMenu key.Binding +} + +type ( + SourceSelect struct { + common.Common + sourcesList list.Model + keys *listKeyMap + delegateKeys *delegateKeyMap + selector *selector.Selector + } +) + +func New(c common.Common) *SourceSelect { + var ( + delegateKeys = newDelegateKeyMap() + listKeys = &listKeyMap{ + toggleHelpMenu: key.NewBinding( + key.WithKeys("H"), + key.WithHelp("H", "toggle help"), + ), + } + ) + + // Make list of SourceItems. + SourceItems := []list.Item{ + // Open source sources. + OssItem("Git", "Scan git repositories."), + OssItem("GitHub", "Scan GitHub repositories and/or organizations."), + OssItem("GitLab", "Scan GitLab repositories."), + OssItem("Filesystem", "Scan your filesystem by selecting what directories to scan."), + OssItem("AWS S3", "Scan Amazon S3 buckets."), + OssItem("CircleCI", "Scan CircleCI, a CI/CD platform."), + OssItem("Syslog", "Scan syslog, event data logs."), + OssItem("Docker", "Scan a Docker instance, a containerized application."), + OssItem("GCS (Google Cloud Storage)", "Scan a Google Cloud Storage instance."), + // Enterprise sources. + EnterpriseItem("Artifactory", "Scan JFrog Artifactory packages."), + EnterpriseItem("BitBucket", "Scan Atlassian's Git-based source code repository hosting service."), + EnterpriseItem("Buildkite", "Scan Buildkite, a CI/CD platform."), + EnterpriseItem("Confluence", "Scan Atlassian's web-based wiki and knowledge base."), + EnterpriseItem("Gerrit", "Scan Gerrit, a code collaboration tool"), + EnterpriseItem("Jenkins ", "Scan Jenkins, a CI/CD platform."), + EnterpriseItem("Jira", "Scan Atlassian's issue & project tracking software."), + EnterpriseItem("Slack", "Scan Slack, a messaging and communication platform."), + EnterpriseItem("Microsoft Teams", "Scan Microsoft Teams, a messaging and communication platform."), + EnterpriseItem("Microsoft Sharepoint", "Scan Microsoft Sharepoint, a collaboration and document management platform."), + EnterpriseItem("Google Drive", "Scan Google Drive, a cloud-based storage and file sync service."), + } + + // Setup list + delegate := newSourceItemDelegate(delegateKeys) + delegate.Styles.SelectedTitle = selectedSourceItemStyle + delegate.Styles.SelectedDesc = selectedDescription + + sourcesList := list.New(SourceItems, delegate, 0, 0) + sourcesList.Title = "Sources" + sourcesList.Styles.Title = titleStyle + sourcesList.StatusMessageLifetime = 10 * time.Second + + sourcesList.AdditionalFullHelpKeys = func() []key.Binding { + return []key.Binding{ + listKeys.toggleHelpMenu, + } + } + + sourcesList.SetShowStatusBar(false) + sel := selector.New(c, []selector.IdentifiableItem{}, delegate) + + return &SourceSelect{ + Common: c, + sourcesList: sourcesList, + keys: listKeys, + delegateKeys: delegateKeys, + selector: sel, + } +} + +func (m *SourceSelect) Init() tea.Cmd { + return nil +} + +func (m *SourceSelect) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var cmds []tea.Cmd + + switch msg := msg.(type) { + case tea.WindowSizeMsg: + h, v := styles.AppStyle.GetFrameSize() + m.sourcesList.SetSize(msg.Width-h, msg.Height-v) + + case tea.KeyMsg: + // Don't match any of the keys below if we're actively filtering. + if m.sourcesList.FilterState() == list.Filtering { + break + } + + switch { + case key.Matches(msg, m.keys.toggleHelpMenu): + m.sourcesList.SetShowHelp(!m.sourcesList.ShowHelp()) + return m, nil + } + } + + // This will also call our delegate's update function. + newListModel, cmd := m.sourcesList.Update(msg) + m.sourcesList = newListModel + cmds = append(cmds, cmd) + + if m.selector != nil { + sel, cmd := m.selector.Update(msg) + m.selector = sel.(*selector.Selector) + cmds = append(cmds, cmd) + } + + return m, tea.Batch(cmds...) +} + +func (m *SourceSelect) View() string { + return styles.AppStyle.Render(m.sourcesList.View()) +} + +func (m *SourceSelect) ShortHelp() []key.Binding { + // TODO: actually return something + return nil +} + +func (m *SourceSelect) FullHelp() [][]key.Binding { + // TODO: actually return something + return nil +} + +func newSourceItemDelegate(keys *delegateKeyMap) list.DefaultDelegate { + d := list.NewDefaultDelegate() + + d.UpdateFunc = func(msg tea.Msg, m *list.Model) tea.Cmd { + selectedSourceItem, ok := m.SelectedItem().(SourceItem) + if !ok { + return nil + } + + if msg, ok := msg.(tea.KeyMsg); ok && key.Matches(msg, keys.choose) { + if selectedSourceItem.enterprise { + return m.NewStatusMessage(errorStatusMessageStyle( + "That's an enterprise only source. Learn more at trufflesecurity.com", + )) + } + + return func() tea.Msg { + return selector.SelectMsg{IdentifiableItem: selectedSourceItem} + } + } + return nil + } + + help := []key.Binding{keys.choose} + d.ShortHelpFunc = func() []key.Binding { return help } + d.FullHelpFunc = func() [][]key.Binding { return [][]key.Binding{help} } + + return d +} + +type delegateKeyMap struct { + choose key.Binding +} + +// Additional short help entries. This satisfies the help.KeyMap interface and +// is entirely optional. +func (d delegateKeyMap) ShortHelp() []key.Binding { + return []key.Binding{d.choose} +} + +// Additional full help entries. This satisfies the help.KeyMap interface and +// is entirely optional. +func (d delegateKeyMap) FullHelp() [][]key.Binding { + return [][]key.Binding{{d.choose}} +} + +func newDelegateKeyMap() *delegateKeyMap { + return &delegateKeyMap{ + choose: key.NewBinding( + key.WithKeys("enter"), + key.WithHelp("enter", "choose"), + ), + } +} diff --git a/pkg/tui/pages/view_oss/view_oss.go b/pkg/tui/pages/view_oss/view_oss.go new file mode 100644 index 00000000..13829003 --- /dev/null +++ b/pkg/tui/pages/view_oss/view_oss.go @@ -0,0 +1,59 @@ +package view_oss + +import ( + "strings" + + "github.com/charmbracelet/bubbles/key" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +type ViewOSS struct { + common.Common + viewed bool +} + +var ( + linkStyle = lipgloss.NewStyle().Foreground( + lipgloss.Color("28")) // green +) + +func New(c common.Common) *ViewOSS { + return &ViewOSS{ + Common: c, + viewed: false, + } +} + +func (m *ViewOSS) Init() tea.Cmd { + return nil +} + +func (m *ViewOSS) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + if m.viewed { + return m, tea.Quit + } + + return m, func() tea.Msg { return nil } +} + +func (m *ViewOSS) View() string { + s := strings.Builder{} + s.WriteString("View our open-source project on GitHub\n") + s.WriteString(linkStyle.Render("šŸ”— https://github.com/trufflesecurity/trufflehog ")) + + m.viewed = true + return styles.AppStyle.Render(s.String()) +} + +func (m *ViewOSS) ShortHelp() []key.Binding { + // TODO: actually return something + return nil +} + +func (m *ViewOSS) FullHelp() [][]key.Binding { + // TODO: actually return something + return nil +} diff --git a/pkg/tui/pages/wizard_intro/item.go b/pkg/tui/pages/wizard_intro/item.go new file mode 100644 index 00000000..f9eef55d --- /dev/null +++ b/pkg/tui/pages/wizard_intro/item.go @@ -0,0 +1,137 @@ +package wizard_intro + +import ( + "fmt" + "io" + "strings" + + "github.com/charmbracelet/bubbles/key" + "github.com/charmbracelet/bubbles/list" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" +) + +// Item represents a single item in the selector. +type Item int + +// ID implements selector.IdentifiableItem. +func (i Item) ID() string { + return i.String() +} + +// Title returns the item title. Implements list.DefaultItem. +func (i Item) Title() string { return i.String() } + +// Description returns the item description. Implements list.DefaultItem. +func (i Item) Description() string { return "" } + +// FilterValue implements list.Item. +func (i Item) FilterValue() string { return i.Title() } + +// Command returns the item Command view. +func (i Item) Command() string { + return i.Title() +} + +// ItemDelegate is the delegate for the item. +type ItemDelegate struct { + common *common.Common +} + +// Width returns the item width. +func (d ItemDelegate) Width() int { + width := d.common.Styles.MenuItem.GetHorizontalFrameSize() + d.common.Styles.MenuItem.GetWidth() + return width +} + +// Height returns the item height. Implements list.ItemDelegate. +func (d ItemDelegate) Height() int { + height := d.common.Styles.MenuItem.GetVerticalFrameSize() + d.common.Styles.MenuItem.GetHeight() + return height +} + +// Spacing returns the spacing between items. Implements list.ItemDelegate. +func (d ItemDelegate) Spacing() int { return 1 } + +// Update implements list.ItemDelegate. +func (d ItemDelegate) Update(msg tea.Msg, m *list.Model) tea.Cmd { + idx := m.Index() + item, ok := m.SelectedItem().(Item) + if !ok { + return nil + } + switch msg := msg.(type) { + case tea.KeyMsg: + switch { + case key.Matches(msg, d.common.KeyMap.Copy): + d.common.Copy.Copy(item.Command()) + return m.SetItem(idx, item) + } + } + return nil +} + +// Render implements list.ItemDelegate. +func (d ItemDelegate) Render(w io.Writer, m list.Model, index int, listItem list.Item) { + i := listItem.(Item) + s := strings.Builder{} + var matchedRunes []int + + // Conditions + var ( + isSelected = index == m.Index() + isFiltered = m.FilterState() == list.Filtering || m.FilterState() == list.FilterApplied + ) + + styles := d.common.Styles.RepoSelector.Normal + if isSelected { + styles = d.common.Styles.RepoSelector.Active + } + + title := i.Title() + title = common.TruncateString(title, m.Width()-styles.Base.GetHorizontalFrameSize()) + // if i.repo.IsPrivate() { + // title += " šŸ”’" + // } + if isSelected { + title += " " + } + updatedStr := " Updated" + if m.Width()-styles.Base.GetHorizontalFrameSize()-lipgloss.Width(updatedStr)-lipgloss.Width(title) <= 0 { + updatedStr = "" + } + updatedStyle := styles.Updated.Copy(). + Align(lipgloss.Right). + Width(m.Width() - styles.Base.GetHorizontalFrameSize() - lipgloss.Width(title)) + updated := updatedStyle.Render(updatedStr) + + if isFiltered && index < len(m.VisibleItems()) { + // Get indices of matched characters + matchedRunes = m.MatchesForItem(index) + } + + if isFiltered { + unmatched := styles.Title.Copy().Inline(true) + matched := unmatched.Copy().Underline(true) + title = lipgloss.StyleRunes(title, matchedRunes, matched, unmatched) + } + title = styles.Title.Render(title) + desc := i.Description() + desc = common.TruncateString(desc, m.Width()-styles.Base.GetHorizontalFrameSize()) + desc = styles.Desc.Render(desc) + + s.WriteString(lipgloss.JoinHorizontal(lipgloss.Bottom, title, updated)) + s.WriteRune('\n') + s.WriteString(desc) + s.WriteRune('\n') + cmd := common.TruncateString(i.Command(), m.Width()-styles.Base.GetHorizontalFrameSize()) + cmd = styles.Command.Render(cmd) + + s.WriteString(cmd) + fmt.Fprint(w, + d.common.Zone.Mark(i.ID(), + styles.Base.Render(s.String()), + ), + ) +} diff --git a/pkg/tui/pages/wizard_intro/wizard_intro.go b/pkg/tui/pages/wizard_intro/wizard_intro.go new file mode 100644 index 00000000..dcc412f3 --- /dev/null +++ b/pkg/tui/pages/wizard_intro/wizard_intro.go @@ -0,0 +1,109 @@ +package wizard_intro + +import ( + "strings" + + "github.com/charmbracelet/bubbles/key" + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/selector" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +const ( + ScanSourceWithWizard Item = iota + // ScanSourceWithConfig + ViewHelpDocs + ViewOSSProject + EnterpriseInquire + Quit +) + +func (w Item) String() string { + switch w { + case ScanSourceWithWizard: + return "Scan a source using wizard" + //case ScanSourceWithConfig: + // return "Scan a source with a config file" + case ViewHelpDocs: + return "View help docs" + case ViewOSSProject: + return "View open-source project" + case EnterpriseInquire: + return "Inquire about TruffleHog Enterprise" + case Quit: + return "Quit" + } + panic("unreachable") +} + +type WizardIntro struct { + common.Common + selector *selector.Selector +} + +func New(cmn common.Common) *WizardIntro { + sel := selector.New(cmn, + []selector.IdentifiableItem{ + ScanSourceWithWizard, + // ScanSourceWithConfig, + ViewHelpDocs, + ViewOSSProject, + EnterpriseInquire, + Quit, + }, + ItemDelegate{&cmn}) + + return &WizardIntro{Common: cmn, selector: sel} +} + +func (m *WizardIntro) Init() tea.Cmd { + m.selector.Select(0) + return nil +} + +func (m *WizardIntro) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + cmds := make([]tea.Cmd, 0) + + s, cmd := m.selector.Update(msg) + m.selector = s.(*selector.Selector) + if cmd != nil { + cmds = append(cmds, cmd) + } + + return m, tea.Batch(cmds...) +} + +func (m *WizardIntro) View() string { + s := strings.Builder{} + s.WriteString("What do you want to do?\n\n") + + for i, selectorItem := range m.selector.Items() { + // Cast the interface to the concrete Item struct. + item := selectorItem.(Item) + if m.selector.Index() == i { + selectedStyle := lipgloss.NewStyle().Foreground(lipgloss.Color(styles.Colors["sprout"])) + s.WriteString(selectedStyle.Render(" (ā€¢) " + item.Title())) + } else { + s.WriteString(" ( ) " + item.Title()) + } + s.WriteString("\n") + } + + return styles.AppStyle.Render(s.String()) +} + +func (m *WizardIntro) ShortHelp() []key.Binding { + kb := make([]key.Binding, 0) + kb = append(kb, + m.Common.KeyMap.UpDown, + m.Common.KeyMap.Section, + ) + return kb +} + +func (m *WizardIntro) FullHelp() [][]key.Binding { + // TODO: actually return something + return nil +} diff --git a/pkg/tui/sources/circleci/circleci.go b/pkg/tui/sources/circleci/circleci.go new file mode 100644 index 00000000..cd57a21d --- /dev/null +++ b/pkg/tui/sources/circleci/circleci.go @@ -0,0 +1,44 @@ +package circleci + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type circleCiCmdModel struct { + textinputs.Model +} + +func GetFields() circleCiCmdModel { + token := textinputs.InputConfig{ + Label: "API Token", + Key: "token", + Required: true, + Placeholder: "top secret token", + } + + return circleCiCmdModel{textinputs.New([]textinputs.InputConfig{token})} +} + +func (m circleCiCmdModel) Cmd() string { + var command []string + command = append(command, "trufflehog", "circleci") + + inputs := m.GetInputs() + + if inputs["token"] != "" { + command = append(command, "--token="+inputs["token"]) + } + + return strings.Join(command, " ") +} + +func (m circleCiCmdModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + keys := []string{"token"} + + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/sources/docker/docker.go b/pkg/tui/sources/docker/docker.go new file mode 100644 index 00000000..942459aa --- /dev/null +++ b/pkg/tui/sources/docker/docker.go @@ -0,0 +1,49 @@ +package docker + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type dockerCmdModel struct { + textinputs.Model +} + +func GetFields() dockerCmdModel { + images := textinputs.InputConfig{ + Label: "Docker image(s)", + Key: "images", + Required: true, + Help: "Separate by space if multiple.", + Placeholder: "trufflesecurity/secrets", + } + + return dockerCmdModel{textinputs.New([]textinputs.InputConfig{images})} +} + +func (m dockerCmdModel) Cmd() string { + + var command []string + command = append(command, "trufflehog", "docker") + + inputs := m.GetInputs() + vals := inputs["images"] + if vals != "" { + images := strings.Fields(vals) + for _, image := range images { + command = append(command, "--image="+image) + } + } + + return strings.Join(command, " ") +} + +func (m dockerCmdModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + keys := []string{"images"} + + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/sources/filesystem/filesystem.go b/pkg/tui/sources/filesystem/filesystem.go new file mode 100644 index 00000000..68b98c8c --- /dev/null +++ b/pkg/tui/sources/filesystem/filesystem.go @@ -0,0 +1,45 @@ +package filesystem + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type fsModel struct { + textinputs.Model +} + +func GetFields() fsModel { + path := textinputs.InputConfig{ + Label: "Path", + Key: "path", + Required: true, + Help: "Files and directories to scan. Separate by space if multiple.", + Placeholder: "path/to/file.txt path/to/another/dir", + } + + return fsModel{textinputs.New([]textinputs.InputConfig{path})} +} + +func (m fsModel) Cmd() string { + var command []string + command = append(command, "trufflehog", "filesystem") + + inputs := m.GetInputs() + + if inputs["path"] != "" { + command = append(command, inputs["path"]) + } + + return strings.Join(command, " ") +} + +func (m fsModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + + keys := []string{"path"} + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/sources/gcs/gcs.go b/pkg/tui/sources/gcs/gcs.go new file mode 100644 index 00000000..0342a120 --- /dev/null +++ b/pkg/tui/sources/gcs/gcs.go @@ -0,0 +1,44 @@ +package gcs + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type gcsCmdModel struct { + textinputs.Model +} + +func GetFields() gcsCmdModel { + projectId := textinputs.InputConfig{ + Label: "Project ID", + Key: "project_id", + Required: true, + Placeholder: "my-project", + } + + return gcsCmdModel{textinputs.New([]textinputs.InputConfig{projectId})} +} + +func (m gcsCmdModel) Cmd() string { + var command []string + command = append(command, "trufflehog", "gcs") + + inputs := m.GetInputs() + if inputs["project_id"] != "" { + command = append(command, "--project_id="+inputs["project_id"]) + } + + command = append(command, "--cloud-environment") + return strings.Join(command, " ") +} + +func (m gcsCmdModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + + keys := []string{"project_id"} + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/sources/git/git.go b/pkg/tui/sources/git/git.go new file mode 100644 index 00000000..882c7c1c --- /dev/null +++ b/pkg/tui/sources/git/git.go @@ -0,0 +1,44 @@ +package git + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type gitCmdModel struct { + textinputs.Model +} + +func GetFields() gitCmdModel { + uri := textinputs.InputConfig{ + Label: "Git URI", + Key: "uri", + Required: true, + Placeholder: "git@github.com:trufflesecurity/trufflehog.git.", + } + + return gitCmdModel{textinputs.New([]textinputs.InputConfig{uri})} +} + +func (m gitCmdModel) Cmd() string { + var command []string + command = append(command, "trufflehog", "git") + + inputs := m.GetInputs() + + if inputs["uri"] != "" { + command = append(command, inputs["uri"]) + } + + return strings.Join(command, " ") +} + +func (m gitCmdModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + + keys := []string{"uri"} + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/sources/github/github.go b/pkg/tui/sources/github/github.go new file mode 100644 index 00000000..2ee09c6a --- /dev/null +++ b/pkg/tui/sources/github/github.go @@ -0,0 +1,61 @@ +package github + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type githubCmdModel struct { + textinputs.Model +} + +func GetNote() string { + return "Please enter an organization OR repository." +} + +func GetFields() githubCmdModel { + org := textinputs.InputConfig{ + Label: "Organization", + Key: "org", + Required: false, + Help: "GitHub organization to scan.", + Placeholder: "trufflesecurity", + } + + repo := textinputs.InputConfig{ + Label: "Repository", + Key: "repo", + Required: false, + Help: "GitHub repo to scan.", + Placeholder: "https://github.com/trufflesecurity/test_keys", + } + + return githubCmdModel{textinputs.New([]textinputs.InputConfig{org, repo})} +} + +func (m githubCmdModel) Cmd() string { + var command []string + command = append(command, "trufflehog", "github") + + inputs := m.GetInputs() + + if inputs["org"] != "" { + command = append(command, "--org="+inputs["org"]) + } + + if inputs["repo"] != "" { + command = append(command, "--repo="+inputs["repo"]) + } + + return strings.Join(command, " ") +} + +func (m githubCmdModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + + keys := []string{"org", "repo"} + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/sources/gitlab/gitlab.go b/pkg/tui/sources/gitlab/gitlab.go new file mode 100644 index 00000000..f272fc8d --- /dev/null +++ b/pkg/tui/sources/gitlab/gitlab.go @@ -0,0 +1,45 @@ +package gitlab + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type gitlabCmdModel struct { + textinputs.Model +} + +func GetFields() gitlabCmdModel { + token := textinputs.InputConfig{ + Label: "GitLab token", + Key: "token", + Required: true, + Help: "Personal access token with read access", + Placeholder: "glpat-", + } + + return gitlabCmdModel{textinputs.New([]textinputs.InputConfig{token})} +} + +func (m gitlabCmdModel) Cmd() string { + var command []string + command = append(command, "trufflehog", "gitlab") + + inputs := m.GetInputs() + + if inputs["token"] != "" { + command = append(command, "--token="+inputs["token"]) + } + + return strings.Join(command, " ") +} + +func (m gitlabCmdModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + + keys := []string{"token"} + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/sources/s3/s3.go b/pkg/tui/sources/s3/s3.go new file mode 100644 index 00000000..42433e33 --- /dev/null +++ b/pkg/tui/sources/s3/s3.go @@ -0,0 +1,48 @@ +package s3 + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type s3CmdModel struct { + textinputs.Model +} + +func GetFields() s3CmdModel { + bucket := textinputs.InputConfig{ + Label: "S3 bucket name(s)", + Key: "buckets", + Required: true, + Placeholder: "my-bucket-name", + Help: "Buckets to scan. Separate by space if multiple.", + } + + return s3CmdModel{textinputs.New([]textinputs.InputConfig{bucket})} +} + +func (m s3CmdModel) Cmd() string { + var command []string + command = append(command, "trufflehog", "s3") + + inputs := m.GetInputs() + vals := inputs["buckets"] + if vals != "" { + buckets := strings.Fields(vals) + for _, bucket := range buckets { + command = append(command, "--bucket="+bucket) + } + } + + return strings.Join(command, " ") +} + +func (m s3CmdModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + + keys := []string{"buckets"} + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/sources/sources.go b/pkg/tui/sources/sources.go new file mode 100644 index 00000000..8269fcaf --- /dev/null +++ b/pkg/tui/sources/sources.go @@ -0,0 +1,60 @@ +package sources + +import ( + "strings" + + tea "github.com/charmbracelet/bubbletea" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/circleci" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/docker" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/filesystem" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/gcs" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/git" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/gitlab" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/s3" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/sources/syslog" +) + +func GetSourceNotes(sourceName string) string { + source := strings.ToLower(sourceName) + switch source { + case "github": + return github.GetNote() + + default: + return "" + } +} + +type CmdModel interface { + tea.Model + Cmd() string + Summary() string +} + +func GetSourceFields(sourceName string) CmdModel { + source := strings.ToLower(sourceName) + + switch source { + case "git": + return git.GetFields() + case "github": + return github.GetFields() + case "gitlab": + return gitlab.GetFields() + case "filesystem": + return filesystem.GetFields() + case "aws s3": + return s3.GetFields() + case "gcs (google cloud storage)": + return gcs.GetFields() + case "syslog": + return syslog.GetFields() + case "circleci": + return circleci.GetFields() + case "docker": + return docker.GetFields() + } + + return nil +} diff --git a/pkg/tui/sources/syslog/syslog.go b/pkg/tui/sources/syslog/syslog.go new file mode 100644 index 00000000..c220d398 --- /dev/null +++ b/pkg/tui/sources/syslog/syslog.go @@ -0,0 +1,82 @@ +package syslog + +import ( + "strings" + + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/textinputs" +) + +type syslogCmdModel struct { + textinputs.Model +} + +// TODO: review fields +func GetFields() syslogCmdModel { + protocol := textinputs.InputConfig{ + Label: "Protocol", + Key: "protocol", + Required: true, + Help: "udp or tcp", + Placeholder: "tcp", + } + + listenAddress := textinputs.InputConfig{ + Label: "Address", + Key: "address", + Help: "Address and port to listen on for syslog", + Required: true, + Placeholder: "127.0.0.1:514", + } + + tlsCert := textinputs.InputConfig{ + Label: "TLS Certificate", + Key: "cert", + Required: true, + Help: "Path to TLS certificate", + Placeholder: "/path/to/cert", + } + + tlsKey := textinputs.InputConfig{ + Label: "TLS Key", + Key: "key", + Required: true, + Help: "Path to TLS key", + Placeholder: "/path/to/key", + } + + format := textinputs.InputConfig{ + Label: "Log format", + Key: "format", + Required: true, + Help: "Can be rfc3164 or rfc5424", + Placeholder: "rfc3164", + } + + return syslogCmdModel{textinputs.New([]textinputs.InputConfig{listenAddress, protocol, tlsCert, tlsKey, format})} +} + +func (m syslogCmdModel) Cmd() string { + var command []string + command = append(command, "trufflehog", "syslog") + + inputs := m.GetInputs() + syslogKeys := [5]string{"address", "protocol", "cert", "key", "format"} + + for _, key := range syslogKeys { + if inputs[key] != "" { + flag := "--" + key + "=" + inputs[key] + command = append(command, flag) + } + } + + return strings.Join(command, " ") +} + +func (m syslogCmdModel) Summary() string { + inputs := m.GetInputs() + labels := m.GetLabels() + keys := []string{"address", "protocol", "cert", "key", "format"} + + return common.SummarizeSource(keys, inputs, labels) +} diff --git a/pkg/tui/styles/styles.go b/pkg/tui/styles/styles.go new file mode 100644 index 00000000..09c92521 --- /dev/null +++ b/pkg/tui/styles/styles.go @@ -0,0 +1,493 @@ +package styles + +import ( + "github.com/charmbracelet/lipgloss" +) + +// XXX: For now, this is in its own package so that it can be shared between +// different packages without incurring an illegal import cycle. + +// https://github.com/charmbracelet/lipgloss#colors +var Colors = map[string]string{ + "softblack": "#1e1e1e", + "charcoal": "#252525", + "stone": "#5a5a5a", + "smoke": "#999999", + "sand": "#e1deda", + "cloud": "#f4efe9", + "offwhite": "#faf8f7", + "fern": "#38645a", + "sprout": "#5bb381", + "gold": "#ae8c57", + "bronze": "#89553d", + "coral": "#c15750", + "violet": "#6b5b9a", +} + +var ( + BoldTextStyle = lipgloss.NewStyle().Bold(true) + + PrimaryTextStyle = lipgloss.NewStyle().Foreground( + lipgloss.Color("28")) // green + + HintTextStyle = lipgloss.NewStyle().Foreground( + lipgloss.Color("8")) // grey + + CodeTextStyle = lipgloss.NewStyle().Background(lipgloss.Color("130")).Foreground(lipgloss.Color("15")) +) + +var AppStyle = lipgloss.NewStyle().Padding(1, 2) + +// Styles defines styles for the UI. +type Styles struct { + ActiveBorderColor lipgloss.Color + InactiveBorderColor lipgloss.Color + + App lipgloss.Style + ServerName lipgloss.Style + TopLevelNormalTab lipgloss.Style + TopLevelActiveTab lipgloss.Style + TopLevelActiveTabDot lipgloss.Style + + MenuItem lipgloss.Style + MenuLastUpdate lipgloss.Style + + RepoSelector struct { + Normal struct { + Base lipgloss.Style + Title lipgloss.Style + Desc lipgloss.Style + Command lipgloss.Style + Updated lipgloss.Style + } + Active struct { + Base lipgloss.Style + Title lipgloss.Style + Desc lipgloss.Style + Command lipgloss.Style + Updated lipgloss.Style + } + } + + Repo struct { + Base lipgloss.Style + Title lipgloss.Style + Command lipgloss.Style + Body lipgloss.Style + Header lipgloss.Style + HeaderName lipgloss.Style + HeaderDesc lipgloss.Style + } + + Footer lipgloss.Style + Branch lipgloss.Style + HelpKey lipgloss.Style + HelpValue lipgloss.Style + HelpDivider lipgloss.Style + URLStyle lipgloss.Style + + Error lipgloss.Style + ErrorTitle lipgloss.Style + ErrorBody lipgloss.Style + + AboutNoReadme lipgloss.Style + + LogItem struct { + Normal struct { + Base lipgloss.Style + Hash lipgloss.Style + Title lipgloss.Style + Desc lipgloss.Style + Keyword lipgloss.Style + } + Active struct { + Base lipgloss.Style + Hash lipgloss.Style + Title lipgloss.Style + Desc lipgloss.Style + Keyword lipgloss.Style + } + } + + Log struct { + Commit lipgloss.Style + CommitHash lipgloss.Style + CommitAuthor lipgloss.Style + CommitDate lipgloss.Style + CommitBody lipgloss.Style + CommitStatsAdd lipgloss.Style + CommitStatsDel lipgloss.Style + Paginator lipgloss.Style + } + + Ref struct { + Normal struct { + Item lipgloss.Style + ItemTag lipgloss.Style + } + Active struct { + Item lipgloss.Style + ItemTag lipgloss.Style + } + ItemSelector lipgloss.Style + ItemBranch lipgloss.Style + Paginator lipgloss.Style + } + + Tree struct { + Normal struct { + FileName lipgloss.Style + FileDir lipgloss.Style + FileMode lipgloss.Style + FileSize lipgloss.Style + } + Active struct { + FileName lipgloss.Style + FileDir lipgloss.Style + FileMode lipgloss.Style + FileSize lipgloss.Style + } + Selector lipgloss.Style + FileContent lipgloss.Style + Paginator lipgloss.Style + NoItems lipgloss.Style + } + + Spinner lipgloss.Style + + CodeNoContent lipgloss.Style + + StatusBar lipgloss.Style + StatusBarKey lipgloss.Style + StatusBarValue lipgloss.Style + StatusBarInfo lipgloss.Style + StatusBarBranch lipgloss.Style + StatusBarHelp lipgloss.Style + + Tabs lipgloss.Style + TabInactive lipgloss.Style + TabActive lipgloss.Style + TabSeparator lipgloss.Style +} + +// DefaultStyles returns default styles for the UI. +func DefaultStyles() *Styles { + highlightColor := lipgloss.Color("210") + highlightColorDim := lipgloss.Color("174") + selectorColor := lipgloss.Color("167") + hashColor := lipgloss.Color("185") + + s := new(Styles) + + s.ActiveBorderColor = lipgloss.Color("62") + s.InactiveBorderColor = lipgloss.Color("241") + + s.App = lipgloss.NewStyle(). + Margin(1, 2) + + s.ServerName = lipgloss.NewStyle(). + Height(1). + MarginLeft(1). + MarginBottom(1). + Padding(0, 1). + Background(lipgloss.Color("57")). + Foreground(lipgloss.Color("229")). + Bold(true) + + s.TopLevelNormalTab = lipgloss.NewStyle(). + MarginRight(2) + + s.TopLevelActiveTab = s.TopLevelNormalTab.Copy(). + Foreground(lipgloss.Color("36")) + + s.TopLevelActiveTabDot = lipgloss.NewStyle(). + Foreground(lipgloss.Color("36")) + + s.RepoSelector.Normal.Base = lipgloss.NewStyle(). + PaddingLeft(1). + Border(lipgloss.Border{Left: " "}, false, false, false, true). + Height(3) + + s.RepoSelector.Normal.Title = lipgloss.NewStyle().Bold(true) + + s.RepoSelector.Normal.Desc = lipgloss.NewStyle(). + Foreground(lipgloss.Color("243")) + + s.RepoSelector.Normal.Command = lipgloss.NewStyle(). + Foreground(lipgloss.Color("132")) + + s.RepoSelector.Normal.Updated = lipgloss.NewStyle(). + Foreground(lipgloss.Color("243")) + + s.RepoSelector.Active.Base = s.RepoSelector.Normal.Base.Copy(). + BorderStyle(lipgloss.Border{Left: "ā”ƒ"}). + BorderForeground(lipgloss.Color("176")) + + s.RepoSelector.Active.Title = s.RepoSelector.Normal.Title.Copy(). + Foreground(lipgloss.Color("212")) + + s.RepoSelector.Active.Desc = s.RepoSelector.Normal.Desc.Copy(). + Foreground(lipgloss.Color("246")) + + s.RepoSelector.Active.Updated = s.RepoSelector.Normal.Updated.Copy(). + Foreground(lipgloss.Color("212")) + + s.RepoSelector.Active.Command = s.RepoSelector.Normal.Command.Copy(). + Foreground(lipgloss.Color("204")) + + s.MenuItem = lipgloss.NewStyle(). + PaddingLeft(1). + Border(lipgloss.Border{ + Left: " ", + }, false, false, false, true). + Height(3) + + s.MenuLastUpdate = lipgloss.NewStyle(). + Foreground(lipgloss.Color("241")). + Align(lipgloss.Right) + + s.Repo.Base = lipgloss.NewStyle() + + s.Repo.Title = lipgloss.NewStyle(). + Padding(0, 2) + + s.Repo.Command = lipgloss.NewStyle(). + Foreground(lipgloss.Color("168")) + + s.Repo.Body = lipgloss.NewStyle(). + Margin(1, 0) + + s.Repo.Header = lipgloss.NewStyle(). + Height(2). + Border(lipgloss.NormalBorder(), false, false, true, false). + BorderForeground(lipgloss.Color("236")) + + s.Repo.HeaderName = lipgloss.NewStyle(). + Foreground(lipgloss.Color("212")). + Bold(true) + + s.Repo.HeaderDesc = lipgloss.NewStyle(). + Foreground(lipgloss.Color("243")) + + s.Footer = lipgloss.NewStyle(). + MarginTop(1). + Padding(0, 1). + Height(1) + + s.Branch = lipgloss.NewStyle(). + Foreground(lipgloss.Color("203")). + Background(lipgloss.Color("236")). + Padding(0, 1) + + s.HelpKey = lipgloss.NewStyle(). + Foreground(lipgloss.Color("241")) + + s.HelpValue = lipgloss.NewStyle(). + Foreground(lipgloss.Color("239")) + + s.HelpDivider = lipgloss.NewStyle(). + Foreground(lipgloss.Color("237")). + SetString(" ā€¢ ") + + s.URLStyle = lipgloss.NewStyle(). + MarginLeft(1). + Foreground(lipgloss.Color("168")) + + s.Error = lipgloss.NewStyle(). + MarginTop(2) + + s.ErrorTitle = lipgloss.NewStyle(). + Foreground(lipgloss.Color("230")). + Background(lipgloss.Color("204")). + Bold(true). + Padding(0, 1) + + s.ErrorBody = lipgloss.NewStyle(). + Foreground(lipgloss.Color("252")). + MarginLeft(2) + + s.AboutNoReadme = lipgloss.NewStyle(). + MarginTop(1). + MarginLeft(2). + Foreground(lipgloss.Color("242")) + + s.LogItem.Normal.Base = lipgloss.NewStyle(). + Border(lipgloss.Border{ + Left: " ", + }, false, false, false, true). + PaddingLeft(1) + + s.LogItem.Active.Base = s.LogItem.Normal.Base.Copy(). + Border(lipgloss.Border{ + Left: "ā”ƒ", + }, false, false, false, true). + BorderForeground(selectorColor) + + s.LogItem.Active.Hash = s.LogItem.Normal.Hash.Copy(). + Foreground(hashColor) + + s.LogItem.Active.Hash = lipgloss.NewStyle(). + Bold(true). + Foreground(highlightColor) + + s.LogItem.Normal.Title = lipgloss.NewStyle(). + Foreground(lipgloss.Color("105")) + + s.LogItem.Active.Title = lipgloss.NewStyle(). + Foreground(highlightColor). + Bold(true) + + s.LogItem.Normal.Desc = lipgloss.NewStyle(). + Foreground(lipgloss.Color("246")) + + s.LogItem.Active.Desc = lipgloss.NewStyle(). + Foreground(lipgloss.Color("95")) + + s.LogItem.Active.Keyword = s.LogItem.Active.Desc.Copy(). + Foreground(highlightColorDim) + + s.LogItem.Normal.Hash = lipgloss.NewStyle(). + Foreground(hashColor) + + s.LogItem.Active.Hash = lipgloss.NewStyle(). + Foreground(highlightColor) + + s.Log.Commit = lipgloss.NewStyle(). + Margin(0, 2) + + s.Log.CommitHash = lipgloss.NewStyle(). + Foreground(hashColor). + Bold(true) + + s.Log.CommitBody = lipgloss.NewStyle(). + MarginTop(1). + MarginLeft(2) + + s.Log.CommitStatsAdd = lipgloss.NewStyle(). + Foreground(lipgloss.Color("42")). + Bold(true) + + s.Log.CommitStatsDel = lipgloss.NewStyle(). + Foreground(lipgloss.Color("203")). + Bold(true) + + s.Log.Paginator = lipgloss.NewStyle(). + Margin(0). + Align(lipgloss.Center) + + s.Ref.Normal.Item = lipgloss.NewStyle() + + s.Ref.ItemSelector = lipgloss.NewStyle(). + Foreground(selectorColor). + SetString("> ") + + s.Ref.Active.Item = lipgloss.NewStyle(). + Foreground(highlightColorDim) + + s.Ref.ItemBranch = lipgloss.NewStyle() + + s.Ref.Normal.ItemTag = lipgloss.NewStyle(). + Foreground(lipgloss.Color("39")) + + s.Ref.Active.ItemTag = lipgloss.NewStyle(). + Bold(true). + Foreground(highlightColor) + + s.Ref.Active.Item = lipgloss.NewStyle(). + Bold(true). + Foreground(highlightColor) + + s.Ref.Paginator = s.Log.Paginator.Copy() + + s.Tree.Selector = s.Tree.Normal.FileName.Copy(). + Width(1). + Foreground(selectorColor) + + s.Tree.Normal.FileName = lipgloss.NewStyle(). + MarginLeft(1) + + s.Tree.Active.FileName = s.Tree.Normal.FileName.Copy(). + Bold(true). + Foreground(highlightColor) + + s.Tree.Normal.FileDir = lipgloss.NewStyle(). + Foreground(lipgloss.Color("39")) + + s.Tree.Active.FileDir = lipgloss.NewStyle(). + Foreground(highlightColor) + + s.Tree.Normal.FileMode = s.Tree.Active.FileName.Copy(). + Width(10). + Foreground(lipgloss.Color("243")) + + s.Tree.Active.FileMode = s.Tree.Normal.FileMode.Copy(). + Foreground(highlightColorDim) + + s.Tree.Normal.FileSize = s.Tree.Normal.FileName.Copy(). + Foreground(lipgloss.Color("243")) + + s.Tree.Active.FileSize = s.Tree.Normal.FileName.Copy(). + Foreground(highlightColorDim) + + s.Tree.FileContent = lipgloss.NewStyle() + + s.Tree.Paginator = s.Log.Paginator.Copy() + + s.Tree.NoItems = s.AboutNoReadme.Copy() + + s.Spinner = lipgloss.NewStyle(). + MarginTop(1). + MarginLeft(2). + Foreground(lipgloss.Color("205")) + + s.CodeNoContent = lipgloss.NewStyle(). + SetString("No Content."). + MarginTop(1). + MarginLeft(2). + Foreground(lipgloss.Color("242")) + + s.StatusBar = lipgloss.NewStyle(). + Height(1) + + s.StatusBarKey = lipgloss.NewStyle(). + Bold(true). + Padding(0, 1). + Background(lipgloss.Color("206")). + Foreground(lipgloss.Color("228")) + + s.StatusBarValue = lipgloss.NewStyle(). + Padding(0, 1). + Background(lipgloss.Color("235")). + Foreground(lipgloss.Color("243")) + + s.StatusBarInfo = lipgloss.NewStyle(). + Padding(0, 1). + Background(lipgloss.Color("212")). + Foreground(lipgloss.Color("230")) + + s.StatusBarBranch = lipgloss.NewStyle(). + Padding(0, 1). + Background(lipgloss.Color("62")). + Foreground(lipgloss.Color("230")) + + s.StatusBarHelp = lipgloss.NewStyle(). + Padding(0, 1). + Background(lipgloss.Color("237")). + Foreground(lipgloss.Color("243")) + + s.Tabs = lipgloss.NewStyle(). + Height(1) + + s.TabInactive = lipgloss.NewStyle() + + s.TabActive = lipgloss.NewStyle(). + Underline(true). + Foreground(lipgloss.Color("36")) + + s.TabSeparator = lipgloss.NewStyle(). + SetString("ā”‚"). + Padding(0, 1). + Foreground(lipgloss.Color("238")) + + return s +} diff --git a/pkg/tui/tui.go b/pkg/tui/tui.go new file mode 100644 index 00000000..023e9b6a --- /dev/null +++ b/pkg/tui/tui.go @@ -0,0 +1,198 @@ +package tui + +import ( + "fmt" + "os" + "strings" + + "github.com/charmbracelet/bubbles/key" + tea "github.com/charmbracelet/bubbletea" + zone "github.com/lrstanley/bubblezone" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/components/selector" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/keymap" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/pages/contact_enterprise" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/pages/source_configure" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/pages/source_select" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/pages/view_oss" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/pages/wizard_intro" + "github.com/trufflesecurity/trufflehog/v3/pkg/tui/styles" +) + +type page int + +const ( + wizardIntroPage page = iota + sourceSelectPage + sourceConfigurePage + viewOSSProjectPage + contactEnterprisePage +) + +type sessionState int + +const ( + startState sessionState = iota + errorState + loadedState +) + +// TUI is the main TUI model. +type TUI struct { + common common.Common + pages []common.Component + activePage page + state sessionState + args []string +} + +// New returns a new TUI model. +func New(c common.Common) *TUI { + ui := &TUI{ + common: c, + pages: make([]common.Component, 5), + activePage: wizardIntroPage, + state: startState, + } + return ui +} + +// SetSize implements common.Component. +func (ui *TUI) SetSize(width, height int) { + ui.common.SetSize(width, height) + for _, p := range ui.pages { + if p != nil { + p.SetSize(width, height) + } + } +} + +// Init implements tea.Model. +func (ui *TUI) Init() tea.Cmd { + ui.pages[wizardIntroPage] = wizard_intro.New(ui.common) + ui.pages[sourceSelectPage] = source_select.New(ui.common) + ui.pages[sourceConfigurePage] = source_configure.New(ui.common) + ui.pages[viewOSSProjectPage] = view_oss.New(ui.common) + ui.pages[contactEnterprisePage] = contact_enterprise.New(ui.common) + ui.SetSize(ui.common.Width, ui.common.Height) + cmds := make([]tea.Cmd, 0) + cmds = append(cmds, + ui.pages[wizardIntroPage].Init(), + ui.pages[sourceSelectPage].Init(), + ui.pages[sourceConfigurePage].Init(), + ui.pages[viewOSSProjectPage].Init(), + ui.pages[contactEnterprisePage].Init(), + ) + ui.state = loadedState + ui.SetSize(ui.common.Width, ui.common.Height) + return tea.Batch(cmds...) +} + +// Update implements tea.Model. +func (ui *TUI) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + cmds := make([]tea.Cmd, 0) + switch msg := msg.(type) { + case tea.WindowSizeMsg: + ui.SetSize(msg.Width, msg.Height) + for i, p := range ui.pages { + m, cmd := p.Update(msg) + ui.pages[i] = m.(common.Component) + if cmd != nil { + cmds = append(cmds, cmd) + } + } + case tea.KeyMsg, tea.MouseMsg: + switch msg := msg.(type) { + case tea.KeyMsg: + switch { + case key.Matches(msg, ui.common.KeyMap.Help): + case key.Matches(msg, ui.common.KeyMap.CmdQuit) && ui.activePage != sourceConfigurePage: + return ui, tea.Quit + case key.Matches(msg, ui.common.KeyMap.Quit): + return ui, tea.Quit + case ui.activePage > 0 && key.Matches(msg, ui.common.KeyMap.Back): + ui.activePage -= 1 + return ui, nil + } + case tea.MouseMsg: + switch msg.Type { + case tea.MouseLeft: + } + } + case common.ErrorMsg: + return ui, nil + case selector.SelectMsg: + switch item := msg.IdentifiableItem.(type) { + case wizard_intro.Item: + switch item { + case wizard_intro.Quit: + cmds = append(cmds, tea.Quit) + case wizard_intro.ViewOSSProject: + ui.activePage = viewOSSProjectPage + case wizard_intro.ViewHelpDocs: + ui.args = []string{"--help"} + + return ui, tea.Batch(nil, tea.Quit) + case wizard_intro.EnterpriseInquire: + ui.activePage = contactEnterprisePage + case wizard_intro.ScanSourceWithWizard: + ui.activePage = sourceSelectPage + } + case source_select.SourceItem: + ui.activePage = sourceConfigurePage + cmds = append(cmds, func() tea.Msg { + return source_configure.SetSourceMsg{Source: item.ID()} + }) + } + case source_configure.SetArgsMsg: + ui.args = strings.Split(string(msg), " ")[1:] + return ui, tea.Quit + } + + if ui.state == loadedState { + m, cmd := ui.pages[ui.activePage].Update(msg) + ui.pages[ui.activePage] = m.(common.Component) + if cmd != nil { + cmds = append(cmds, cmd) + } + } + + // This fixes determining the height margin of the footer. + // ui.SetSize(ui.common.Width, ui.common.Height) + return ui, tea.Batch(cmds...) +} + +// View implements tea.Model. +func (ui *TUI) View() string { + var view string + switch ui.state { + case startState: + view = "Loading..." + case loadedState: + view = ui.pages[ui.activePage].View() + default: + view = "Unknown state :/ this is a bug!" + } + return ui.common.Zone.Scan( + ui.common.Styles.App.Render(view), + ) +} + +func Run() []string { + c := common.Common{ + Copy: nil, + Styles: styles.DefaultStyles(), + KeyMap: keymap.DefaultKeyMap(), + Width: 0, + Height: 0, + Zone: zone.New(), + } + m := New(c) + p := tea.NewProgram(m) + // TODO: Print normal help message. + if _, err := p.Run(); err != nil { + fmt.Printf("Alas, there's been an error: %v", err) + os.Exit(1) + } + return m.args +} diff --git a/proto/detectors.proto b/proto/detectors.proto index 909e5936..e1ee6a64 100644 --- a/proto/detectors.proto +++ b/proto/detectors.proto @@ -932,6 +932,9 @@ enum DetectorType { Dockerhub = 921; TrufflehogEnterprise = 922; EnvoyApiKey = 923; + GitHubOauth2 = 924; + Salesforce = 925; + HuggingFace = 926; } message Result { diff --git a/proto/sources.proto b/proto/sources.proto index 6c70b9a2..fafed4eb 100644 --- a/proto/sources.proto +++ b/proto/sources.proto @@ -227,6 +227,7 @@ message S3 { } repeated string buckets = 3; int64 max_object_size = 6; + repeated string roles = 7; } message Slack {