Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Dustin Decker
|
843dd7f056 |
@@ -0,0 +1,15 @@
|
||||
# To get started with Dependabot version updates, you'll need to specify which
|
||||
# package ecosystems to update and where the package manifests are located.
|
||||
# Please see the documentation for all configuration options:
|
||||
# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "gomod" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||
"extends": [
|
||||
"config:base"
|
||||
],
|
||||
"prConcurrentLimit": 3,
|
||||
"prHourlyLimit": 2
|
||||
}
|
||||
+3
-3
@@ -8,9 +8,9 @@ RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .
|
||||
|
||||
FROM alpine:3.18
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
FROM alpine:3.15
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates \
|
||||
&& update-ca-certificates
|
||||
COPY --from=builder /build/trufflehog /usr/bin/trufflehog
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
RUN chmod +x /etc/entrypoint.sh
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
FROM alpine:3.18
|
||||
FROM alpine:3.15
|
||||
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates rpm2cpio \
|
||||
&& rm -rf /var/cache/apk/* && update-ca-certificates
|
||||
RUN apk add --no-cache bash git openssh-client ca-certificates \
|
||||
&& rm -rf /var/cache/apk/* && \
|
||||
update-ca-certificates
|
||||
WORKDIR /usr/bin/
|
||||
COPY trufflehog .
|
||||
COPY entrypoint.sh /etc/entrypoint.sh
|
||||
|
||||
@@ -52,10 +52,10 @@ run-debug:
|
||||
CGO_ENABLED=0 go run . git file://. --json --debug
|
||||
|
||||
protos:
|
||||
docker run --rm -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
|
||||
docker run -u "$(shell id -u)" -v "$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; /pwd/scripts/gen_proto.sh"
|
||||
|
||||
protos-windows:
|
||||
docker run --rm -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
docker run -v "$(shell cygpath -w $(shell pwd))":/pwd "${PROTOS_IMAGE}" bash -c "cd /pwd; ./scripts/gen_proto.sh"
|
||||
|
||||
release-protos-image:
|
||||
docker buildx build --push --platform=linux/amd64,linux/arm64 \
|
||||
|
||||
@@ -164,7 +164,7 @@ trufflehog docker --image trufflesecurity/secrets --only-verified
|
||||
|
||||
+ All I see is `🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷` and the program exits, what gives?
|
||||
+ That means no secrets were detected
|
||||
+ Why is the scan taking a long time when I scan a GitHub org
|
||||
+ Why is the scan is taking a long time when I scan a GitHub org
|
||||
+ Unauthenticated GitHub scans have rate limits. To improve your rate limits, include the `--token` flag with a personal access token
|
||||
+ It says a private key was verified, what does that mean?
|
||||
+ Check out our Driftwood blog post to learn how to do this, in short we've confirmed the key can be used live for SSH or SSL [Blog post](https://trufflesecurity.com/blog/driftwood-know-if-private-keys-are-sensitive/)
|
||||
@@ -219,7 +219,7 @@ Flags:
|
||||
--concurrency=10 Number of concurrent workers.
|
||||
--no-verification Don't verify the results.
|
||||
--only-verified Only output verified results.
|
||||
--filter-unverified Only output first unverified result per chunk per detector if there is more than one result.
|
||||
--filter-unverified Only output first unverified result per chunk per detector if there are more than one results.
|
||||
--config=CONFIG Path to configuration file.
|
||||
--print-avg-detector-time Print the average time spent on each detector.
|
||||
--no-update Don't check for updates.
|
||||
|
||||
@@ -2,17 +2,21 @@ module github.com/trufflesecurity/trufflehog/v3
|
||||
|
||||
go 1.21
|
||||
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.2.7
|
||||
replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
|
||||
|
||||
// go wants to pull in the latest but after v10.2.5 has a breaking API change
|
||||
replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v10.2.4
|
||||
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.11.2
|
||||
cloud.google.com/go/secretmanager v1.11.1
|
||||
cloud.google.com/go/storage v1.33.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0
|
||||
github.com/BobuSumisu/aho-corasick v1.0.3
|
||||
github.com/TheZeroSlave/zapsentry v1.17.0
|
||||
github.com/aws/aws-sdk-go v1.45.19
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
|
||||
github.com/bradleyfalzon/ghinstallation/v2 v2.7.0
|
||||
@@ -20,14 +24,14 @@ require (
|
||||
github.com/charmbracelet/bubbletea v0.24.2
|
||||
github.com/charmbracelet/glamour v0.6.0
|
||||
github.com/charmbracelet/lipgloss v0.7.1
|
||||
github.com/couchbase/gocb/v2 v2.6.5
|
||||
github.com/couchbase/gocb/v2 v2.6.3
|
||||
github.com/crewjam/rfc5424 v0.1.0
|
||||
github.com/denisenkom/go-mssqldb v0.12.3
|
||||
github.com/envoyproxy/protoc-gen-validate v1.0.2
|
||||
github.com/fatih/color v1.15.0
|
||||
github.com/felixge/fgprof v0.9.3
|
||||
github.com/getsentry/sentry-go v0.24.1
|
||||
github.com/go-errors/errors v1.5.1
|
||||
github.com/go-errors/errors v1.4.2
|
||||
github.com/go-git/go-git/v5 v5.8.1
|
||||
github.com/go-ldap/ldap/v3 v3.4.6
|
||||
github.com/go-logr/logr v1.2.4
|
||||
@@ -36,24 +40,22 @@ require (
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/gobwas/glob v0.2.3
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/google/go-cmp v0.5.9
|
||||
github.com/google/go-containerregistry v0.15.2
|
||||
github.com/google/go-github/v42 v42.0.0
|
||||
github.com/google/go-github/v56 v56.0.0
|
||||
github.com/googleapis/gax-go/v2 v2.12.0
|
||||
github.com/h2non/filetype v1.1.3
|
||||
github.com/hashicorp/go-retryablehttp v0.7.4
|
||||
github.com/hashicorp/golang-lru v0.5.1
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/jpillora/overseer v1.1.6
|
||||
github.com/kylelemons/godebug v1.1.0
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.1
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
|
||||
github.com/marusama/semaphore/v2 v2.5.0
|
||||
github.com/mattn/go-isatty v0.0.20
|
||||
github.com/mattn/go-isatty v0.0.19
|
||||
github.com/mattn/go-sqlite3 v1.14.17
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.8
|
||||
github.com/muesli/reflow v0.3.0
|
||||
@@ -63,24 +65,22 @@ require (
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/snowflakedb/gosnowflake v1.6.25
|
||||
github.com/snowflakedb/gosnowflake v1.6.23
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/trufflesecurity/disk-buffer-reader v0.1.9
|
||||
github.com/xanzy/go-gitlab v0.92.3
|
||||
go.mongodb.org/mongo-driver v1.12.1
|
||||
go.uber.org/mock v0.2.0
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.14.0
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
|
||||
golang.org/x/oauth2 v0.13.0
|
||||
golang.org/x/sync v0.4.0
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
|
||||
golang.org/x/oauth2 v0.12.0
|
||||
golang.org/x/sync v0.3.0
|
||||
golang.org/x/text v0.13.0
|
||||
google.golang.org/api v0.132.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
pgregory.net/rapid v1.1.0
|
||||
sigs.k8s.io/yaml v1.3.0
|
||||
)
|
||||
|
||||
@@ -139,7 +139,7 @@ require (
|
||||
github.com/connesc/cipherio v0.2.1 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.9 // indirect
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 // indirect
|
||||
github.com/danieljoos/wincred v1.1.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dimchansky/utfbom v1.1.1 // indirect
|
||||
@@ -192,7 +192,7 @@ require (
|
||||
github.com/launchdarkly/eventsource v1.6.2 // indirect
|
||||
github.com/launchdarkly/go-jsonstream/v3 v3.0.0 // indirect
|
||||
github.com/launchdarkly/go-sdk-common/v3 v3.0.1 // indirect
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 // indirect
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 // indirect
|
||||
github.com/launchdarkly/go-semver v1.0.2 // indirect
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
@@ -245,12 +245,12 @@ require (
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
|
||||
golang.org/x/mod v0.13.0 // indirect
|
||||
golang.org/x/mod v0.12.0 // indirect
|
||||
golang.org/x/net v0.17.0 // indirect
|
||||
golang.org/x/sys v0.13.0 // indirect
|
||||
golang.org/x/term v0.13.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.14.0 // indirect
|
||||
golang.org/x/tools v0.12.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect
|
||||
|
||||
@@ -22,8 +22,6 @@ cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2k
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE4+zkuEQs=
|
||||
cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw=
|
||||
cloud.google.com/go/secretmanager v1.11.2 h1:52Z78hH8NBWIqbvIG0wi0EoTaAmSx99KIOAmDXIlX0M=
|
||||
cloud.google.com/go/secretmanager v1.11.2/go.mod h1:MQm4t3deoSub7+WNwiC4/tRYgDBHJgJPvswqQVB1Vss=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
|
||||
@@ -54,8 +52,6 @@ github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQr
|
||||
github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA=
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk=
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg=
|
||||
github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4=
|
||||
github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg=
|
||||
github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
|
||||
@@ -151,8 +147,6 @@ github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
|
||||
github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1 h1:q2sWUyDcozPLcLabEMd+a+7Ea2DitxZVN9hTxab9L4E=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.1/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2 h1:NT7wkhEhPTcKnBCdPi9djmyy9L3JOL4+3SsfJyqptCo=
|
||||
github.com/aymanbagabas/go-osc52 v1.2.2/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
|
||||
@@ -161,6 +155,8 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7 h1:oPgDjsbeqErOrFJeWID/51LhIOGjU3cqN1Tj3V6RiwE=
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
|
||||
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
|
||||
@@ -208,12 +204,8 @@ github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSk
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o=
|
||||
github.com/couchbase/gocb/v2 v2.6.3 h1:5RsMo+RRfK0mVxHLAfpBz3/tHlgXZb1WBNItLk9Ab+c=
|
||||
github.com/couchbase/gocb/v2 v2.6.3/go.mod h1:yF5F6BHTZ/ZowhEuZbySbXrlI4rHd1TIhm5azOaMbJU=
|
||||
github.com/couchbase/gocb/v2 v2.6.5 h1:xaZu29o8UJEV1ZQ3n2s9jcRCUHz/JsQ6+y6JBnVsy5A=
|
||||
github.com/couchbase/gocb/v2 v2.6.5/go.mod h1:0vFM09y+VPhnXeNrIb8tS0wKHGpJvjJBrJnriWEiwGs=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4 h1:TDTQ1mSBUw9eajuV71ZDJcLRbmfGcsvZV9SMeUDAVGc=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.4/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.9 h1:zph/+ceu3JtZEDKhJMTRc6lGrahq+mnlQY/1dSepJuE=
|
||||
github.com/couchbase/gocbcore/v10 v10.2.9/go.mod h1:lYQIIk+tzoMcwtwU5GzPbDdqEkwkH3isI2rkSpfL0oM=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230307083111-cc3960c624b1/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259 h1:2TXy68EGEzIMHOx9UvczR5ApVecwCfQZ0LjkmwMI6g4=
|
||||
github.com/couchbaselabs/gocaves/client v0.0.0-20230404095311-05e3ba4f0259/go.mod h1:AVekAZwIY2stsJOMWLAS/0uA/+qdp7pjO8EHnl61QkY=
|
||||
@@ -279,8 +271,6 @@ github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
|
||||
github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
|
||||
github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
|
||||
@@ -373,15 +363,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
|
||||
github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
|
||||
github.com/google/go-github/v42 v42.0.0 h1:YNT0FwjPrEysRkLIiKuEfSvBPCGKphW5aS5PxwaoLec=
|
||||
github.com/google/go-github/v42 v42.0.0/go.mod h1:jgg/jvyI0YlDOM1/ps6XYh04HNQ3vKf0CVko62/EhRg=
|
||||
github.com/google/go-github/v55 v55.0.0 h1:4pp/1tNMB9X/LuAhs5i0KQAE40NmiR/y6prLNb9x9cg=
|
||||
github.com/google/go-github/v55 v55.0.0/go.mod h1:JLahOTA1DnXzhxEymmFF5PP2tSS9JVNj68mSZNDwskA=
|
||||
github.com/google/go-github/v56 v56.0.0/go.mod h1:D8cdcX98YWJvi7TLo7zM4/h8ZTx6u6fwGEkCdisopo0=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
|
||||
@@ -431,8 +418,6 @@ github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
@@ -488,16 +473,12 @@ github.com/launchdarkly/go-sdk-common/v3 v3.0.1 h1:rVdLusAIViduNvyjNKy06RA+SPwk0
|
||||
github.com/launchdarkly/go-sdk-common/v3 v3.0.1/go.mod h1:H/zISoCNhviHTTqqBjIKQy2YgSHT8ioL1FtgBKpiEGg=
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.1 h1:vnUN2Y7og/5wtOCcCZW7wYpmZcS++GAyclasc7gaTIY=
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.1/go.mod h1:Msqbl6brgFO83RUxmLaJAUx2sYG+WKULcy+Vf3+tKww=
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.2 h1:Ngp050AGYUyN3YRxX7zfu110NETHEPEOVys1ey2Hyvw=
|
||||
github.com/launchdarkly/go-sdk-events/v2 v2.0.2/go.mod h1:Msqbl6brgFO83RUxmLaJAUx2sYG+WKULcy+Vf3+tKww=
|
||||
github.com/launchdarkly/go-semver v1.0.2 h1:sYVRnuKyvxlmQCnCUyDkAhtmzSFRoX6rG2Xa21Mhg+w=
|
||||
github.com/launchdarkly/go-semver v1.0.2/go.mod h1:xFmMwXba5Mb+3h72Z+VeSs9ahCvKo2QFUTHRNHVqR28=
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2 h1:PAM0GvE0nIUBeOkjdiymIEKI+8FFLJ+fEsWTupW1yGU=
|
||||
github.com/launchdarkly/go-server-sdk-evaluation/v2 v2.0.2/go.mod h1:Mztipcz+7ZMatXVun3k/IfPa8IOgUnAqiZawtFh2MRg=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0 h1:pp/VBIon5JNbtSw7ih7I5MXN9mXHfo6T5xikKVy52dI=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.0/go.mod h1:3TfS6vNsNksPT4LGeGuGKtT3zTjTbbOsCUK3nrj1neA=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.1 h1:L57BC9Fy4IJG0zRZoYBkoxko6QTAC2OF8QzTVUqn2p8=
|
||||
github.com/launchdarkly/go-server-sdk/v6 v6.1.1/go.mod h1:67QqqyeRNyXx7S/fZiSZesRvaePCyxhzsnY+4fhK55U=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0 h1:L3kGILP/6ewikhzhdNkHy1b5y4zs50LueWenVF0sBbs=
|
||||
github.com/launchdarkly/go-test-helpers/v2 v2.2.0/go.mod h1:L7+th5govYp5oKU9iN7To5PgznBuIjBPn+ejqKR0avw=
|
||||
github.com/launchdarkly/go-test-helpers/v3 v3.0.2 h1:rh0085g1rVJM5qIukdaQ8z1XTWZztbJ49vRZuveqiuU=
|
||||
@@ -506,8 +487,6 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e h1:XoxHx8K6ZKoMtjzWOMDuM69LCdjDDsTOtTfWGrT/fns=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9 h1:+7bxeCzFs4bfFPAnIZrjNmRt/MCffIy7aw2mPc9mxkU=
|
||||
github.com/lrstanley/bubblezone v0.0.0-20230911164824-e3824f1adde9/go.mod h1:v5lEwWaguF1o2MW/ucO0ZIA/IZymdBYJJ+2cMRLE7LU=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
@@ -521,8 +500,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
|
||||
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||
@@ -639,8 +616,6 @@ github.com/smartystreets/gunit v1.1.3 h1:32x+htJCu3aMswhPw3teoJ+PnWPONqdNgaGs6Qt
|
||||
github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ=
|
||||
github.com/snowflakedb/gosnowflake v1.6.23 h1:uO+zMTXJcSHzOm6ks5To8ergNjt5Dy6cr5QtStpRFT8=
|
||||
github.com/snowflakedb/gosnowflake v1.6.23/go.mod h1:KfO4F7bk+aXPUIvBqYxvPhxLlu2/w4TtSC8Rw/yr5Mg=
|
||||
github.com/snowflakedb/gosnowflake v1.6.25 h1:o5zUmxTOo0Eo9AdkEj8blCeiMuILrQJ+rjUMAeZhcRE=
|
||||
github.com/snowflakedb/gosnowflake v1.6.25/go.mod h1:KfO4F7bk+aXPUIvBqYxvPhxLlu2/w4TtSC8Rw/yr5Mg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
@@ -662,12 +637,8 @@ github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HP
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
|
||||
github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
|
||||
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
|
||||
github.com/trufflesecurity/disk-buffer-reader v0.1.9 h1:1dji/Cx5aPL5DlsqI1B86xFMo6OfbBEhkQhXxgbX9WA=
|
||||
github.com/trufflesecurity/disk-buffer-reader v0.1.9/go.mod h1:uYwTCdxzV0o+qaeBMxflOsq4eu2WjrE46qGR2e80O9Y=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
|
||||
github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
|
||||
github.com/trufflesecurity/overseer v1.2.7 h1:DEiu2wrL/0f+XqshIT9C4/0nj7SHZvLmFeE4jELh4Oo=
|
||||
github.com/trufflesecurity/overseer v1.2.7/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
|
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
|
||||
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
@@ -749,8 +720,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
|
||||
golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -775,8 +744,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
|
||||
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -816,8 +783,6 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
|
||||
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
|
||||
golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
|
||||
golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -830,8 +795,6 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
|
||||
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -942,8 +905,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss=
|
||||
golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM=
|
||||
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -1052,8 +1013,6 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
||||
pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
|
||||
pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
package glob
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
)
|
||||
|
||||
// Filter is a generic filter for excluding and including globs (limited
|
||||
// regular expressions). Exclusion takes precedence if both include and exclude
|
||||
// lists are provided.
|
||||
type Filter struct {
|
||||
exclude []glob.Glob
|
||||
include []glob.Glob
|
||||
}
|
||||
|
||||
type globFilterOpt func(*Filter) error
|
||||
|
||||
// WithExcludeGlobs adds exclude globs to the filter.
|
||||
func WithExcludeGlobs(excludes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, exclude := range excludes {
|
||||
g, err := glob.Compile(exclude)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid exclude glob %q: %w", exclude, err)
|
||||
}
|
||||
f.exclude = append(f.exclude, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithIncludeGlobs adds include globs to the filter.
|
||||
func WithIncludeGlobs(includes ...string) globFilterOpt {
|
||||
return func(f *Filter) error {
|
||||
for _, include := range includes {
|
||||
g, err := glob.Compile(include)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid include glob %q: %w", include, err)
|
||||
}
|
||||
f.include = append(f.include, g)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewGlobFilter creates a new Filter with the provided options.
|
||||
func NewGlobFilter(opts ...globFilterOpt) (*Filter, error) {
|
||||
filter := &Filter{}
|
||||
for _, opt := range opts {
|
||||
if err := opt(filter); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
// ShouldInclude returns whether the object is in the include list or not in
|
||||
// the exclude list (exclude taking precedence).
|
||||
func (f *Filter) ShouldInclude(object string) bool {
|
||||
if f == nil {
|
||||
return true
|
||||
}
|
||||
exclude, include := len(f.exclude), len(f.include)
|
||||
if exclude == 0 && include == 0 {
|
||||
return true
|
||||
} else if exclude > 0 && include == 0 {
|
||||
return f.shouldIncludeFromExclude(object)
|
||||
} else if exclude == 0 && include > 0 {
|
||||
return f.shouldIncludeFromInclude(object)
|
||||
} else {
|
||||
if ok, err := f.shouldIncludeFromBoth(object); err == nil {
|
||||
return ok
|
||||
}
|
||||
// Ambiguous case.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// shouldIncludeFromExclude checks for explicitly excluded paths. This should
|
||||
// only be called when the include list is empty.
|
||||
func (f *Filter) shouldIncludeFromExclude(object string) bool {
|
||||
for _, glob := range f.exclude {
|
||||
if glob.Match(object) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// shouldIncludeFromInclude checks for explicitly included paths. This should
|
||||
// only be called when the exclude list is empty.
|
||||
func (f *Filter) shouldIncludeFromInclude(object string) bool {
|
||||
for _, glob := range f.include {
|
||||
if glob.Match(object) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// shouldIncludeFromBoth checks for either excluded or included paths. Exclusion
|
||||
// takes precedence. If neither list contains the object, true is returned.
|
||||
func (f *Filter) shouldIncludeFromBoth(object string) (bool, error) {
|
||||
// Exclude takes precedence. If we find the object in the exclude list,
|
||||
// we should not match.
|
||||
for _, glob := range f.exclude {
|
||||
if glob.Match(object) {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
// If we find the object in the include list, we should match.
|
||||
for _, glob := range f.include {
|
||||
if glob.Match(object) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
// If we find it in neither, return an error to let the caller decide.
|
||||
return false, fmt.Errorf("ambiguous match")
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
package glob
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"pgregory.net/rapid"
|
||||
)
|
||||
|
||||
type globTest struct {
|
||||
input string
|
||||
shouldInclude bool
|
||||
}
|
||||
|
||||
func testGlobs(t *testing.T, filter *Filter, tests ...globTest) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
// Invert because mentally it's easier to say whether an
|
||||
// input should be included.
|
||||
assert.Equal(t, tt.shouldInclude, filter.ShouldInclude(tt.input))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobFilterExclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"bar", false},
|
||||
globTest{"bara", false},
|
||||
globTest{"barb", false},
|
||||
globTest{"barbosa", false},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithIncludeGlobs("foo", "bar*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", false},
|
||||
globTest{"food", false},
|
||||
globTest{"anything else", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterEmpty(t *testing.T) {
|
||||
filter, err := NewGlobFilter()
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", true},
|
||||
globTest{"bar", true},
|
||||
globTest{"bara", true},
|
||||
globTest{"barb", true},
|
||||
globTest{"barbosa", true},
|
||||
globTest{"foobar", true},
|
||||
globTest{"food", true},
|
||||
globTest{"anything else", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludeInclude(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("/foo/bar/**"), WithIncludeGlobs("/foo/**"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"/foo/a", true},
|
||||
globTest{"/foo/b", true},
|
||||
globTest{"/foo/c/d/e", true},
|
||||
globTest{"/foo/bar/a", false},
|
||||
globTest{"/foo/bar/b", false},
|
||||
globTest{"/foo/bar/c/d/e", false},
|
||||
globTest{"/any/other/path", false},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobFilterExcludePrecedence(t *testing.T) {
|
||||
filter, err := NewGlobFilter(WithExcludeGlobs("foo"), WithIncludeGlobs("foo*"))
|
||||
assert.NoError(t, err)
|
||||
|
||||
testGlobs(t, filter,
|
||||
globTest{"foo", false},
|
||||
globTest{"foobar", true},
|
||||
)
|
||||
}
|
||||
|
||||
func TestGlobErrorContainsGlob(t *testing.T) {
|
||||
invalidGlob := "[this is invalid because it doesn't close the capture group"
|
||||
_, err := NewGlobFilter(WithExcludeGlobs(invalidGlob))
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), invalidGlob)
|
||||
}
|
||||
|
||||
// The filters in this test should be mutually exclusive because one includes
|
||||
// and the other excludes the same glob.
|
||||
func TestGlobInverse(t *testing.T) {
|
||||
for _, glob := range []string{
|
||||
"a",
|
||||
"a*",
|
||||
"a**",
|
||||
"*a",
|
||||
"**a",
|
||||
"*",
|
||||
} {
|
||||
include, err := NewGlobFilter(WithIncludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
exclude, err := NewGlobFilter(WithExcludeGlobs(glob))
|
||||
assert.NoError(t, err)
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
input := rapid.String().Draw(t, "input")
|
||||
a, b := include.ShouldInclude(input), exclude.ShouldInclude(input)
|
||||
if a == b {
|
||||
t.Fatalf("Filter(Include(%q)) == Filter(Exclude(%q)) == %v for input %q", glob, glob, a, input)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlobDefaultFilters(t *testing.T) {
|
||||
for _, filter := range []*Filter{nil, {}} {
|
||||
rapid.Check(t, func(t *rapid.T) {
|
||||
if !filter.ShouldInclude(rapid.String().Draw(t, "input")) {
|
||||
t.Fatalf("filter %#v did not include input", filter)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -26,8 +25,7 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
|
||||
func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
|
||||
decodedSubstrings := make(map[string][]byte)
|
||||
|
||||
@@ -63,7 +61,7 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
}
|
||||
result.Write(chunk.Data[start:])
|
||||
chunk.Data = result.Bytes()
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package decoders
|
||||
|
||||
import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
@@ -14,15 +13,8 @@ func DefaultDecoders() []Decoder {
|
||||
}
|
||||
}
|
||||
|
||||
// DecodableChunk is a chunk that includes the type of decoder used.
|
||||
// This allows us to avoid a type assertion on each decoder.
|
||||
type DecodableChunk struct {
|
||||
*sources.Chunk
|
||||
DecoderType detectorspb.DecoderType
|
||||
}
|
||||
|
||||
type Decoder interface {
|
||||
FromChunk(chunk *sources.Chunk) *DecodableChunk
|
||||
FromChunk(chunk *sources.Chunk) *sources.Chunk
|
||||
}
|
||||
|
||||
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
|
||||
|
||||
@@ -5,24 +5,22 @@ import (
|
||||
"encoding/binary"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF16 struct{}
|
||||
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
|
||||
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
|
||||
if len(utf16Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
chunk.Data = utf16Data
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -4,25 +4,22 @@ import (
|
||||
"bytes"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
)
|
||||
|
||||
type UTF8 struct{}
|
||||
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
|
||||
func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
|
||||
if chunk == nil || len(chunk.Data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
|
||||
|
||||
if !utf8.Valid(chunk.Data) {
|
||||
chunk.Data = extractSubstrings(chunk.Data)
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
return decodableChunk
|
||||
return chunk
|
||||
}
|
||||
|
||||
// extractSubstrings performs similarly to the strings binutil,
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
package privacy
|
||||
package baseapiio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,26 +13,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"privacy"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"baseapi", "base-api"}) + `\b([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"privacy"}
|
||||
return []string{"baseapi", "base-api"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Privacy secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify BaseApiIO secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -44,38 +44,34 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
DetectorType: detectorspb.DetectorType_BaseApiIO,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.privacy.com/v1/card?page=1&page_size=50", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.base-api.io/v1/forms?page=1&per_page=10", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Authorization", "api-key "+resMatch)
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
if strings.Contains(body, "items") {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
@@ -85,5 +81,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Privacy
|
||||
return detectorspb.DetectorType_BaseApiIO
|
||||
}
|
||||
+23
-27
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package instamojo
|
||||
package baseapiio
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -16,16 +16,15 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestInstamojo_FromChunk(t *testing.T) {
|
||||
func TestBaseApiIO_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
id := testSecrets.MustGetField("INSTAMOJO_CLIENT_ID")
|
||||
secret := testSecrets.MustGetField("INSTAMOJO_SECRET")
|
||||
inactiveSecret := testSecrets.MustGetField("INSTAMOJO_INACTIVE")
|
||||
secret := testSecrets.MustGetField("BASEAPIIO")
|
||||
inactiveSecret := testSecrets.MustGetField("BASEAPIIO_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -33,46 +32,43 @@ func TestInstamojo_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within id %s", secret, id)),
|
||||
data: []byte(fmt.Sprintf("You can find a baseapiio secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Instamojo,
|
||||
DetectorType: detectorspb.DetectorType_BaseApiIO,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a instamojo secret %s within but not valid, within id %s", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a baseapiio secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Instamojo,
|
||||
VerificationError: fmt.Errorf("unexpected HTTP response status 401"),
|
||||
DetectorType: detectorspb.DetectorType_BaseApiIO,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -82,16 +78,16 @@ func TestInstamojo_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Instamojo.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("BaseApiIO.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
@@ -101,7 +97,7 @@ func TestInstamojo_FromChunk(t *testing.T) {
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Instamojo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
t.Errorf("BaseApiIO.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package ip2location
|
||||
package blablabus
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -12,24 +12,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ip2location"}) + `\b([0-9A-Z]{32})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"blablabus"}) + `\b([0-9A-Za-z]{22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"ip2location"}
|
||||
return []string{"blablabus"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Ip2location secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify Blablabus secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -42,33 +43,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Ip2location,
|
||||
DetectorType: detectorspb.DetectorType_Blablabus,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.ip2location.io/?key=" + resMatch, nil)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.idbus.com/v3/stops", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
@@ -77,8 +75,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Ip2location
|
||||
return detectorspb.DetectorType_Blablabus
|
||||
}
|
||||
|
||||
|
||||
|
||||
+25
-36
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package portainer
|
||||
package blablabus
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,25 +9,22 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestPortainer_FromChunk(t *testing.T) {
|
||||
func TestBlablabus_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PORTAINER")
|
||||
endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
|
||||
inactiveSecret := testSecrets.MustGetField("PORTAINER_INACTIVE")
|
||||
secret := testSecrets.MustGetField("BLABLABUS")
|
||||
inactiveSecret := testSecrets.MustGetField("BLABLABUS_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -35,48 +32,43 @@ func TestPortainer_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer url %s", secret, endpoint)),
|
||||
data: []byte(fmt.Sprintf("You can find a blablabus secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Portainer,
|
||||
DetectorType: detectorspb.DetectorType_Blablabus,
|
||||
Verified: true,
|
||||
RawV2: []byte(secret + endpoint),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a portainer secret %s for portainer %s within but not valid", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a blablabus secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Portainer,
|
||||
DetectorType: detectorspb.DetectorType_Blablabus,
|
||||
Verified: false,
|
||||
RawV2: []byte(inactiveSecret + endpoint),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -86,29 +78,26 @@ func TestPortainer_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Portainer.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("Blablabus.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Portainer.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("Blablabus.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package cloudsmith
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -30,10 +29,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"cloudsmith"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Authenticated bool `json:"authenticated"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Cloudsmith secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -62,14 +57,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var r response
|
||||
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if r.Authenticated {
|
||||
s1.Verified = true
|
||||
}
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
@@ -2,7 +2,6 @@ package codeclimate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
@@ -31,12 +30,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"codeclimate"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Data struct {
|
||||
Id string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Codeclimate secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -65,14 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var r response
|
||||
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if r.Data.Id != "" {
|
||||
s1.Verified = true
|
||||
}
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
+14
-26
@@ -1,4 +1,4 @@
|
||||
package vagrantcloudpersonaltoken
|
||||
package datafire
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -12,26 +12,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"vagrant"}) + `\b([A-Za-z0-9]{14}.atlasv1.[A-Za-z0-9]{67})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"datafire"}) + `\b([a-z0-9\S]{175,190})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"vagrant"}
|
||||
return []string{"datafire"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Vagrantcloudpersonaltoken secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify DataFire secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -44,17 +43,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
|
||||
DetectorType: detectorspb.DetectorType_DataFire,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.vagrantup.com/api/v2/authenticate", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.datafire.io/projects/", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -64,21 +58,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
@@ -86,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_VagrantCloudPersonalToken
|
||||
return detectorspb.DetectorType_DataFire
|
||||
}
|
||||
+25
-33
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package ip2location
|
||||
package datafire
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,24 +9,22 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestIp2location_FromChunk(t *testing.T) {
|
||||
func TestDataFire_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("IP2LOCATION")
|
||||
inactiveSecret := testSecrets.MustGetField("IP2LOCATION_INACTIVE")
|
||||
secret := testSecrets.MustGetField("DATAFIRE")
|
||||
inactiveSecret := testSecrets.MustGetField("DATAFIRE_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -34,46 +32,43 @@ func TestIp2location_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ip2location secret %s within", secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a datafire secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Ip2location,
|
||||
DetectorType: detectorspb.DetectorType_DataFire,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ip2location secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a datafire secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Ip2location,
|
||||
DetectorType: detectorspb.DetectorType_DataFire,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -83,29 +78,26 @@ func TestIp2location_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Ip2location.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("DataFire.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Ip2location.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("DataFire.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -59,8 +59,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
if verify {
|
||||
// thanks https://stackoverflow.com/questions/15621471/validate-a-facebook-app-id-and-app-secret
|
||||
// https://stackoverflow.com/questions/24401241/how-to-get-a-facebook-access-token-using-appid-and-app-secret-without-any-login
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://graph.facebook.com/me?access_token=%s|%s", apiIdRes, apiSecretRes), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://graph.facebook.com/%s?fields=roles&access_token=%s|%s", apiIdRes, apiIdRes, apiSecretRes), nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -97,6 +97,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
s1.ExtraData["site_admin"] = fmt.Sprintf("%t", userResponse.SiteAdmin)
|
||||
s1.ExtraData["name"] = userResponse.Name
|
||||
s1.ExtraData["company"] = userResponse.Company
|
||||
s1.ExtraData["scopes"] = res.Header.Get("X-OAuth-Scopes")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ func TestGitHub_FromChunk(t *testing.T) {
|
||||
unverifiedGhu := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHU")
|
||||
unverifiedGhs := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHS")
|
||||
unverifiedGhr := testSecrets.MustGetField("GITHUB_UNVERIFIED_GHR")
|
||||
verifiedGhp := testSecrets.MustGetField("GITHUB_VERIFIED_GHP")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -41,6 +42,32 @@ func TestGitHub_FromChunk(t *testing.T) {
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified ghp",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a github secret %s within", verifiedGhp)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Github,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"account_type": "User",
|
||||
"company": "",
|
||||
"name": "",
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
|
||||
"scopes": "notifications",
|
||||
"site_admin": "false",
|
||||
"url": "https://github.com/truffle-sandbox",
|
||||
"username": "truffle-sandbox",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified ghp",
|
||||
s: Scanner{},
|
||||
@@ -53,6 +80,9 @@ func TestGitHub_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Github,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -69,6 +99,9 @@ func TestGitHub_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Github,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -85,6 +118,9 @@ func TestGitHub_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Github,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -101,6 +137,9 @@ func TestGitHub_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Github,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -117,6 +156,9 @@ func TestGitHub_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Github,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
@@ -133,6 +175,9 @@ func TestGitHub_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Github,
|
||||
Verified: false,
|
||||
ExtraData: map[string]string{
|
||||
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
|
||||
@@ -38,6 +38,7 @@ type userRes struct {
|
||||
SiteAdmin bool `json:"site_admin"`
|
||||
Name string `json:"name"`
|
||||
Company string `json:"company"`
|
||||
UserURL string `json:"html_url"`
|
||||
}
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -91,6 +92,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
res.Body.Close()
|
||||
if err == nil {
|
||||
s1.Verified = true
|
||||
|
||||
if err == nil {
|
||||
s1.Verified = true
|
||||
s1.ExtraData["username"] = userResponse.Login
|
||||
s1.ExtraData["url"] = userResponse.UserURL
|
||||
s1.ExtraData["account_type"] = userResponse.Type
|
||||
s1.ExtraData["site_admin"] = fmt.Sprintf("%t", userResponse.SiteAdmin)
|
||||
s1.ExtraData["name"] = userResponse.Name
|
||||
s1.ExtraData["company"] = userResponse.Company
|
||||
s1.ExtraData["scopes"] = res.Header.Get("X-OAuth-Scopes")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
package ipinfo
|
||||
package glitterlyapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -12,26 +11,25 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ipinfo"}) + `\b([a-f0-9]{14})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"glitterlyapi"}) + `\b([0-9a-z-]{36})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"ipinfo"}
|
||||
return []string{"glitterlyapi"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Ipinfo secrets in a given set of bytes.
|
||||
// FromData will find and optionally verify GlitterlyAPI secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
@@ -44,40 +42,31 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://ipinfo.io/json?token="+resMatch, nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://www.glitterlyapi.com/auth", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("x-api-key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
fmt.Println(res.Status, resMatch)
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
@@ -85,5 +74,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_IPInfo
|
||||
return detectorspb.DetectorType_GlitterlyAPI
|
||||
}
|
||||
+25
-33
@@ -1,7 +1,7 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package vagrantcloudpersonaltoken
|
||||
package glitterlyapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -9,24 +9,22 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
|
||||
func TestGlitterlyAPI_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("VAGRANTCLOUDPERSONALTOKEN")
|
||||
inactiveSecret := testSecrets.MustGetField("VAGRANTCLOUDPERSONALTOKEN_INACTIVE")
|
||||
secret := testSecrets.MustGetField("GLITTERLYAPI")
|
||||
inactiveSecret := testSecrets.MustGetField("GLITTERLYAPI_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -34,46 +32,43 @@ func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a vagrantcloudpersonaltoken secret %s within", secret)),
|
||||
data: []byte(fmt.Sprintf("You can find a glitterlyapi secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
|
||||
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a vagrantcloudpersonaltoken secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
data: []byte(fmt.Sprintf("You can find a glitterlyapi secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_VagrantCloudPersonalToken,
|
||||
DetectorType: detectorspb.DetectorType_GlitterlyAPI,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
@@ -83,29 +78,26 @@ func TestVagrantcloudpersonaltoken_FromChunk(t *testing.T) {
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Vagrantcloudpersonaltoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
t.Errorf("GlitterlyAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Vagrantcloudpersonaltoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("GlitterlyAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,105 +0,0 @@
|
||||
package instamojo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
//KeyPat is client_id
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"instamojo"}) + `\b([0-9a-zA-Z]{40})\b`)
|
||||
//Secretpat is Client_secret
|
||||
secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"instamojo"}) + `\b([0-9a-zA-Z]{128})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"instamojo"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Instamojo secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
clientIdmatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range secretMatches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resSecret := strings.TrimSpace(match[1])
|
||||
|
||||
for _, clientIdMatch := range clientIdmatches {
|
||||
if len(clientIdMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
resClientId := strings.TrimSpace(clientIdMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Instamojo,
|
||||
Raw: []byte(resClientId),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
payload := strings.NewReader("grant_type=client_credentials&client_id=" + resClientId + "&client_secret=" + resSecret)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://api.instamojo.com/oauth2/token/", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
bodyBytes, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
body := string(bodyBytes)
|
||||
if (res.StatusCode >= 200 && res.StatusCode < 300) && strings.Contains(body, "access_token") {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Instamojo
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package ipinfo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestIpinfo_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("IPINFO")
|
||||
inactiveSecret := testSecrets.MustGetField("IPINFO_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a ipinfo secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_IPInfo,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Ipinfo.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Ipinfo.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package loggly
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.loggly\.com)\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loggly"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"loggly"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Loggly secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
keyMatches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range keyMatches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(match[1])
|
||||
|
||||
for _, domainMatch := range domainMatches {
|
||||
if len(domainMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
domainRes := strings.TrimSpace(domainMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Raw: []byte(key),
|
||||
RawV2: []byte(fmt.Sprintf("%s:%s", domainRes, key)),
|
||||
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domainRes+"/apiv2/customer", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", key))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(key, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Loggly
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package loggly
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestLoggly_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("LOGGLY")
|
||||
inactiveSecret := testSecrets.MustGetField("LOGGLY_INACTIVE")
|
||||
domain := testSecrets.MustGetField("LOGGLY_DOMAIN")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a loggly secret %s within loggly Domain %s", secret, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a loggly secret %s within loggly Domain %s but not valid", inactiveSecret, domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a loggly secret %s within %s", secret, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a loggly secret %s within %s", secret, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Loggly,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Loggly.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Loggly.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package meaningcloud
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
@@ -33,10 +32,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"meaningcloud"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
DeepTime float64 `json:"deepTime"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify MeaningCloud secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -83,14 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
var r response
|
||||
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if r.DeepTime > 0 {
|
||||
s1.Verified = true
|
||||
}
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
@@ -20,7 +20,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monday"}) + `\b(eyJ[A-Za-z0-9-_]{15,100}\.eyJ[A-Za-z0-9-_]{100,300}\.[A-Za-z0-9-_]{25,100})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"monday"}) + `\b(ey[a-zA-Z0-9_.]{210,225})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
func TestMonday_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
|
||||
@@ -48,16 +48,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.newrelic.com/v2/users.json", nil)
|
||||
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://api.eu.newrelic.com/v2/users.json", nil)
|
||||
if err != nil || errEU != nil {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("X-Api-Key", resMatch)
|
||||
reqEU.Header.Add("X-Api-Key", resMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
resEU, errEU := client.Do(reqEU)
|
||||
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
@@ -68,16 +63,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else if errEU == nil {
|
||||
defer resEU.Body.Close()
|
||||
if resEU.StatusCode >= 200 && resEU.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, false) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
package openvpn
|
||||
|
||||
import (
|
||||
"context"
|
||||
b64 "encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
|
||||
clientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openvpn"}) + `\b([A-Za-z0-9-]{3,40}\.[A-Za-z0-9-]{3,40})\b`)
|
||||
clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)
|
||||
domainPat = regexp.MustCompile(`\b(https?://[A-Za-z0-9-]+\.api\.openvpn\.com)\b`)
|
||||
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"openvpn"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Openvpn secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
domainMatches := domainPat.FindAllStringSubmatch(dataStr, -1)
|
||||
clientIdMatches := clientIDPat.FindAllStringSubmatch(dataStr, -1)
|
||||
clientSecretMatches := clientSecretPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, clientIdMatch := range clientIdMatches {
|
||||
clientIDRes := strings.TrimSpace(clientIdMatch[1])
|
||||
for _, clientSecretMatch := range clientSecretMatches {
|
||||
clientSecretRes := strings.TrimSpace(clientSecretMatch[1])
|
||||
for _, domainMatch := range domainMatches {
|
||||
domainRes := strings.TrimSpace(domainMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_OpenVpn,
|
||||
Raw: []byte(clientSecretRes),
|
||||
RawV2: []byte(clientIDRes + clientSecretRes),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
|
||||
payload := strings.NewReader("grant_type=client_credentials")
|
||||
// OpenVPN API is in beta, We'll have to update the API endpoint once
|
||||
// Docs: https://openvpn.net/cloud-docs/developer/creating-api-credentials.html
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", domainRes + "/api/beta/oauth/token", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
data := fmt.Sprintf("%s:%s", clientIDRes, clientSecretRes)
|
||||
sEnc := b64.StdEncoding.EncodeToString([]byte(data))
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
|
||||
req.Header.Add("content-type", "application/x-www-form-urlencoded")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(clientSecretRes, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_OpenVpn
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package openvpn
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestOpenvpn_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
|
||||
domain := testSecrets.MustGetField("OPENVPN_DOMAIN")
|
||||
clientId := testSecrets.MustGetField("OPENVPN_CLIENT_ID")
|
||||
clientSecret := testSecrets.MustGetField("OPENVPN_CLIENT_SECRET")
|
||||
inactiveClientSecret := testSecrets.MustGetField("OPENVPN_CLIENT_SECRET_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a openvpn secret %s openvpn clientId %s and domain %s within", clientSecret, clientId, domain)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_OpenVpn,
|
||||
Verified: true,
|
||||
RawV2: []byte(clientId + clientSecret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a openvpn secret %s openvpn clientId %s and domain %s within but not valid", inactiveClientSecret, clientId, domain)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_OpenVpn,
|
||||
Verified: false,
|
||||
RawV2: []byte(clientId + inactiveClientSecret),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Openvpn.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Openvpn.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ const verifyURL = "https://api.pagerduty.com/users"
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b(u\+[a-zA-Z0-9_+-]{18})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"pagerduty", "pager_duty", "pd_", "pd-"}) + `\b([a-zA-Z0-9_+-]{20})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
|
||||
@@ -2,7 +2,6 @@ package plaidkey
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -54,36 +53,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
DetectorType: detectorspb.DetectorType_PlaidKey,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
environments := []string{"development", "production"}
|
||||
|
||||
if verify {
|
||||
for _, env := range environments {
|
||||
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://"+env+".plaid.com/link/token/create", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
s1.ExtraData = map[string]string{"environment": fmt.Sprintf("https://%s.plaid.com", env)}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
payload := strings.NewReader(`{"client_id":"` + idresMatch + `","secret":"` + resMatch + `","user":{"client_user_id":"60e3ee4019a2660010f8bc54","phone_number_verified_time":"0001-01-01T00:00:00Z","email_address_verified_time":"0001-01-01T00:00:00Z"},"client_name":"Plaid Test App","products":["auth","transactions"],"country_codes":["US"],"webhook":"https://webhook-uri.com","account_filters":{"depository":{"account_subtypes":["checking","savings"]}},"language":"en","link_customization_name":"default"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://development.plaid.com/link/token/create", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
results = append(results, s1)
|
||||
// if the environment is dev, we don't need to check production
|
||||
if s1.Verified {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return results, nil
|
||||
|
||||
@@ -26,8 +26,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
|
||||
secret := testSecrets.MustGetField("PLAIDKEY_SECRET")
|
||||
inactiveSecret := testSecrets.MustGetField("PLAIDKEY_SECRET_INACTIVE")
|
||||
id := testSecrets.MustGetField("PLAIDKEY_CLIENTID")
|
||||
// env := testSecrets.MustGetField("PLAIDKEY_ENVIRONMENT") // development or production
|
||||
env := "development"
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
@@ -53,9 +51,6 @@ func TestPlaidKey_FromChunk(t *testing.T) {
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PlaidKey,
|
||||
Verified: true,
|
||||
ExtraData: map[string]string{
|
||||
"environment": fmt.Sprintf("https://%s.plaid.com", env),
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
package portainer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)
|
||||
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{50,310}\.[0-9A-Z-a-z\-_]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"portainer"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Portainer secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := tokenPat.FindAllStringSubmatch(dataStr, -1)
|
||||
endpointMatches := endpointPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, endpointMatch := range endpointMatches {
|
||||
resEndpointMatch := strings.TrimSpace(endpointMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Portainer,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resEndpointMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch + "/api/endpoints", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 || res.StatusCode == 403 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(endpointMatches) > 0 {
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Portainer
|
||||
}
|
||||
@@ -1,99 +0,0 @@
|
||||
package portainertoken
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainertoken"}) + `\b(ptr_[A-Za-z0-9\/_\-+=]{20,60})`)
|
||||
endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"portainertoken"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Portainertoken secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
endpointMatches := endpointPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
for _, endpointMatch := range endpointMatches {
|
||||
resEndpointMatch := strings.TrimSpace(endpointMatch[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_PortainerToken,
|
||||
Raw: []byte(resMatch),
|
||||
RawV2: []byte(resMatch + resEndpointMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch + "/api/stacks", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req.Header.Add("X-API-Key", resMatch)
|
||||
|
||||
res, err := client.Do(req)
|
||||
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
|
||||
}
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(endpointMatches) > 0 {
|
||||
results = append(results, s1)
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_PortainerToken
|
||||
}
|
||||
@@ -1,131 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package portainertoken
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestPortainertoken_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PORTAINERTOKEN")
|
||||
inactiveSecret := testSecrets.MustGetField("PORTAINERTOKEN_INACTIVE")
|
||||
endpoint := testSecrets.MustGetField("PORTAINER_ENDPOINT")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within for portainer url %s", secret, endpoint)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PortainerToken,
|
||||
Verified: true,
|
||||
RawV2: []byte(secret + endpoint),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a portainertoken secret %s within but not valid for portainer url %s", inactiveSecret, endpoint)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_PortainerToken,
|
||||
Verified: false,
|
||||
RawV2: []byte(inactiveSecret + endpoint),
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Portainertoken.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Portainertoken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -20,13 +20,13 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(`\b(phx_[a-zA-Z0-9_]{43})\b`)
|
||||
keyPat = regexp.MustCompile(`\b(phc_[a-zA-Z0-9_]{43})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"phx_"}
|
||||
return []string{"phc_"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify AppPosthog secrets in a given set of bytes.
|
||||
@@ -47,33 +47,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://app.posthog.com/api/event/?personal_api_key="+resMatch, nil)
|
||||
reqEU, errEU := http.NewRequestWithContext(ctx, "GET", "https://eu.posthog.com/api/event/?personal_api_key="+resMatch, nil)
|
||||
|
||||
if err != nil || errEU != nil {
|
||||
payload := strings.NewReader(` {
|
||||
"api_key": "` + resMatch + `",
|
||||
"distinct_id": "1234"
|
||||
}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://app.posthog.com/decide/", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
reqEU.Header.Add("Content-Type", "application/json")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == 401 {
|
||||
//Try EU Endpoint only if other one fails.
|
||||
res, err := client.Do(reqEU)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package privacy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestPrivacy_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("PRIVACY")
|
||||
inactiveSecret := testSecrets.MustGetField("PRIVACY_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a privacy secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Privacy,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Privacy.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Privacy.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package quickmetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct{}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"quickmetrics"}) + `\b([a-zA-Z0-9_-]{22})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"quickmetrics"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify QuickMetrics secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_QuickMetrics,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
payload := strings.NewReader(`[{"name":"api.response.time","dimension":null,"values":[[1568319841,12.4],[1568319856,9.3],[1568319860,234],[1568319863,3.2]]},{"name":"click.color","dimension":"green","values":[[1568319841,1],[1568319856,1],[1568319860,1],[1568319863,1]]}]`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://qckm.io/list", payload)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("x-qm-key", resMatch)
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_QuickMetrics
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package quickmetrics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestQuickMetrics_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors1")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("QUICKMETRICS")
|
||||
inactiveSecret := testSecrets.MustGetField("QUICKMETRICS_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a quickmetrics secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_QuickMetrics,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a quickmetrics secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_QuickMetrics,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("QuickMetrics.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
got[i].Raw = nil
|
||||
}
|
||||
if diff := pretty.Compare(got, tt.want); diff != "" {
|
||||
t.Errorf("QuickMetrics.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package screenshotapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -31,10 +30,6 @@ func (s Scanner) Keywords() []string {
|
||||
return []string{"screenshotapi"}
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Screenshot string `json:"screenshot"`
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify ScreenshotAPI secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
@@ -55,19 +50,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
if verify {
|
||||
timeout := 10 * time.Second
|
||||
client.Timeout = timeout
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://shot.screenshotapi.net/screenshot?token="+resMatch+"&url=https://google.com&width=1920&height=1080", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://shot.screenshotapi.net/screenshot?token="+resMatch+"&url=https://google.com&width=1920&height=1080&output=image", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
defer res.Body.Close()
|
||||
var r response
|
||||
if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
|
||||
s1.VerificationError = err
|
||||
continue
|
||||
}
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 && r.Screenshot != "" {
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
|
||||
@@ -47,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
}
|
||||
|
||||
if verify {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.supernotes.app/v1/user", nil)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.supernotes.app/v1/user/", nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
|
||||
accessToken = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b(?:oauth2:)?([0-9a-f]{40})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"twist"}) + `\b([0-9a-f:]{40,47})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -34,7 +34,7 @@ func (s Scanner) Keywords() []string {
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
|
||||
matches := accessToken.FindAllStringSubmatch(dataStr, -1)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package voiceflow
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
type Scanner struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// Ensure the Scanner satisfies the interface at compile time.
|
||||
var _ detectors.Detector = (*Scanner)(nil)
|
||||
|
||||
var (
|
||||
defaultClient = common.SaneHttpClient()
|
||||
// Reference: https://developer.voiceflow.com/reference/project#dialog-manager-api-keys
|
||||
//
|
||||
//TODO: This includes Workspace and Legacy Workspace API keys; I haven't validated whether these actually work.
|
||||
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
|
||||
keyPat = regexp.MustCompile(`\b(VF\.(?:(?:DM|WS)\.)?[a-fA-F0-9]{24}\.[a-zA-Z0-9]{16})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
return []string{"vf", "dm"}
|
||||
}
|
||||
|
||||
// FromData will find and optionally verify Voiceflow secrets in a given set of bytes.
|
||||
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
|
||||
dataStr := string(data)
|
||||
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
|
||||
|
||||
for _, match := range matches {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
resMatch := strings.TrimSpace(match[1])
|
||||
|
||||
s1 := detectors.Result{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Raw: []byte(resMatch),
|
||||
}
|
||||
|
||||
if verify {
|
||||
client := s.client
|
||||
if client == nil {
|
||||
client = defaultClient
|
||||
}
|
||||
// Fetch the state for a random user.
|
||||
payload := []byte(`{"question": "why is the sky blue?"}`)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", "https://general-runtime.voiceflow.com/knowledge-base/query", bytes.NewBuffer(payload))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Authorization", resMatch)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err == nil {
|
||||
if res.StatusCode == http.StatusOK {
|
||||
s1.Verified = true
|
||||
} else if res.StatusCode == http.StatusUnauthorized {
|
||||
// The secret is determinately not verified (nothing to do)
|
||||
} else {
|
||||
var buf bytes.Buffer
|
||||
var bodyString string
|
||||
_, err = io.Copy(&buf, res.Body)
|
||||
if err == nil {
|
||||
bodyString = buf.String()
|
||||
}
|
||||
s1.VerificationError = fmt.Errorf("unexpected HTTP response [status=%d, body=%s]", res.StatusCode, bodyString)
|
||||
}
|
||||
_ = res.Body.Close()
|
||||
} else {
|
||||
s1.VerificationError = err
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, s1)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (s Scanner) Type() detectorspb.DetectorType {
|
||||
return detectorspb.DetectorType_Voiceflow
|
||||
}
|
||||
@@ -1,302 +0,0 @@
|
||||
//go:build detectors
|
||||
// +build detectors
|
||||
|
||||
package voiceflow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestVoiceflow_Pattern(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
shouldMatch bool
|
||||
match string
|
||||
}{
|
||||
// True positives
|
||||
// https://github.com/funDAOmental/endlessquest/blob/5c008f7c6a7e58c45a88b72fef4b965c258d665c/Voiceflow/agent-api/index.js#L6
|
||||
{
|
||||
name: `valid_result1`,
|
||||
data: `// z0MG IT'S NOT A SECRET (but we'll delete it)
|
||||
const API_KEY = "VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.6469b4e5909a470007b96250.k4ip0SMy84jWlCsF`,
|
||||
},
|
||||
// https://github.com/sherifButt/ll-site/blob/b98b268214324da42a84e996e4c03c242e122680/src/components/Chatbot.jsx#L14
|
||||
{
|
||||
name: `valid_result2`,
|
||||
data: ` const runtime = useRuntime({
|
||||
verify: { authorization: 'VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f' },
|
||||
session: { userID: 'user_123' },
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.652da078cde70b0008e1c5df.zsIo23VTxNXKfb9f`,
|
||||
},
|
||||
// https://github.com/the-vv/Voiceflow-chatbot/blob/324db17693dd46387ea7a020e92c4e79b94306c6/src/app/chat/chat.component.ts#L27
|
||||
{
|
||||
name: `valid_result3`,
|
||||
data: ` this.http.delete('https://general-runtime.voiceflow.com/state/user/TEST_USER', {
|
||||
headers: {
|
||||
Authorization: "VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq"
|
||||
}
|
||||
}).subscribe(res => {
|
||||
this.loading = false;
|
||||
this.doPrompt('', { action: { type: 'launch' } });
|
||||
})`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.652ecc210267ec00078fc726.ZFPdEwvU0d1jiIMq`,
|
||||
},
|
||||
// https://github.com/legionX7/Graduation-Project-API/blob/451431771d3fba1d8c634b8855274b414d7aed6d/mainAPI.py#L547
|
||||
{
|
||||
name: `valid_result4`,
|
||||
data: `
|
||||
API_KEY = 'VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl'
|
||||
VERSION_ID = '646bc'`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.646388eb1419c80007bbbaa4.XHOqETFO3cvTxlGl`,
|
||||
},
|
||||
// https://github.com/voiceflow/general-runtime/blob/master/tests/runtime/lib/DataAPI/utils.unit.ts
|
||||
{
|
||||
name: `valid_result5`,
|
||||
data: ` it('extracts ID from a Dialog Manager API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('628d5d92faf688001bda7907');
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.DM.628d5d92faf688001bda7907.dmC8KKO1oX8JO5ai`,
|
||||
},
|
||||
{
|
||||
name: `valid_result6_legacy`,
|
||||
data: ` it('extracts ID from a Workspace API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('62bcb0cca5184300066f5ac7');
|
||||
});
|
||||
`,
|
||||
shouldMatch: true,
|
||||
match: `VF.WS.62bcb0cca5184300066f5ac7.egnKyyzZksiS5iGa`,
|
||||
},
|
||||
{
|
||||
name: `valid_result7_legacy`,
|
||||
data: ` it('extracts ID from a Legacy Workspace API key', () => {
|
||||
// eslint-disable-next-line no-secrets/no-secrets
|
||||
const key = 'VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az';
|
||||
const result = utils.extractAPIKeyID(key);
|
||||
|
||||
expect(result).to.equal('62bcb0cca5184300066f5ac7');
|
||||
});`,
|
||||
shouldMatch: true,
|
||||
match: `VF.62bcb0cca5184300066f5ac7.dmC8KKO1oX8JO5az`,
|
||||
},
|
||||
|
||||
// False positives
|
||||
// https://github.com/ImperialCollegeLondon/voiceflow-integration-whatsapp/blob/0f3d6a5638b9acb4989d5bf8e77081cc78e9b976/README.md?plain=1#L155
|
||||
{
|
||||
name: `invalid_result1`,
|
||||
data: "Now, paste it in your .env file for the **VF_PROJECT_API** variable<br>\n```VF_PROJECT_API='VF.DM.62xxxxxxxxxxxxxxxxxxxxxxx'```",
|
||||
shouldMatch: false,
|
||||
},
|
||||
// https://github.com/voiceflow/api-examples/blob/c3d8ba9ee8eced7ec8d241973b1eb0284aaec212/rust/src/main.rs#L5
|
||||
{
|
||||
name: `invalid_result2`,
|
||||
data: `const API_KEY: &str = "YOUR_API_KEY_HERE"; // it should look like this: VF.DM.XXXXXXX.XXXXXX... keep this a secret!`,
|
||||
shouldMatch: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
s := Scanner{}
|
||||
|
||||
results, err := s.FromData(context.Background(), true, []byte(test.data))
|
||||
if err != nil {
|
||||
t.Errorf("CoinbaseWaaS.FromData() error = %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if test.shouldMatch {
|
||||
if len(results) == 0 {
|
||||
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
|
||||
return
|
||||
}
|
||||
expected := test.data
|
||||
if test.match != "" {
|
||||
expected = test.match
|
||||
}
|
||||
result := results[0]
|
||||
if result.VerificationError != nil {
|
||||
fmt.Printf("VerificationError: %v\n", result.VerificationError)
|
||||
}
|
||||
resultData := string(result.Raw)
|
||||
if resultData != expected {
|
||||
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if len(results) > 0 {
|
||||
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVoiceflow_FromChunk(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
|
||||
if err != nil {
|
||||
t.Fatalf("could not get test secrets from GCP: %s", err)
|
||||
}
|
||||
secret := testSecrets.MustGetField("VOICEFLOW")
|
||||
inactiveSecret := testSecrets.MustGetField("VOICEFLOW_INACTIVE")
|
||||
|
||||
type args struct {
|
||||
ctx context.Context
|
||||
data []byte
|
||||
verify bool
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
s Scanner
|
||||
args args
|
||||
want []detectors.Result
|
||||
wantErr bool
|
||||
wantVerificationErr bool
|
||||
}{
|
||||
{
|
||||
name: "found, verified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: true,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, unverified",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "not found",
|
||||
s: Scanner{},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte("You cannot find the secret within"),
|
||||
verify: true,
|
||||
},
|
||||
want: nil,
|
||||
wantErr: false,
|
||||
wantVerificationErr: false,
|
||||
},
|
||||
{
|
||||
name: "found, would be verified if not for timeout",
|
||||
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
{
|
||||
name: "found, verified but unexpected api surface",
|
||||
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
|
||||
args: args{
|
||||
ctx: context.Background(),
|
||||
data: []byte(fmt.Sprintf("You can find a voiceflow secret %s within", secret)),
|
||||
verify: true,
|
||||
},
|
||||
want: []detectors.Result{
|
||||
{
|
||||
DetectorType: detectorspb.DetectorType_Voiceflow,
|
||||
Verified: false,
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
wantVerificationErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Voiceflow.FromData() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
for i := range got {
|
||||
if len(got[i].Raw) == 0 {
|
||||
t.Fatalf("no raw secret present: \n %+v", got[i])
|
||||
}
|
||||
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
|
||||
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
|
||||
}
|
||||
}
|
||||
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
|
||||
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
|
||||
t.Errorf("Voiceflow.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkFromData(benchmark *testing.B) {
|
||||
ctx := context.Background()
|
||||
s := Scanner{}
|
||||
for name, data := range detectors.MustGetBenchmarkData() {
|
||||
benchmark.Run(name, func(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
_, err := s.FromData(ctx, false, data)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
package engine
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
// detectorKey is used to identify a detector in the keywordsToDetectors map.
|
||||
// Multiple detectors can have the same detector type but different versions.
|
||||
// This allows us to identify a detector by its type and version.
|
||||
type detectorKey struct {
|
||||
detectorType detectorspb.DetectorType
|
||||
version int
|
||||
}
|
||||
|
||||
// DetectorInfo is used to store a detector and whether it should be verified.
|
||||
type DetectorInfo struct {
|
||||
detectors.Detector
|
||||
ShouldVerify bool
|
||||
}
|
||||
|
||||
// AhoCorasickCore encapsulates the operations and data structures used for keyword matching via the
|
||||
// Aho-Corasick algorithm. It is responsible for constructing and managing the trie for efficient
|
||||
// substring searches, as well as mapping keywords to their associated detectors for rapid lookups.
|
||||
type AhoCorasickCore struct {
|
||||
// prefilter is a ahocorasick struct used for doing efficient string
|
||||
// matching given a set of words. (keywords from the rules in the config)
|
||||
prefilter ahocorasick.Trie
|
||||
// Maps for efficient lookups during detection.
|
||||
detectorTypeToDetectorInfo map[detectorKey]DetectorInfo
|
||||
detectors map[bool][]detectors.Detector
|
||||
keywordsToDetectors map[string][]detectorKey
|
||||
}
|
||||
|
||||
// NewAhoCorasickCore allocates and initializes a new instance of AhoCorasickCore.
|
||||
// It creates an empty keyword-to-detectors map for future string matching operations.
|
||||
// The map detectorTypeToDetectorInfo is pre-allocated based on the size of detectors
|
||||
// provided, for efficient storage and lookup of detector information.
|
||||
func NewAhoCorasickCore(detectors map[bool][]detectors.Detector) *AhoCorasickCore {
|
||||
return &AhoCorasickCore{
|
||||
keywordsToDetectors: make(map[string][]detectorKey),
|
||||
detectors: detectors,
|
||||
detectorTypeToDetectorInfo: make(map[detectorKey]DetectorInfo, len(detectors[true])+len(detectors[false])),
|
||||
}
|
||||
}
|
||||
|
||||
// Setup initializes the internal state of AhoCorasickCore to prepare it for keyword matching.
|
||||
// This involves pre-filtering setup and lookup optimization, critical for the engine's performance.
|
||||
func (ac *AhoCorasickCore) Setup(ctx context.Context) {
|
||||
// Prepare maps for fast detector lookups, instead of scanning through an array of detectors for every chunk.
|
||||
var keywords []string
|
||||
for verify, detectorsSet := range ac.detectors {
|
||||
for _, d := range detectorsSet {
|
||||
key := createDetectorKey(d)
|
||||
ac.detectorTypeToDetectorInfo[key] = DetectorInfo{Detector: d, ShouldVerify: verify}
|
||||
keywords = ac.extractAndMapKeywords(d, key, keywords)
|
||||
}
|
||||
}
|
||||
|
||||
// Use the Ahocorasick algorithm to create a trie structure for efficient keyword matching.
|
||||
// This ensures that we can rapidly match against a vast set of keywords without individually comparing each one.
|
||||
ac.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
|
||||
ctx.Logger().V(4).Info("AhoCorasickCore Setup complete")
|
||||
}
|
||||
|
||||
// createDetectorKey creates a unique key for each detector. This key based on type and version,
|
||||
// it ensures faster lookups and reduces redundancy in our main detector store.
|
||||
func createDetectorKey(d detectors.Detector) detectorKey {
|
||||
detectorType := d.Type()
|
||||
var version int
|
||||
if v, ok := d.(detectors.Versioner); ok {
|
||||
version = v.Version()
|
||||
}
|
||||
return detectorKey{detectorType: detectorType, version: version}
|
||||
}
|
||||
|
||||
// extractAndMapKeywords captures keywords associated with each detector and maps them.
|
||||
// This allows us to quickly determine which detectors are relevant based on the presence of certain keywords.
|
||||
func (ac *AhoCorasickCore) extractAndMapKeywords(d detectors.Detector, key detectorKey, keywords []string) []string {
|
||||
for _, kw := range d.Keywords() {
|
||||
kwLower := strings.ToLower(kw)
|
||||
keywords = append(keywords, kwLower)
|
||||
ac.keywordsToDetectors[kwLower] = append(ac.keywordsToDetectors[kwLower], key)
|
||||
}
|
||||
return keywords
|
||||
}
|
||||
|
||||
// MatchString performs a string match using the Aho-Corasick algorithm, returning an array of matches.
|
||||
// Designed for internal use within the AhoCorasickCore component.
|
||||
func (ac *AhoCorasickCore) MatchString(input string) []*ahocorasick.Match {
|
||||
return ac.prefilter.MatchString(strings.ToLower(input))
|
||||
}
|
||||
|
||||
// PopulateDetectorsByMatch populates the given detectorMap based on the Aho-Corasick match results.
|
||||
// This method is designed to reuse the same map for performance optimization,
|
||||
// reducing the need for repeated allocations within each detector worker in the engine.
|
||||
func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]DetectorInfo) bool {
|
||||
matchedKeys, ok := ac.keywordsToDetectors[match.MatchString()]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
for _, key := range matchedKeys {
|
||||
detectors[key.detectorType] = ac.detectorTypeToDetectorInfo[key]
|
||||
}
|
||||
return true
|
||||
}
|
||||
+10
-10
@@ -4,8 +4,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/anthropic"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/envoyapikey"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/huggingface"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ip2location"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/openvpn"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ramp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/salesforce"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/snowflake"
|
||||
@@ -13,7 +11,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/sourcegraphcody"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/tailscale"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/trufflehogenterprise"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/voiceflow"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
|
||||
@@ -76,6 +73,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/azure"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bannerbear"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baremetrics"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/baseapiio"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beamer"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/beebole"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/besnappy"
|
||||
@@ -86,6 +84,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitfinex"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitlyaccesstoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/bitmex"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blablabus"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blazemeter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blitapp"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/blocknative"
|
||||
@@ -192,6 +191,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/databox"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/databrickstoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datadogtoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datafire"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/datagov"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/deepai"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/deepgram"
|
||||
@@ -304,6 +304,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitlabv2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gitter"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/glassnode"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/glitterlyapi"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gocanvas"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/gocardless"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/goodday"
|
||||
@@ -386,7 +387,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/livestorm"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loadmill"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/locationiq"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loggly"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loginradius"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/lokalisetoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/loyverse"
|
||||
@@ -532,6 +532,7 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qase"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qualaroo"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/qubole"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/quickmetrics"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rapidapi"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rawg"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/razorpay"
|
||||
@@ -712,7 +713,6 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/urlscan"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userflow"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/userstack"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vagrantcloudpersonaltoken"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vatlayer"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vbout"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/vercel"
|
||||
@@ -1061,6 +1061,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
deepgram.Scanner{},
|
||||
brandfetch.Scanner{},
|
||||
typeform.Scanner{},
|
||||
blablabus.Scanner{},
|
||||
fxmarket.Scanner{},
|
||||
ipapi.Scanner{},
|
||||
clearbit.Scanner{},
|
||||
@@ -1186,6 +1187,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
restpackscreenshotapi.Scanner{},
|
||||
shortcut.Scanner{},
|
||||
// nasdaqdatalink.Scanner{},
|
||||
quickmetrics.Scanner{},
|
||||
neutrinoapi.Scanner{},
|
||||
bitmex.Scanner{},
|
||||
deepai.Scanner{},
|
||||
@@ -1211,6 +1213,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
scraperbox.Scanner{},
|
||||
passbase.Scanner{},
|
||||
ticketmaster.Scanner{},
|
||||
datafire.Scanner{},
|
||||
iexcloud.Scanner{},
|
||||
partnerstack.Scanner{},
|
||||
qubole.Scanner{},
|
||||
@@ -1244,6 +1247,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
scrapeowl.Scanner{},
|
||||
rebrandly.Scanner{},
|
||||
dandelion.Scanner{},
|
||||
baseapiio.Scanner{},
|
||||
purestake.Scanner{},
|
||||
carboninterface.Scanner{},
|
||||
signaturit.Scanner{},
|
||||
@@ -1398,6 +1402,7 @@ func DefaultDetectors() []detectors.Detector {
|
||||
convier.Scanner{},
|
||||
loadmill.Scanner{},
|
||||
magicbell.Scanner{},
|
||||
glitterlyapi.Scanner{},
|
||||
apitemplate.Scanner{},
|
||||
knapsackpro.Scanner{},
|
||||
twitter.Scanner{},
|
||||
@@ -1545,15 +1550,10 @@ func DefaultDetectors() []detectors.Detector {
|
||||
salesforce.Scanner{},
|
||||
sourcegraph.Scanner{},
|
||||
tailscale.Scanner{},
|
||||
loggly.Scanner{},
|
||||
web3storage.Scanner{},
|
||||
&ramp.Scanner{},
|
||||
&anthropic.Scanner{},
|
||||
&sourcegraphcody.Scanner{},
|
||||
voiceflow.Scanner{},
|
||||
ip2location.Scanner{},
|
||||
vagrantcloudpersonaltoken.Scanner{},
|
||||
openvpn.Scanner{},
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+111
-99
@@ -3,11 +3,14 @@ package engine
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
ahocorasick "github.com/BobuSumisu/aho-corasick"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
@@ -63,8 +66,9 @@ type Engine struct {
|
||||
onlyVerified bool
|
||||
printAvgDetectorTime bool
|
||||
|
||||
// ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups.
|
||||
ahoCorasickCore *AhoCorasickCore
|
||||
// prefilter is a ahocorasick struct used for doing efficient string
|
||||
// matching given a set of words (keywords from the rules in the config)
|
||||
prefilter ahocorasick.Trie
|
||||
|
||||
// Engine synchronization primitives.
|
||||
sourceManager *sources.SourceManager
|
||||
@@ -89,10 +93,9 @@ type Engine struct {
|
||||
dedupeCache *lru.Cache
|
||||
}
|
||||
|
||||
// Option is used to configure the engine during initialization using functional options.
|
||||
type Option func(*Engine)
|
||||
type EngineOption func(*Engine)
|
||||
|
||||
func WithConcurrency(concurrency uint8) Option {
|
||||
func WithConcurrency(concurrency uint8) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.concurrency = concurrency
|
||||
}
|
||||
@@ -100,7 +103,7 @@ func WithConcurrency(concurrency uint8) Option {
|
||||
|
||||
const ignoreTag = "trufflehog:ignore"
|
||||
|
||||
func WithDetectors(verify bool, d ...detectors.Detector) Option {
|
||||
func WithDetectors(verify bool, d ...detectors.Detector) EngineOption {
|
||||
return func(e *Engine) {
|
||||
if e.detectors == nil {
|
||||
e.detectors = make(map[bool][]detectors.Detector)
|
||||
@@ -113,7 +116,7 @@ func WithDetectors(verify bool, d ...detectors.Detector) Option {
|
||||
}
|
||||
}
|
||||
|
||||
func WithDecoders(decoders ...decoders.Decoder) Option {
|
||||
func WithDecoders(decoders ...decoders.Decoder) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.decoders = decoders
|
||||
}
|
||||
@@ -121,14 +124,14 @@ func WithDecoders(decoders ...decoders.Decoder) Option {
|
||||
|
||||
// WithFilterUnverified sets the filterUnverified flag on the engine. If set to
|
||||
// true, the engine will only return the first unverified result for a chunk for a detector.
|
||||
func WithFilterUnverified(filter bool) Option {
|
||||
func WithFilterUnverified(filter bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.filterUnverified = filter
|
||||
}
|
||||
}
|
||||
|
||||
// WithFilterEntropy filters out unverified results using Shannon entropy.
|
||||
func WithFilterEntropy(entropy float64) Option {
|
||||
func WithFilterEntropy(entropy float64) EngineOption {
|
||||
return func(e *Engine) {
|
||||
if entropy > 0 {
|
||||
e.filterEntropy = &entropy
|
||||
@@ -138,7 +141,7 @@ func WithFilterEntropy(entropy float64) Option {
|
||||
|
||||
// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
|
||||
// the engine will only print verified results.
|
||||
func WithOnlyVerified(onlyVerified bool) Option {
|
||||
func WithOnlyVerified(onlyVerified bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.onlyVerified = onlyVerified
|
||||
}
|
||||
@@ -150,7 +153,7 @@ func WithOnlyVerified(onlyVerified bool) Option {
|
||||
// the engine is configured to print the results.
|
||||
// Calculating the average time taken by each detector is an expensive operation
|
||||
// and should be avoided unless specified by the user.
|
||||
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
|
||||
func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printAvgDetectorTime = printAvgDetectorTime
|
||||
}
|
||||
@@ -160,7 +163,7 @@ func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
|
||||
// the filterFunc returns true, the detector will be included for scanning.
|
||||
// This option applies to the existing list of detectors configured, so the
|
||||
// order this option appears matters. All filtering happens before scanning.
|
||||
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
|
||||
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
// If no detectors are configured, do nothing.
|
||||
if e.detectors == nil {
|
||||
@@ -172,7 +175,7 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
|
||||
}
|
||||
|
||||
// WithPrinter sets the Printer on the engine.
|
||||
func WithPrinter(printer Printer) Option {
|
||||
func WithPrinter(printer Printer) EngineOption {
|
||||
return func(e *Engine) {
|
||||
e.printer = printer
|
||||
}
|
||||
@@ -266,60 +269,32 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
|
||||
return avgTime
|
||||
}
|
||||
|
||||
// Start initializes and activates the engine's processing pipeline.
|
||||
// It sets up various default configurations, prepares lookup structures for
|
||||
// detectors, conducts basic sanity checks, and kickstarts all necessary workers.
|
||||
// Once started, the engine begins processing input data to identify secrets.
|
||||
func Start(ctx context.Context, options ...Option) (*Engine, error) {
|
||||
e := &Engine{}
|
||||
|
||||
if err := e.initialize(ctx, options...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.setDefaults(ctx)
|
||||
ctx.Logger().V(4).Info("setting up aho-corasick core")
|
||||
e.ahoCorasickCore.Setup(ctx)
|
||||
e.sanityChecks(ctx)
|
||||
e.startWorkers(ctx)
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
const defaultChannelBuffer = 1
|
||||
|
||||
// initialize prepares the engine's internal structures. The LRU cache optimizes
|
||||
// deduplication efforts, allowing the engine to quickly check if a chunk has
|
||||
// been processed before, thereby saving computational overhead.
|
||||
func (e *Engine) initialize(ctx context.Context, options ...Option) error {
|
||||
// TODO (ahrav): Determine the optimal cache size.
|
||||
const cacheSize = 512 // number of entries in the LRU cache
|
||||
// Start the engine with options.
|
||||
func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
|
||||
const (
|
||||
defaultChannelBuffer = 1
|
||||
// TODO (ahrav): Determine the optimal cache size.
|
||||
cacheSize = 512 // number of entries in the LRU cache
|
||||
)
|
||||
|
||||
cache, err := lru.New(cacheSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize LRU cache: %w", err)
|
||||
return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
|
||||
}
|
||||
|
||||
// Channels are used for communication between different parts of the engine,
|
||||
// ensuring that data flows smoothly without race conditions.
|
||||
e.detectableChunksChan = make(chan detectableChunk, defaultChannelBuffer)
|
||||
e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer)
|
||||
e.dedupeCache = cache
|
||||
e.printer = new(output.PlainPrinter)
|
||||
e.metrics = runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}}
|
||||
e := &Engine{
|
||||
detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
|
||||
results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
|
||||
dedupeCache: cache,
|
||||
printer: new(output.PlainPrinter), // default printer
|
||||
metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
option(e)
|
||||
}
|
||||
ctx.Logger().V(4).Info("engine initialized")
|
||||
e.ahoCorasickCore = NewAhoCorasickCore(e.detectors)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// setDefaults ensures that if specific engine properties aren't provided,
|
||||
// they're set to reasonable default values. It makes the engine robust to
|
||||
// incomplete configuration.
|
||||
func (e *Engine) setDefaults(ctx context.Context) {
|
||||
// Set defaults.
|
||||
if e.concurrency == 0 {
|
||||
numCPU := runtime.NumCPU()
|
||||
ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
|
||||
@@ -327,13 +302,12 @@ func (e *Engine) setDefaults(ctx context.Context) {
|
||||
}
|
||||
ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
|
||||
|
||||
// Create SourceManager.
|
||||
e.sourceManager = sources.NewManager(
|
||||
sources.WithConcurrentSources(int(e.concurrency)),
|
||||
sources.WithConcurrentUnits(int(e.concurrency)),
|
||||
sources.WithBufferedOutput(defaultChannelBuffer),
|
||||
)
|
||||
|
||||
// Default decoders handle common encoding formats.
|
||||
if len(e.decoders) == 0 {
|
||||
e.decoders = decoders.DefaultDecoders()
|
||||
}
|
||||
@@ -343,30 +317,46 @@ func (e *Engine) setDefaults(ctx context.Context) {
|
||||
e.detectors[true] = DefaultDetectors()
|
||||
e.detectors[false] = []detectors.Detector{}
|
||||
}
|
||||
ctx.Logger().V(4).Info("default engine options set")
|
||||
}
|
||||
|
||||
// Sanity check detectors for duplicate configuration. Only log in case
|
||||
// a detector has been configured in a way that isn't represented by
|
||||
// the DetectorID (type and version).
|
||||
func (e *Engine) sanityChecks(ctx context.Context) {
|
||||
dets := append(e.detectors[true], e.detectors[false]...)
|
||||
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
|
||||
for _, det := range dets {
|
||||
id := config.GetDetectorID(det)
|
||||
if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
|
||||
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
|
||||
// build ahocorasick prefilter for efficient string matching
|
||||
// on keywords
|
||||
keywords := []string{}
|
||||
for _, d := range e.detectors[false] {
|
||||
for _, kw := range d.Keywords() {
|
||||
keywords = append(keywords, strings.ToLower(kw))
|
||||
}
|
||||
}
|
||||
for _, d := range e.detectors[true] {
|
||||
for _, kw := range d.Keywords() {
|
||||
keywords = append(keywords, strings.ToLower(kw))
|
||||
}
|
||||
}
|
||||
e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
|
||||
|
||||
ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
|
||||
ctx.Logger().V(3).Info("loaded detectors",
|
||||
"total", len(e.detectors[true])+len(e.detectors[false]),
|
||||
"verification_enabled", len(e.detectors[true]),
|
||||
"verification_disabled", len(e.detectors[false]),
|
||||
)
|
||||
|
||||
// Sanity check detectors for duplicate configuration. Only log in case
|
||||
// a detector has been configured in a way that isn't represented by
|
||||
// the DetectorID (type and version).
|
||||
{
|
||||
dets := append(e.detectors[true], e.detectors[false]...)
|
||||
seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
|
||||
for _, det := range dets {
|
||||
id := config.GetDetectorID(det)
|
||||
if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
|
||||
ctx.Logger().Info("possible duplicate detector configured", "detector", id)
|
||||
}
|
||||
seenDetectors[id] = struct{}{}
|
||||
}
|
||||
seenDetectors[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// startWorkers initiates all necessary workers. Workers handle processing of
|
||||
// chunks concurrently. Separating the initialization of different types of
|
||||
// workers helps in scalability and makes it easier to diagnose issues.
|
||||
func (e *Engine) startWorkers(ctx context.Context) {
|
||||
// Scanner workers process input data and extract chunks for detectors.
|
||||
ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
|
||||
// Run the Secret scanner workers and Notifier pipelines.
|
||||
for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
|
||||
e.workersWg.Add(1)
|
||||
go func() {
|
||||
@@ -377,7 +367,6 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Detector workers apply keyword matching, regexes and API calls to detect secrets in chunks.
|
||||
const detectorWorkerMultiplier = 50
|
||||
ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
|
||||
for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
|
||||
@@ -390,7 +379,6 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Notifier workers communicate detected issues to the user or any downstream systems.
|
||||
// We want 1/4th of the notifier workers as the number of scanner workers.
|
||||
const notifierWorkerRatio = 4
|
||||
maxNotifierWorkers := 1
|
||||
@@ -407,6 +395,8 @@ func (e *Engine) startWorkers(ctx context.Context) {
|
||||
e.notifyResults(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// Finish waits for running sources to complete and workers to finish scanning
|
||||
@@ -455,42 +445,64 @@ type detectableChunk struct {
|
||||
func (e *Engine) detectorWorker(ctx context.Context) {
|
||||
var wgDetect sync.WaitGroup
|
||||
|
||||
// Reuse the same map to avoid allocations.
|
||||
const avgDetectorsPerChunk = 2
|
||||
chunkSpecificDetectors := make(map[detectorspb.DetectorType]DetectorInfo, avgDetectorsPerChunk)
|
||||
for originalChunk := range e.ChunksChan() {
|
||||
for chunk := range sources.Chunker(originalChunk) {
|
||||
matchedKeywords := make(map[string]struct{})
|
||||
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
|
||||
for _, decoder := range e.decoders {
|
||||
var decoderType detectorspb.DecoderType
|
||||
switch decoder.(type) {
|
||||
case *decoders.UTF8:
|
||||
decoderType = detectorspb.DecoderType_PLAIN
|
||||
case *decoders.Base64:
|
||||
decoderType = detectorspb.DecoderType_BASE64
|
||||
case *decoders.UTF16:
|
||||
decoderType = detectorspb.DecoderType_UTF16
|
||||
default:
|
||||
ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
|
||||
decoderType = detectorspb.DecoderType_UNKNOWN
|
||||
}
|
||||
|
||||
decoded := decoder.FromChunk(chunk)
|
||||
|
||||
if decoded == nil {
|
||||
ctx.Logger().V(4).Info("no decoder found for chunk", "chunk", chunk)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, match := range e.ahoCorasickCore.MatchString(string(decoded.Chunk.Data)) {
|
||||
if !e.ahoCorasickCore.PopulateDetectorsByMatch(match, chunkSpecificDetectors) {
|
||||
continue
|
||||
}
|
||||
// build a map of all keywords that were matched in the chunk
|
||||
for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
|
||||
matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
|
||||
}
|
||||
|
||||
for k, detector := range chunkSpecificDetectors {
|
||||
decoded.Chunk.Verify = detector.ShouldVerify
|
||||
wgDetect.Add(1)
|
||||
e.detectableChunksChan <- detectableChunk{
|
||||
chunk: *decoded.Chunk,
|
||||
detector: detector,
|
||||
decoder: decoded.DecoderType,
|
||||
wgDoneFn: wgDetect.Done,
|
||||
for verify, detectorsSet := range e.detectors {
|
||||
for _, detector := range detectorsSet {
|
||||
chunkContainsKeyword := false
|
||||
for _, kw := range detector.Keywords() {
|
||||
if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
|
||||
chunkContainsKeyword = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !chunkContainsKeyword {
|
||||
continue
|
||||
}
|
||||
|
||||
decoded.Verify = verify
|
||||
wgDetect.Add(1)
|
||||
e.detectableChunksChan <- detectableChunk{
|
||||
chunk: *decoded,
|
||||
detector: detector,
|
||||
decoder: decoderType,
|
||||
wgDoneFn: wgDetect.Done,
|
||||
}
|
||||
}
|
||||
delete(chunkSpecificDetectors, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
atomic.AddUint64(&e.metrics.ChunksScanned, 1)
|
||||
}
|
||||
wgDetect.Wait()
|
||||
ctx.Logger().V(4).Info("finished scanning chunks")
|
||||
}
|
||||
|
||||
func (e *Engine) detectChunks(ctx context.Context) {
|
||||
@@ -623,7 +635,7 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
|
||||
if !found {
|
||||
return 0, false
|
||||
}
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")))
|
||||
lineNumber := int64(bytes.Count(before, []byte("\n")) + 1)
|
||||
// If the line contains the ignore tag, we should ignore the result.
|
||||
endLine := bytes.Index(after, []byte("\n"))
|
||||
if endLine == -1 {
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 2,
|
||||
expectedLine: 3,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
@@ -43,7 +43,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 2,
|
||||
expectedLine: 3,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -54,7 +54,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
expectedLine: 5,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -65,7 +65,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 4,
|
||||
expectedLine: 5,
|
||||
ignore: false,
|
||||
},
|
||||
{
|
||||
@@ -76,7 +76,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret\nhere"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
expectedLine: 4,
|
||||
ignore: true,
|
||||
},
|
||||
{
|
||||
@@ -87,7 +87,7 @@ func TestFragmentLineOffset(t *testing.T) {
|
||||
result: &detectors.Result{
|
||||
Raw: []byte("secret here"),
|
||||
},
|
||||
expectedLine: 3,
|
||||
expectedLine: 4,
|
||||
ignore: true,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package engine
|
||||
|
||||
import (
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -121,10 +120,9 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
defer cancel()
|
||||
|
||||
e, err := Start(ctx,
|
||||
WithConcurrency(uint8(runtime.NumCPU())),
|
||||
WithConcurrency(1),
|
||||
WithDecoders(decoders.DefaultDecoders()...),
|
||||
WithDetectors(false, DefaultDetectors()...),
|
||||
WithPrinter(new(discardPrinter)),
|
||||
)
|
||||
assert.Nil(b, err)
|
||||
|
||||
@@ -135,7 +133,6 @@ func BenchmarkGitEngine(b *testing.B) {
|
||||
}
|
||||
}()
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// TODO: this is measuring the time it takes to initialize the source
|
||||
// and not to do the full scan
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
|
||||
logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
|
||||
+1010
-1056
File diff suppressed because it is too large
Load Diff
+482
-501
File diff suppressed because it is too large
Load Diff
@@ -121,8 +121,6 @@ func (m *LocalSource) validate(all bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
// no validation rules for ScanPeriod
|
||||
|
||||
if len(errors) > 0 {
|
||||
return LocalSourceMultiError(errors)
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
"testing/iotest"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-logr/logr"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-logr/logr"
|
||||
"golang.org/x/oauth2"
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
|
||||
@@ -222,10 +222,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
|
||||
}
|
||||
s.conn = &conn
|
||||
|
||||
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(),
|
||||
append(s.conn.GetRepositories(), s.conn.GetIncludeRepos()...),
|
||||
s.conn.GetIgnoreRepos(),
|
||||
)
|
||||
s.filteredRepoCache = s.newFilteredRepoCache(memory.New(), s.conn.GetRepositories(), s.conn.GetIgnoreRepos())
|
||||
s.memberCache = make(map[string]struct{})
|
||||
|
||||
s.repoSizes = newRepoSize()
|
||||
@@ -511,12 +508,12 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin
|
||||
|
||||
for _, org := range s.orgsCache.Keys() {
|
||||
if err := s.getReposByOrg(ctx, org); err != nil {
|
||||
s.log.Error(err, "error fetching repos for org")
|
||||
s.log.Error(err, "error fetching repos for org or user")
|
||||
}
|
||||
|
||||
// We probably don't need to do this, since getting repos by org makes more sense?
|
||||
if err := s.getReposByUser(ctx, org); err != nil {
|
||||
s.log.Error(err, "error fetching repos for user")
|
||||
s.log.Error(err, "error fetching repos for org or user")
|
||||
}
|
||||
|
||||
if s.conn.ScanUsers {
|
||||
@@ -828,51 +825,38 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
|
||||
// Unauthenticated access to most github endpoints has a rate limit of 60 requests per hour.
|
||||
// This will likely only be exhausted if many users/orgs are scanned without auth
|
||||
func (s *Source) handleRateLimit(errIn error, res *github.Response) bool {
|
||||
var (
|
||||
knownWait = true
|
||||
remaining = 0
|
||||
retryAfter time.Duration
|
||||
)
|
||||
|
||||
// GitHub has both primary (RateLimit) and secondary (AbuseRateLimit) errors.
|
||||
var rateLimit *github.RateLimitError
|
||||
var abuseLimit *github.AbuseRateLimitError
|
||||
if errors.As(errIn, &rateLimit) {
|
||||
// Do nothing
|
||||
} else if errors.As(errIn, &abuseLimit) {
|
||||
retryAfter = abuseLimit.GetRetryAfter()
|
||||
} else {
|
||||
limit, ok := errIn.(*github.RateLimitError)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
githubNumRateLimitEncountered.WithLabelValues(s.name).Inc()
|
||||
// Parse retry information from response headers, unless a Retry-After value was already provided.
|
||||
// https://docs.github.com/en/rest/overview/resources-in-the-rest-api#exceeding-the-rate-limit
|
||||
if retryAfter <= 0 && res != nil {
|
||||
var err error
|
||||
remaining, err = strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
|
||||
|
||||
if res != nil {
|
||||
knownWait := true
|
||||
remaining, err := strconv.Atoi(res.Header.Get("x-ratelimit-remaining"))
|
||||
if err != nil {
|
||||
knownWait = false
|
||||
}
|
||||
|
||||
resetTime, err := strconv.Atoi(res.Header.Get("x-ratelimit-reset"))
|
||||
if err != nil || resetTime == 0 {
|
||||
knownWait = false
|
||||
} else if resetTime > 0 {
|
||||
retryAfter = time.Duration(int64(resetTime)-time.Now().Unix()) * time.Second
|
||||
}
|
||||
|
||||
if knownWait && remaining == 0 {
|
||||
waitTime := int64(resetTime) - time.Now().Unix()
|
||||
if waitTime > 0 {
|
||||
duration := time.Duration(waitTime+1) * time.Second
|
||||
s.log.V(2).Info("rate limited", "resumeTime", time.Now().Add(duration).String())
|
||||
time.Sleep(duration)
|
||||
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(duration.Seconds())
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resumeTime := time.Now().Add(retryAfter).String()
|
||||
if knownWait && remaining == 0 && retryAfter > 0 {
|
||||
s.log.V(2).Info("rate limited", "retry_after", retryAfter.String(), "resume_time", resumeTime)
|
||||
} else {
|
||||
// TODO: Use exponential backoff instead of static retry time.
|
||||
retryAfter = time.Minute * 5
|
||||
s.log.V(2).Error(errIn, "unexpected rate limit error", "retry_after", retryAfter.String(), "resume_time", resumeTime)
|
||||
}
|
||||
time.Sleep(retryAfter)
|
||||
githubSecondsSpentRateLimited.WithLabelValues(s.name).Add(retryAfter.Seconds())
|
||||
s.log.V(2).Info("handling rate limit (5 minutes retry)", "retry-after", limit.Message)
|
||||
time.Sleep(time.Minute * 5)
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -1092,7 +1076,7 @@ func (s *Source) scanComments(ctx context.Context, repoPath string, chunksChan c
|
||||
}
|
||||
|
||||
func (s *Source) processGistComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
|
||||
ctx.Logger().V(2).Info("scanning github gist comments", "repository", repoPath)
|
||||
s.log.Info("scanning github gist comments", "repository", repoPath)
|
||||
// GitHub Gist URL.
|
||||
gistID, err := extractGistID(trimmedURL)
|
||||
if err != nil {
|
||||
@@ -1147,10 +1131,9 @@ var (
|
||||
)
|
||||
|
||||
type repoInfo struct {
|
||||
owner string
|
||||
repo string
|
||||
repoPath string
|
||||
visibility source_metadatapb.Visibility
|
||||
owner string
|
||||
repo string
|
||||
repoPath string
|
||||
}
|
||||
|
||||
func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimmedURL []string, repoURL *url.URL, chunksChan chan *sources.Chunk) error {
|
||||
@@ -1161,33 +1144,22 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
|
||||
owner := trimmedURL[1]
|
||||
repo := trimmedURL[2]
|
||||
|
||||
if !(s.includeIssueComments || s.includePRComments) {
|
||||
return nil
|
||||
}
|
||||
|
||||
repoInfo := repoInfo{
|
||||
owner: owner,
|
||||
repo: repo,
|
||||
repoPath: repoPath,
|
||||
visibility: s.visibilityOf(ctx, repoPath),
|
||||
}
|
||||
repoInfo := repoInfo{owner: owner, repo: repo, repoPath: repoPath}
|
||||
|
||||
if s.includeIssueComments {
|
||||
ctx.Logger().V(2).Info("scanning github issues", "repository", repoInfo.repoPath)
|
||||
if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
|
||||
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.processIssueComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
if err := s.processIssues(ctx, repoInfo, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if s.includePRComments {
|
||||
ctx.Logger().V(2).Info("scanning github pull requests", "repository", repoInfo.repoPath)
|
||||
if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
|
||||
if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.processPRComments(ctx, repoInfo, chunksChan); err != nil {
|
||||
if err := s.processPRs(ctx, repoInfo, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1197,6 +1169,8 @@ func (s *Source) processRepoComments(ctx context.Context, repoPath string, trimm
|
||||
}
|
||||
|
||||
func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning github issue descriptions", "repository", info.repoPath)
|
||||
|
||||
bodyTextsOpts := &github.IssueListByRepoOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
@@ -1217,7 +1191,7 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkIssues(ctx, info, issues, chunksChan); err != nil {
|
||||
if err = s.chunkIssues(ctx, info.repo, info.repoPath, issues, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1231,6 +1205,8 @@ func (s *Source) processIssues(ctx context.Context, info repoInfo, chunksChan ch
|
||||
}
|
||||
|
||||
func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning github issue comments", "repository", info.repoPath)
|
||||
|
||||
issueOpts := &github.IssueListCommentsOptions{
|
||||
Sort: &sortType,
|
||||
Direction: &directionType,
|
||||
@@ -1250,7 +1226,7 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkIssueComments(ctx, info, issueComments, chunksChan); err != nil {
|
||||
if err = s.chunkIssueComments(ctx, info.repo, info.repoPath, issueComments, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1264,6 +1240,8 @@ func (s *Source) processIssueComments(ctx context.Context, info repoInfo, chunks
|
||||
}
|
||||
|
||||
func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning github pull request descriptions", "repository", info.repoPath)
|
||||
|
||||
prOpts := &github.PullRequestListOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
@@ -1284,7 +1262,7 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkPullRequests(ctx, info, prs, chunksChan); err != nil {
|
||||
if err = s.chunkPullRequests(ctx, info.repo, prs, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1298,6 +1276,8 @@ func (s *Source) processPRs(ctx context.Context, info repoInfo, chunksChan chan
|
||||
}
|
||||
|
||||
func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksChan chan *sources.Chunk) error {
|
||||
s.log.Info("scanning github pull request comments", "repository", info.repoPath)
|
||||
|
||||
prOpts := &github.PullRequestListCommentsOptions{
|
||||
Sort: sortType,
|
||||
Direction: directionType,
|
||||
@@ -1317,7 +1297,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.chunkPullRequestComments(ctx, info, prComments, chunksChan); err != nil {
|
||||
if err = s.chunkPullRequestComments(ctx, info.repo, prComments, chunksChan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1330,7 +1310,7 @@ func (s *Source) processPRComments(ctx context.Context, info repoInfo, chunksCha
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkIssues(ctx context.Context, repo, repoPath string, issues []*github.Issue, chunksChan chan *sources.Chunk) error {
|
||||
for _, issue := range issues {
|
||||
|
||||
// Skip pull requests since covered by processPRs.
|
||||
@@ -1350,9 +1330,9 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
|
||||
Link: sanitizer.UTF8(issue.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(issue.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(issue.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Timestamp: sanitizer.UTF8(issue.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
Visibility: s.visibilityOf(ctx, repoPath),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1369,7 +1349,7 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkIssueComments(ctx context.Context, repo, repoPath string, comments []*github.IssueComment, chunksChan chan *sources.Chunk) error {
|
||||
for _, comment := range comments {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1383,9 +1363,9 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
|
||||
Link: sanitizer.UTF8(comment.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
Visibility: s.visibilityOf(ctx, repoPath),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1402,7 +1382,7 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkPullRequestComments(ctx context.Context, repo string, comments []*github.PullRequestComment, chunksChan chan *sources.Chunk) error {
|
||||
for _, comment := range comments {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1416,9 +1396,8 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
|
||||
Link: sanitizer.UTF8(comment.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(comment.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1435,7 +1414,7 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
|
||||
func (s *Source) chunkPullRequests(ctx context.Context, repo string, prs []*github.PullRequest, chunksChan chan *sources.Chunk) error {
|
||||
for _, pr := range prs {
|
||||
// Create chunk and send it to the channel.
|
||||
chunk := &sources.Chunk{
|
||||
@@ -1449,9 +1428,8 @@ func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs [
|
||||
Link: sanitizer.UTF8(pr.GetHTMLURL()),
|
||||
Username: sanitizer.UTF8(pr.GetUser().GetLogin()),
|
||||
Email: sanitizer.UTF8(pr.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(repoInfo.repo),
|
||||
Repository: sanitizer.UTF8(repo),
|
||||
Timestamp: sanitizer.UTF8(pr.GetCreatedAt().String()),
|
||||
Visibility: repoInfo.visibility,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1484,7 +1462,8 @@ func (s *Source) chunkGistComments(ctx context.Context, gistUrl string, comments
|
||||
Email: sanitizer.UTF8(comment.GetUser().GetEmail()),
|
||||
Repository: sanitizer.UTF8(gistUrl),
|
||||
Timestamp: sanitizer.UTF8(comment.GetCreatedAt().String()),
|
||||
// TODO: Fetching this requires making an additional API call. We may want to include this in the future.
|
||||
// Fetching this information requires making an additional API call.
|
||||
// We may want to include this in the future.
|
||||
// Visibility: s.visibilityOf(ctx, repoPath),
|
||||
},
|
||||
},
|
||||
|
||||
@@ -89,14 +89,14 @@ func (f Fatal) Unwrap() error { return f.error }
|
||||
// ChunkError is a custom error type for errors encountered during chunking of
|
||||
// a specific unit.
|
||||
type ChunkError struct {
|
||||
Unit SourceUnit
|
||||
Err error
|
||||
unit SourceUnit
|
||||
err error
|
||||
}
|
||||
|
||||
func (f ChunkError) Error() string {
|
||||
return fmt.Sprintf("error chunking unit %q: %s", f.Unit.SourceUnitID(), f.Err.Error())
|
||||
return fmt.Sprintf("error chunking unit %q: %s", f.unit.SourceUnitID(), f.err.Error())
|
||||
}
|
||||
func (f ChunkError) Unwrap() error { return f.Err }
|
||||
func (f ChunkError) Unwrap() error { return f.err }
|
||||
|
||||
// JobProgress aggregates information about a run of a Source.
|
||||
type JobProgress struct {
|
||||
@@ -351,17 +351,3 @@ func (m JobProgressMetrics) ElapsedTime() time.Duration {
|
||||
}
|
||||
return m.EndTime.Sub(m.StartTime)
|
||||
}
|
||||
|
||||
// ErrorsFor returns all the errors for the given SourceUnit. If there are no
|
||||
// errors, including the case that the unit has not been encountered, nil will
|
||||
// be returned.
|
||||
func (m JobProgressMetrics) ErrorsFor(unit SourceUnit) []error {
|
||||
var errs []error
|
||||
for _, err := range m.Errors {
|
||||
var chunkErr ChunkError
|
||||
if errors.As(err, &chunkErr) && chunkErr.Unit == unit {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
)
|
||||
|
||||
// UnitHook implements JobProgressHook for tracking the progress of each
|
||||
// individual unit.
|
||||
type UnitHook struct {
|
||||
metrics *lru.Cache[string, *UnitMetrics]
|
||||
mu sync.Mutex
|
||||
NoopHook
|
||||
}
|
||||
|
||||
type UnitHookOpt func(*UnitHook)
|
||||
|
||||
func WithUnitHookCache(cache *lru.Cache[string, *UnitMetrics]) UnitHookOpt {
|
||||
return func(hook *UnitHook) { hook.metrics = cache }
|
||||
}
|
||||
|
||||
func NewUnitHook(ctx context.Context, opts ...UnitHookOpt) *UnitHook {
|
||||
// lru.NewWithEvict can only fail if the size is < 0.
|
||||
cache, _ := lru.NewWithEvict(1024, func(key string, value *UnitMetrics) {
|
||||
if value.handled {
|
||||
return
|
||||
}
|
||||
ctx.Logger().Error(fmt.Errorf("eviction"), "dropping unit metric",
|
||||
"id", key,
|
||||
"metric", value,
|
||||
)
|
||||
})
|
||||
hook := UnitHook{metrics: cache}
|
||||
for _, opt := range opts {
|
||||
opt(&hook)
|
||||
}
|
||||
return &hook
|
||||
}
|
||||
|
||||
// id is a helper method to generate an ID for the given job and unit.
|
||||
func (u *UnitHook) id(ref JobProgressRef, unit SourceUnit) string {
|
||||
unitID := ""
|
||||
if unit != nil {
|
||||
unitID = unit.SourceUnitID()
|
||||
}
|
||||
return fmt.Sprintf("%d/%d/%s", ref.SourceID, ref.JobID, unitID)
|
||||
}
|
||||
|
||||
func (u *UnitHook) StartUnitChunking(ref JobProgressRef, unit SourceUnit, start time.Time) {
|
||||
id := u.id(ref, unit)
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
u.metrics.Add(id, &UnitMetrics{
|
||||
Unit: unit,
|
||||
Parent: ref,
|
||||
StartTime: start,
|
||||
})
|
||||
}
|
||||
|
||||
func (u *UnitHook) EndUnitChunking(ref JobProgressRef, unit SourceUnit, end time.Time) {
|
||||
id := u.id(ref, unit)
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
metrics, ok := u.metrics.Get(id)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
metrics.EndTime = end
|
||||
}
|
||||
|
||||
func (u *UnitHook) ReportChunk(ref JobProgressRef, unit SourceUnit, chunk *Chunk) {
|
||||
id := u.id(ref, unit)
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
metrics, ok := u.metrics.Get(id)
|
||||
if !ok && unit != nil {
|
||||
// The unit has been evicted.
|
||||
return
|
||||
} else if !ok && unit == nil {
|
||||
// This is a chunk from a non-unit source.
|
||||
metrics = &UnitMetrics{
|
||||
Unit: nil,
|
||||
Parent: ref,
|
||||
StartTime: ref.Snapshot().StartTime,
|
||||
}
|
||||
u.metrics.Add(id, metrics)
|
||||
}
|
||||
metrics.TotalChunks++
|
||||
metrics.TotalBytes += uint64(len(chunk.Data))
|
||||
}
|
||||
|
||||
func (u *UnitHook) ReportError(ref JobProgressRef, err error) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
|
||||
// Always add the error to the nil unit if it exists.
|
||||
if metrics, ok := u.metrics.Get(u.id(ref, nil)); ok {
|
||||
metrics.Errors = append(metrics.Errors, err)
|
||||
}
|
||||
|
||||
// Check if it's a ChunkError for a specific unit.
|
||||
var chunkErr ChunkError
|
||||
if !errors.As(err, &chunkErr) {
|
||||
return
|
||||
}
|
||||
id := u.id(ref, chunkErr.Unit)
|
||||
|
||||
metrics, ok := u.metrics.Get(id)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
metrics.Errors = append(metrics.Errors, err)
|
||||
}
|
||||
|
||||
func (u *UnitHook) Finish(ref JobProgressRef) {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
// Clear out any metrics on this job. This covers the case for the
|
||||
// source running without unit support.
|
||||
prefix := u.id(ref, nil)
|
||||
for _, id := range u.metrics.Keys() {
|
||||
if !strings.HasPrefix(id, prefix) {
|
||||
continue
|
||||
}
|
||||
metric, ok := u.metrics.Get(id)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// If the unit is nil, the source does not support units.
|
||||
// Use the overall job metrics instead.
|
||||
if metric.Unit == nil {
|
||||
snap := ref.Snapshot()
|
||||
metric.StartTime = snap.StartTime
|
||||
metric.EndTime = snap.EndTime
|
||||
metric.Errors = snap.Errors
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// UnitMetrics gets all the currently active or newly finished metrics for this
|
||||
// job. If a unit returned from this method has finished, it will be removed
|
||||
// from the cache and no longer returned in successive calls to UnitMetrics().
|
||||
func (u *UnitHook) UnitMetrics() []UnitMetrics {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
output := make([]UnitMetrics, 0, u.metrics.Len())
|
||||
for _, id := range u.metrics.Keys() {
|
||||
metric, ok := u.metrics.Get(id)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
output = append(output, *metric)
|
||||
if metric.IsFinished() {
|
||||
metric.handled = true
|
||||
u.metrics.Remove(id)
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
type UnitMetrics struct {
|
||||
Unit SourceUnit
|
||||
Parent JobProgressRef
|
||||
// Start and end time for chunking this unit.
|
||||
StartTime time.Time
|
||||
EndTime time.Time
|
||||
// Total number of chunks produced from this unit.
|
||||
TotalChunks uint64
|
||||
// Total number of bytes produced from this unit.
|
||||
TotalBytes uint64
|
||||
// All errors encountered by this unit.
|
||||
Errors []error
|
||||
// Flag to mark that these metrics were intentionally evicted from
|
||||
// the cache.
|
||||
handled bool
|
||||
}
|
||||
|
||||
func (u UnitMetrics) IsFinished() bool {
|
||||
return !u.EndTime.IsZero()
|
||||
}
|
||||
|
||||
// ElapsedTime is a convenience method that provides the elapsed time the job
|
||||
// has been running. If it hasn't started yet, 0 is returned. If it has
|
||||
// finished, the total time is returned.
|
||||
func (u UnitMetrics) ElapsedTime() time.Duration {
|
||||
if u.StartTime.IsZero() {
|
||||
return 0
|
||||
}
|
||||
if u.EndTime.IsZero() {
|
||||
return time.Since(u.StartTime)
|
||||
}
|
||||
return u.EndTime.Sub(u.StartTime)
|
||||
}
|
||||
|
||||
// NoopHook implements JobProgressHook by doing nothing. This is useful for
|
||||
// embedding in other structs to overwrite only the methods of the interface
|
||||
// that you care about.
|
||||
type NoopHook struct{}
|
||||
|
||||
func (NoopHook) Start(JobProgressRef, time.Time) {}
|
||||
func (NoopHook) End(JobProgressRef, time.Time) {}
|
||||
func (NoopHook) StartEnumerating(JobProgressRef, time.Time) {}
|
||||
func (NoopHook) EndEnumerating(JobProgressRef, time.Time) {}
|
||||
func (NoopHook) StartUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
|
||||
func (NoopHook) EndUnitChunking(JobProgressRef, SourceUnit, time.Time) {}
|
||||
func (NoopHook) ReportError(JobProgressRef, error) {}
|
||||
func (NoopHook) ReportUnit(JobProgressRef, SourceUnit) {}
|
||||
func (NoopHook) ReportChunk(JobProgressRef, SourceUnit, *Chunk) {}
|
||||
func (NoopHook) Finish(JobProgressRef) {}
|
||||
@@ -126,26 +126,3 @@ func TestJobProgressElapsedTime(t *testing.T) {
|
||||
metrics.EndTime = metrics.StartTime.Add(1 * time.Hour)
|
||||
assert.Equal(t, metrics.ElapsedTime(), 1*time.Hour)
|
||||
}
|
||||
|
||||
func TestJobProgressErrorsFor(t *testing.T) {
|
||||
metrics := JobProgressMetrics{
|
||||
Errors: []error{
|
||||
Fatal{ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "foo"},
|
||||
Err: fmt.Errorf("foo error"),
|
||||
}},
|
||||
ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "foo"},
|
||||
Err: fmt.Errorf("foo again error"),
|
||||
},
|
||||
ChunkError{
|
||||
Unit: CommonSourceUnit{ID: "bar"},
|
||||
Err: fmt.Errorf("bar error"),
|
||||
},
|
||||
fmt.Errorf("hi there"),
|
||||
},
|
||||
}
|
||||
assert.Equal(t, 2, len(metrics.ErrorsFor(CommonSourceUnit{ID: "foo"})))
|
||||
assert.Equal(t, 1, len(metrics.ErrorsFor(CommonSourceUnit{ID: "bar"})))
|
||||
assert.Equal(t, 0, len(metrics.ErrorsFor(CommonSourceUnit{ID: "baz"})))
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/aws/aws-sdk-go/service/sts"
|
||||
diskbufferreader "github.com/trufflesecurity/disk-buffer-reader"
|
||||
diskbufferreader "github.com/bill-rich/disk-buffer-reader"
|
||||
"github.com/go-errors/errors"
|
||||
"github.com/go-logr/logr"
|
||||
"golang.org/x/sync/errgroup"
|
||||
|
||||
@@ -3,10 +3,8 @@ package sources
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/types/known/anypb"
|
||||
|
||||
@@ -202,8 +200,6 @@ func (c *unitChunker) ChunkUnit(ctx context.Context, unit SourceUnit, rep ChunkR
|
||||
if err := rep.ChunkErr(ctx, fmt.Errorf(step.err)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if step.output == "" {
|
||||
continue
|
||||
}
|
||||
if err := rep.ChunkOk(ctx, Chunk{Data: []byte(step.output)}); err != nil {
|
||||
@@ -314,114 +310,3 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
|
||||
<-ref.Done() // Wait for the job to finish.
|
||||
assert.Equal(t, 1337, mgr.AvailableCapacity())
|
||||
}
|
||||
|
||||
func TestSourceManagerUnitHook(t *testing.T) {
|
||||
hook := NewUnitHook(context.TODO())
|
||||
|
||||
input := []unitChunk{
|
||||
{unit: "one", output: "bar"},
|
||||
{unit: "two", err: "oh no"},
|
||||
{unit: "three", err: "not again"},
|
||||
}
|
||||
mgr := NewManager(
|
||||
WithBufferedOutput(8),
|
||||
WithSourceUnits(), WithConcurrentUnits(1),
|
||||
WithReportHook(hook),
|
||||
)
|
||||
source, err := buildDummy(&unitChunker{input})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
|
||||
metrics := hook.UnitMetrics()
|
||||
assert.Equal(t, 3, len(metrics))
|
||||
sort.Slice(metrics, func(i, j int) bool {
|
||||
return metrics[i].EndTime.Before(metrics[j].EndTime)
|
||||
})
|
||||
m0, m1, m2 := metrics[0], metrics[1], metrics[2]
|
||||
|
||||
assert.Equal(t, "one", m0.Unit.SourceUnitID())
|
||||
assert.Equal(t, uint64(1), m0.TotalChunks)
|
||||
assert.Equal(t, uint64(3), m0.TotalBytes)
|
||||
assert.NotZero(t, m0.StartTime)
|
||||
assert.NotZero(t, m0.EndTime)
|
||||
assert.NotZero(t, m0.ElapsedTime())
|
||||
assert.Equal(t, 0, len(m0.Errors))
|
||||
|
||||
assert.Equal(t, "two", m1.Unit.SourceUnitID())
|
||||
assert.Equal(t, uint64(0), m1.TotalChunks)
|
||||
assert.Equal(t, uint64(0), m1.TotalBytes)
|
||||
assert.NotZero(t, m1.StartTime)
|
||||
assert.NotZero(t, m1.EndTime)
|
||||
assert.NotZero(t, m1.ElapsedTime())
|
||||
assert.Equal(t, 1, len(m1.Errors))
|
||||
|
||||
assert.Equal(t, "three", m2.Unit.SourceUnitID())
|
||||
assert.Equal(t, uint64(0), m2.TotalChunks)
|
||||
assert.Equal(t, uint64(0), m2.TotalBytes)
|
||||
assert.NotZero(t, m2.StartTime)
|
||||
assert.NotZero(t, m2.EndTime)
|
||||
assert.NotZero(t, m2.ElapsedTime())
|
||||
assert.Equal(t, 1, len(m2.Errors))
|
||||
}
|
||||
|
||||
// TestSourceManagerUnitHookNoBlock tests that the UnitHook drops metrics if
|
||||
// they aren't handled fast enough.
|
||||
func TestSourceManagerUnitHookNoBlock(t *testing.T) {
|
||||
var evictedKeys []string
|
||||
cache, _ := lru.NewWithEvict(1, func(key string, _ *UnitMetrics) {
|
||||
evictedKeys = append(evictedKeys, key)
|
||||
})
|
||||
hook := NewUnitHook(context.TODO(), WithUnitHookCache(cache))
|
||||
|
||||
input := []unitChunk{
|
||||
{unit: "one", output: "bar"},
|
||||
{unit: "two", err: "oh no"},
|
||||
{unit: "three", err: "not again"},
|
||||
}
|
||||
mgr := NewManager(
|
||||
WithBufferedOutput(8),
|
||||
WithSourceUnits(), WithConcurrentUnits(1),
|
||||
WithReportHook(hook),
|
||||
)
|
||||
source, err := buildDummy(&unitChunker{input})
|
||||
assert.NoError(t, err)
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
|
||||
assert.Equal(t, 2, len(evictedKeys))
|
||||
metrics := hook.UnitMetrics()
|
||||
assert.Equal(t, 1, len(metrics))
|
||||
assert.Equal(t, "three", metrics[0].Unit.SourceUnitID())
|
||||
}
|
||||
|
||||
// TestSourceManagerUnitHookNoUnits tests whether the UnitHook works for
|
||||
// sources that don't support units.
|
||||
func TestSourceManagerUnitHookNoUnits(t *testing.T) {
|
||||
hook := NewUnitHook(context.TODO())
|
||||
|
||||
mgr := NewManager(
|
||||
WithBufferedOutput(8),
|
||||
WithReportHook(hook),
|
||||
)
|
||||
source, err := buildDummy(&counterChunker{count: 5})
|
||||
assert.NoError(t, err)
|
||||
|
||||
ref, err := mgr.Run(context.Background(), "dummy", source)
|
||||
assert.NoError(t, err)
|
||||
<-ref.Done()
|
||||
|
||||
metrics := hook.UnitMetrics()
|
||||
assert.Equal(t, 1, len(metrics))
|
||||
|
||||
m := metrics[0]
|
||||
assert.Equal(t, nil, m.Unit)
|
||||
assert.Equal(t, uint64(5), m.TotalChunks)
|
||||
assert.Equal(t, uint64(5), m.TotalBytes)
|
||||
assert.NotZero(t, m.StartTime)
|
||||
assert.NotZero(t, m.EndTime)
|
||||
assert.NotZero(t, m.ElapsedTime())
|
||||
assert.Equal(t, 0, len(m.Errors))
|
||||
}
|
||||
|
||||
+4
-14
@@ -361,7 +361,7 @@ enum DetectorType {
|
||||
Imagga = 349;
|
||||
SMSApi = 350;
|
||||
Distribusion = 351;
|
||||
Blablabus = 352 [deprecated = true];
|
||||
Blablabus = 352;
|
||||
WordsApi = 353;
|
||||
Currencylayer = 354;
|
||||
Html2Pdf = 355;
|
||||
@@ -588,7 +588,7 @@ enum DetectorType {
|
||||
Qubole = 577;
|
||||
CarbonInterface = 578;
|
||||
Intrinio = 579;
|
||||
QuickMetrics = 580 [deprecated = true];
|
||||
QuickMetrics = 580;
|
||||
ScrapeStack = 581;
|
||||
TechnicalAnalysisApi = 582;
|
||||
Urlscan = 583;
|
||||
@@ -628,7 +628,7 @@ enum DetectorType {
|
||||
Nylas = 617;
|
||||
Squareup = 618;
|
||||
Dandelion = 619;
|
||||
DataFire = 620 [deprecated = true];
|
||||
DataFire = 620;
|
||||
DeepAI = 621;
|
||||
MeaningCloud = 622;
|
||||
NeutrinoApi = 623;
|
||||
@@ -780,7 +780,7 @@ enum DetectorType {
|
||||
ConversionTools = 769;
|
||||
CraftMyPDF = 770;
|
||||
ExportSDK = 771;
|
||||
GlitterlyAPI = 772 [deprecated = true];
|
||||
GlitterlyAPI = 772;
|
||||
Hybiscus = 773;
|
||||
Miro = 774;
|
||||
Statuspage = 775;
|
||||
@@ -945,16 +945,6 @@ enum DetectorType {
|
||||
Ramp = 934;
|
||||
Klaviyo = 935;
|
||||
SourcegraphCody = 936;
|
||||
Voiceflow = 937;
|
||||
Privacy = 938;
|
||||
IPInfo = 939;
|
||||
Ip2location = 940;
|
||||
Instamojo = 941;
|
||||
Portainer = 942;
|
||||
PortainerToken = 943;
|
||||
Loggly = 944;
|
||||
OpenVpn = 945;
|
||||
VagrantCloudPersonalToken = 946;
|
||||
}
|
||||
|
||||
message Result {
|
||||
|
||||
+1
-8
@@ -49,16 +49,9 @@ enum SourceType {
|
||||
message LocalSource {
|
||||
string type = 1 ;
|
||||
string name = 2 ;
|
||||
// DEPRECATED: scan_interval is deprecated and can be removed when we no longer depend on the name.
|
||||
// Deprecating in favor of scan_period due to the fact that scan_interval is a duration
|
||||
// which is a fixed-length span of time represented as a count of seconds and fractions of seconds
|
||||
// at nanosecond resolution. Most of the time, we want to be able to specify a scan interval in
|
||||
// human-readable format (e.g. 45s, 30m, 12h, etc.) which is not possible with a duration.
|
||||
// https://protobuf.dev/reference/protobuf/google.protobuf/#duration
|
||||
google.protobuf.Duration scan_interval = 3 [deprecated = true];
|
||||
google.protobuf.Duration scan_interval = 3;
|
||||
bool verify = 4;
|
||||
google.protobuf.Any connection = 5;
|
||||
string scan_period = 6;
|
||||
}
|
||||
|
||||
message AzureStorage {
|
||||
|
||||
Reference in New Issue
Block a user