junk

wip
2022-08-22 08:46:44 -07:00 · 2022-08-16 08:04:17 -07:00 · 2022-08-16 07:54:10 -07:00 · 2022-08-16 07:36:03 -07:00 · 2022-08-15 13:12:59 -07:00 · 2022-08-12 14:53:37 -07:00
114 changed files with 2241 additions and 1205 deletions
@@ -13,6 +13,8 @@ builds:
 dockers:
  - image_templates: ["trufflesecurity/{{ .ProjectName }}:{{ .Version }}-amd64"]
    dockerfile: Dockerfile.goreleaser
+    extra_files:
+    - entrypoint.sh
    use: buildx
    build_flag_templates:
    - --platform=linux/amd64
@@ -26,6 +28,8 @@ dockers:
  - image_templates: ["trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8"]
    goarch: arm64
    dockerfile: Dockerfile.goreleaser
+    extra_files:
+    - entrypoint.sh
    use: buildx
    build_flag_templates:
    - --platform=linux/arm64/v8
@@ -38,6 +42,8 @@ dockers:
    - --label=org.opencontainers.image.licenses=AGPL-3.0
  - image_templates: ["ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-amd64"]
    dockerfile: Dockerfile.goreleaser
+    extra_files:
+    - entrypoint.sh
    use: buildx
    build_flag_templates:
    - --platform=linux/amd64
@@ -51,6 +57,8 @@ dockers:
  - image_templates: ["ghcr.io/trufflesecurity/{{ .ProjectName }}:{{ .Version }}-arm64v8"]
    goarch: arm64
    dockerfile: Dockerfile.goreleaser
+    extra_files:
+    - entrypoint.sh
    use: buildx
    build_flag_templates:
    - --platform=linux/arm64/v8
@@ -9,8 +9,9 @@ RUN  --mount=type=cache,target=/go/pkg/mod \
     GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -o trufflehog .

 FROM alpine:3.15
-RUN apk add --no-cache git
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+RUN apk add --no-cache git ca-certificates \
+    && rm -rf /var/cache/apk/* && \
+    update-ca-certificates
 COPY --from=builder /build/trufflehog /usr/bin/trufflehog
 COPY entrypoint.sh /etc/entrypoint.sh
 RUN chmod +x /etc/entrypoint.sh
@@ -1,7 +1,10 @@
 FROM alpine:3.15

-RUN apk add --no-cache git
+RUN apk add --no-cache git ca-certificates \
+    && rm -rf /var/cache/apk/* && \
+    update-ca-certificates
 WORKDIR /usr/bin/
 COPY trufflehog .
-
-ENTRYPOINT ["/usr/bin/trufflehog"]
+COPY entrypoint.sh /etc/entrypoint.sh
+RUN chmod +x /etc/entrypoint.sh
+ENTRYPOINT ["/etc/entrypoint.sh"]
@@ -57,3 +57,6 @@ release-protos-image:

 snifftest:
 	./hack/snifftest/snifftest.sh
+
+test-release:
+	goreleaser release --rm-dist --skip-publish --snapshot
@@ -6,14 +6,14 @@

 ---

-<center>
+<div align="center">

 [![CI Status](https://github.com/trufflesecurity/trufflehog/actions/workflows/release.yml/badge.svg)](https://github.com/trufflesecurity/trufflehog/actions)
 [![Go Report Card](https://goreportcard.com/badge/github.com/trufflesecurity/trufflehog/v3)](https://goreportcard.com/report/github.com/trufflesecurity/trufflehog/v3)
 [![License](https://img.shields.io/badge/license-AGPL--3.0-brightgreen)](/LICENSE)
-[![Total Detectors](https://shields-staging.herokuapp.com/github/directory-file-count/trufflesecurity/truffleHog/pkg/detectors?label=Total%20Detectors&type=dir)](/pkg/detectors) <!-- Badge must be run from staging, see badges/shields#5967 -->
+[![Total Detectors](https://img.shields.io/github/directory-file-count/trufflesecurity/truffleHog/pkg/detectors?label=Total%20Detectors&type=dir)](/pkg/detectors)

-</center>
+</div>

 ---

@@ -169,6 +169,8 @@ docker run -it -v "$PWD:/pwd" trufflesecurity/trufflehog:latest github --org=tru
    base: 
    # Scan commits until here (usually dev branch).
    head: # optional
+    # Extra args to be passed to the trufflehog cli.
+    extra_args: # optional
 ```

 The TruffleHog OSS Github Action can be used to scan a range of commits for leaked credentials. The action will fail if
@@ -192,6 +194,26 @@ jobs:
          path: ./
          base: ${{ github.event.repository.default_branch }}
          head: HEAD
+          extra_args: --debug
+```
+
+### Precommit Hook
+
+Trufflehog can be used in a precommit hook to prevent credentials from leaking before they ever leave your computer.
+An example `.pre-commit-config.yaml` is provided (see [pre-commit.com](https://pre-commit.com/) for installation).
+
+```yaml
+repos:
+- repo: local
+  hooks:
+    - id: trufflehog
+      name: TruffleHog
+      description: Detect secrets in your data.
+      entry: bash -c 'trufflehog git file://. --only-verified --fail'
+      # For running trufflehog in docker, use the following entry instead:
+      # entry: bash -c 'docker run -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --only-verified --fail'
+      language: system
+      stages: ["commit", "push"]
 ```

 ## Contributors
@@ -4,14 +4,18 @@ go 1.18

 replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5

-replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom9
+replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10

 replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2

+replace github.com/bill-rich/gitparser => /home/hrich/go/src/github.com/bill-rich/gitparse
+
 require (
 	cloud.google.com/go/secretmanager v1.5.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
 	github.com/aws/aws-sdk-go v1.44.61
+	github.com/bill-rich/disk-buffer-reader v0.1.2
+	github.com/bill-rich/gitparser v0.0.0-00010101000000-000000000000
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
 	github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
@@ -19,7 +23,6 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.6.7
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
-	github.com/gitleaks/go-gitdiff v0.7.4
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible
@@ -31,6 +34,7 @@ require (
 	github.com/jpillora/overseer v1.1.6
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mattn/go-colorable v0.1.12
+	github.com/mholt/archiver/v4 v4.0.0-alpha.7
 	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/razorpay/razorpay-go v0.0.0-20210728161131-0341409a6ab2
@@ -40,13 +44,13 @@ require (
 	github.com/stretchr/testify v1.8.0
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/xanzy/go-gitlab v0.69.0
-	github.com/zricethezav/gitleaks/v8 v8.5.2
+	github.com/zricethezav/gitleaks/v8 v8.10.3
 	golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e
 	golang.org/x/net v0.0.0-20220617184016-355a448f1bc9
 	golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
 	google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad
-	google.golang.org/protobuf v1.28.0
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
 )
@@ -67,19 +71,23 @@ require (
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
+	github.com/andybalholm/brotli v1.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/gitleaks/go-gitdiff v0.7.6 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/go-github/v45 v45.2.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
+	github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
 	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
@@ -89,15 +97,21 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/s3 v1.1.4 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
+	github.com/klauspost/compress v1.15.5 // indirect
+	github.com/klauspost/pgzip v1.2.5 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.14 // indirect
 	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/therootcompany/xz v1.0.1 // indirect
+	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/mod v0.5.0 // indirect
-	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
+	golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
 	golang.org/x/tools v0.1.7 // indirect
@@ -91,6 +91,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafo
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE=
+github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
+github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -98,6 +100,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/aws/aws-sdk-go v1.44.61 h1:NcpLSS3Z0MiVQIYugx4I40vSIEEAXT0baO684ExNRco=
 github.com/aws/aws-sdk-go v1.44.61/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/bill-rich/disk-buffer-reader v0.1.2 h1:pN9K5JoacTcNWp2SCd3n7mPouSwMP9ouTS66Qa+5IPY=
+github.com/bill-rich/disk-buffer-reader v0.1.2/go.mod h1:VVzzsK1Ac2AnpOfp/5r9JlIFaFkZ9uSf7zisZayCt0Y=
 github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c h1:tSME5FDS02qQll3JYodI6RZR/g4EKOHApGv1wMZT+Z0=
 github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c/go.mod h1:+sCc6hztur+oZCLOsNk6wCCy+GLrnSNHSRmTnnL+8iQ=
 github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb h1:9v7Bzlg+1EBYi2IYcUmOwHReBEfqBbYIj3ZCi9cIe1Q=
@@ -129,6 +133,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
+github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
+github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -210,6 +217,8 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -251,8 +260,9 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 h1:8pyqKJvrJqUYaKS851Ule26pwWvey6IDMiczaBLDKLQ=
+github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
@@ -306,6 +316,12 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck=
 github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.15.5 h1:qyCLMz2JCrKADihKOh9FxnW3houKeNsp2h5OEz0QSEA=
+github.com/klauspost/compress v1.15.5/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -326,14 +342,20 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mholt/archiver/v4 v4.0.0-alpha.7 h1:xzByj8G8tj0Oq7ZYYU4+ixL/CVb5ruWCm0EZQ1PjOkE=
+github.com/mholt/archiver/v4 v4.0.0-alpha.7/go.mod h1:Fs8qUkO74HHaidabihzYephJH8qmGD/nCP6tE5xC9BM=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk=
+github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
+github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
+github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -377,12 +399,17 @@ github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PK
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom9 h1:OvS9aj6Fasot5FaTpSyCV4WNq/8SMov9/bNUMoZFwEI=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom9/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
+github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
+github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10 h1:QuGZ5bJcQpVz+3sfvKKPDkQwdUueiBg0V+2eMHzkryo=
+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom10/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
 github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2 h1:srCJzbE3b44+ZIPcgJSfvinHCOQlkMwVghtKf23un6o=
 github.com/trufflesecurity/go-gitdiff v0.7.6-zombies2/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
 github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
 github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
+github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
+github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/xanzy/go-gitlab v0.69.0 h1:sPci9xHzlX+lcJvPqNu3y3BQpePuR2R694Bal4AeyB8=
 github.com/xanzy/go-gitlab v0.69.0/go.mod h1:o4yExCtdaqlM8YGdDJWuZoBmfxBsmA9TPEjs9mx1UO4=
 github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI=
@@ -601,8 +628,9 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -858,8 +886,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -17,9 +17,11 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/jpillora/overseer"
 	"github.com/sirupsen/logrus"
+	"gopkg.in/alecthomas/kingpin.v2"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/version"
-	"gopkg.in/alecthomas/kingpin.v2"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
@@ -189,36 +191,75 @@ func run(state overseer.State) {
 		if remote {
 			defer os.RemoveAll(repoPath)
 		}
-		err = e.ScanGit(ctx, repoPath, *gitScanBranch, *gitScanSinceCommit, *gitScanMaxDepth, filter)
-		if err != nil {
-			logrus.WithError(err).Fatal("Failed to scan git.")
+
+		config := sources.Config{
+			RepoPath: repoPath,
+			HeadRef:  *gitScanBranch,
+			BaseRef:  *gitScanSinceCommit,
+			MaxDepth: *gitScanMaxDepth,
+			Filter:   filter,
+		}
+
+		if err = e.ScanGit(ctx, config); err != nil {
+			logrus.WithError(err).Fatal("Failed to scan Git.")
 		}
 	case githubScan.FullCommand():
 		if len(*githubScanOrgs) == 0 && len(*githubScanRepos) == 0 {
 			log.Fatal("You must specify at least one organization or repository.")
 		}
-		err = e.ScanGitHub(ctx, *githubScanEndpoint, *githubScanRepos, *githubScanOrgs, *githubScanToken, *githubIncludeForks, filter, *concurrency, *githubIncludeMembers)
-		if err != nil {
-			logrus.WithError(err).Fatal("Failed to scan git.")
+
+		github := func(c sources.Config) {
+			c.Endpoint = *githubScanEndpoint
+			c.Repos = *githubScanRepos
+			c.Orgs = *githubScanOrgs
+			c.Token = *githubScanToken
+			c.IncludeForks = *githubIncludeForks
+			c.IncludeMembers = *githubIncludeMembers
+			c.Concurrency = *concurrency
+		}
+
+		if err = e.ScanGitHub(ctx, sources.NewConfig(github)); err != nil {
+			logrus.WithError(err).Fatal("Failed to scan Github.")
 		}
 	case gitlabScan.FullCommand():
-		err := e.ScanGitLab(ctx, *gitlabScanEndpoint, *gitlabScanToken, *gitlabScanRepos)
-		if err != nil {
+		gitlab := func(c sources.Config) {
+			c.Endpoint = *gitlabScanEndpoint
+			c.Token = *gitlabScanToken
+			c.Repos = *gitlabScanRepos
+		}
+
+		if err = e.ScanGitLab(ctx, sources.NewConfig(gitlab)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan GitLab.")
 		}
 	case filesystemScan.FullCommand():
-		err := e.ScanFileSystem(ctx, *filesystemDirectories)
-		if err != nil {
+		fs := func(c sources.Config) {
+			c.Directories = *filesystemDirectories
+		}
+
+		if err = e.ScanFileSystem(ctx, sources.NewConfig(fs)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan filesystem")
 		}
 	case s3Scan.FullCommand():
-		err := e.ScanS3(ctx, *s3ScanKey, *s3ScanSecret, *s3ScanCloudEnv, *s3ScanBuckets)
-		if err != nil {
+		s3 := func(c sources.Config) {
+			c.Key = *s3ScanKey
+			c.Secret = *s3ScanSecret
+			c.Buckets = *s3ScanBuckets
+		}
+
+		if err = e.ScanS3(ctx, sources.NewConfig(s3)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan S3.")
 		}
 	case syslogScan.FullCommand():
-		err := e.ScanSyslog(ctx, *syslogAddress, *syslogProtocol, *syslogTLSCert, *syslogTLSKey, *syslogFormat, *concurrency)
-		if err != nil {
+		syslog := func(c sources.Config) {
+			c.Address = *syslogAddress
+			c.Protocol = *syslogProtocol
+			c.CertPath = *syslogTLSCert
+			c.KeyPath = *syslogTLSKey
+			c.Format = *syslogFormat
+			c.Concurrency = *concurrency
+		}
+
+		if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil {
 			logrus.WithError(err).Fatal("Failed to scan syslog.")
 		}
 	}
@@ -0,0 +1,36 @@
+package common
+
+import (
+	"bufio"
+	"errors"
+	"io"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	ChunkSize = 10 * 1024
+	PeekSize  = 3 * 1024
+)
+
+func ChunkReader(r io.Reader) chan []byte {
+	chunkChan := make(chan []byte)
+	go func() {
+		defer close(chunkChan)
+		reader := bufio.NewReaderSize(bufio.NewReader(r), ChunkSize)
+		for {
+			chunk := make([]byte, ChunkSize)
+			n, err := reader.Read(chunk)
+			if err != nil && !errors.Is(err, io.EOF) {
+				log.WithError(err).Error("Error chunking reader.")
+				break
+			}
+			peekData, _ := reader.Peek(PeekSize)
+			chunkChan <- append(chunk[:n], peekData...)
+			if errors.Is(err, io.EOF) {
+				break
+			}
+		}
+	}()
+	return chunkChan
+}
@@ -0,0 +1,75 @@
+package common
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"io"
+	"net/http"
+	"testing"
+
+	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
+)
+
+func TestChunker(t *testing.T) {
+	resp, err := http.Get("https://raw.githubusercontent.com/bill-rich/bad-secrets/master/FifteenMB.gz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	reReader, err := diskbufferreader.New(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer reReader.Close()
+
+	baseChunkCount := 0
+
+	// Count chunks from looping using chunk size.
+	for {
+		baseChunkCount++
+		tmpChunk := make([]byte, ChunkSize)
+		_, err := reReader.Read(tmpChunk)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			t.Error(err)
+		}
+	}
+	_ = reReader.Reset()
+
+	// Get the first two chunks for comparing later.
+	baseChunkOne := make([]byte, ChunkSize)
+	baseChunkTwo := make([]byte, ChunkSize)
+
+	baseReader := bufio.NewReaderSize(reReader, ChunkSize)
+	_, _ = baseReader.Read(baseChunkOne)
+	peek, _ := baseReader.Peek(PeekSize)
+	baseChunkOne = append(baseChunkOne, peek...)
+	_, _ = baseReader.Read(baseChunkTwo)
+	peek, _ = baseReader.Peek(PeekSize)
+	baseChunkTwo = append(baseChunkTwo, peek...)
+
+	// Reset the reader to the beginning and use ChunkReader.
+	_ = reReader.Reset()
+
+	testChunkCount := 0
+	for chunk := range ChunkReader(reReader) {
+		testChunkCount++
+		switch testChunkCount {
+		case 1:
+			if !bytes.Equal(baseChunkOne, chunk) {
+				t.Errorf("First chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk), len(baseChunkOne))
+			}
+		case 2:
+			if !bytes.Equal(baseChunkTwo, chunk) {
+				t.Errorf("Second chunk did not match expected. Got: %d bytes, expected: %d bytes", len(chunk), len(baseChunkTwo))
+			}
+		}
+	}
+	if testChunkCount != baseChunkCount {
+		t.Errorf("Wrong number of chunks received. Got %d, expected: %d.", testChunkCount, baseChunkCount)
+	}
+
+}
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_AdobeIO,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Adzuna,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Aeroworkflow,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Agora,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_AirbrakeProjectKey,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_AirtableApiKey,
 				Redacted:     appRes,
 				Raw:          []byte(keyRes),
+				RawV2:        []byte(keyRes + appRes),
 			}

 			if verify {
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Alegra,
 				Raw:          []byte(tokenPatMatch),
+				RawV2:        []byte(tokenPatMatch + userPatMatch),
 			}

 			if verify {
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_AlgoliaAdminKey,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -84,6 +84,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Alibaba,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -103,7 +104,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				params.Add("Version", "2014-05-26")

 				stringToSign := buildStringToSign(req.Method, params.Encode())
-				signature := GetSignature(stringToSign, resMatch+"&") //Get Signature HMAC SHA1
+				signature := GetSignature(stringToSign, resMatch+"&") // Get Signature HMAC SHA1
 				params.Add("Signature", signature)
 				req.URL.RawQuery = params.Encode()

@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Amadeus,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_AmplitudeApiKey,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecretMatch),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Anypoint,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + orgRes),
 			}

 			if verify {
@@ -61,7 +61,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result

 				var result Response
 				if errBody == nil {
-					json.Unmarshal(body, &result)
+					if err := json.Unmarshal(body, &result); err != nil {
+						continue
+					}

 					if res.StatusCode >= 200 && res.StatusCode < 300 && result.ReturnCode == 0 {
 						s1.Verified = true
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_ApiDeck,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Apiflash,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resUrlMatch),
 			}

 			if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_APIMatic,
 				Raw:          []byte(userPatMatch),
+				RawV2:        []byte(userPatMatch + passPatMatch),
 			}
 			if verify {
 				timeout := 10 * time.Second
@@ -63,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_Appcues,
 					Raw:          []byte(resMatch),
+					RawV2:        []byte(resMatch + resUserMatch),
 				}
 				if verify {
 					req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://api.appcues.com/v2/accounts/%s/flows", resIdMatch), nil)
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Apptivo,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_ArtifactoryAccessToken,
 			Raw:          []byte(resMatch),
+			RawV2:        []byte(resMatch + resURLMatch),
 		}

 		if verify {
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Artsy,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -58,6 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_Auth0ManagementApiToken,
 				Redacted:     domainRes,
 				Raw:          []byte(managementApiTokenRes),
+				RawV2:        []byte(managementApiTokenRes + domainRes),
 			}

 			if verify {
@@ -2,14 +2,11 @@ package auth0oauth

 import (
 	"context"
-	// "fmt"
-	// "log"
-	"regexp"
-	"strings"
-
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"regexp"
+	"strings"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -65,6 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					DetectorType: detectorspb.DetectorType_Auth0oauth,
 					Redacted:     clientIdRes,
 					Raw:          []byte(clientSecretRes),
+					RawV2:        []byte(clientIdRes + clientSecretRes),
 				}

 				if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Autodesk,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -27,7 +27,7 @@ var (
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
 	idPat     = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
-	secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})[ \r\n'"\x60]`)
+	secretPat = regexp.MustCompile(`[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}`)
 	// Hashes, like those for git, do technically match the secret pattern.
 	// But they are extremely unlikely to be generated as an actual AWS secret.
 	// So when we find them, if they're not verified, we should ignore the result.
@@ -81,6 +81,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_AWS,
 				Raw:          []byte(resIDMatch),
 				Redacted:     resIDMatch,
+				RawV2:        []byte(resIDMatch + resSecretMatch),
 			}

 			if verify {
@@ -144,6 +144,23 @@ func TestAWS_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, unverified, with leading +",
+			s:    Scanner{},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_AWS,
+					Verified:     false,
+					Redacted:     "AKIAWARWQKZNHMZBLY4I",
+				},
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Aylien,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}
 			if verify {
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.aylien.com/news/stories", nil)
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Billomat,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resId),
 			}

 			if verify {
@@ -58,6 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Bitmex,
 				Raw:          []byte(resSecretMatch),
+				RawV2:        []byte(resMatch + resSecretMatch),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_BrowserStack,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resUserMatch),
 			}

 			if verify {
@@ -2,11 +2,11 @@ package bulksms

 import (
 	"context"
+	b64 "encoding/base64"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
-	b64 "encoding/base64"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -21,10 +21,9 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client = common.SaneHttpClient()

-	//Make sure that your group is surrounded in boundry characters such as below to reduce false positives
+	// Make sure that your group is surrounded in boundry characters such as below to reduce false positives
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([a-fA-Z0-9*]{29})\b`)
-	idPat = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([A-F0-9-]{37})\b`)
-
+	idPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"bulksms"}) + `\b([A-F0-9-]{37})\b`)
 )

 // Keywords are used for efficiently pre-filtering chunks.
@@ -54,10 +53,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Bulksms,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}
-	
+
 			if verify {
-				data := fmt.Sprintf("%s:%s", resIdMatch,resMatch)
+				data := fmt.Sprintf("%s:%s", resIdMatch, resMatch)
 				sEnc := b64.StdEncoding.EncodeToString([]byte(data))
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.bulksms.com/v1/messages", nil)
 				if err != nil {
@@ -70,17 +70,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
-						//This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+						// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}
 				}
 			}
-	
+
 			results = append(results, s1)
 		}
-		
+
 	}

 	return detectors.CleanResults(results), nil
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_CaptainData,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resProjIdMatch + resMatch),
 			}

 			if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Cashboard,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resUser),
 			}

 			if verify {
@@ -63,6 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_Caspio,
 					Raw:          []byte(resMatch),
+					RawV2:        []byte(resMatch + resIdMatch + resDomainMatch),
 				}

 				if verify {
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Censys,
 				Raw:          []byte(tokenPatMatch),
+				RawV2:        []byte(tokenPatMatch + userPatMatch),
 			}

 			if verify {
@@ -69,6 +69,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_CexIO,
 					Raw:          []byte(resKeyMatch),
+					RawV2:        []byte(resUserIdMatch + resSecretMatch),
 				}

 				if verify {
@@ -102,7 +103,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						}

 						var responseObject Response
-						json.Unmarshal(body, &responseObject)
+						if err := json.Unmarshal(body, &responseObject); err != nil {
+							continue
+						}

 						if res.StatusCode >= 200 && res.StatusCode < 300 && validResponse {
 							s1.Verified = true
@@ -20,7 +20,7 @@ var (
 	client = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	//Tokens starting with sk_test are used for the app's sandbox environment while tokens starting with sk only are for production environment
+	// Tokens starting with sk_test are used for the app's sandbox environment while tokens starting with sk only are for production environment
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"checkout"}) + `\b((sk_|sk_test_)[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\b`)
 	idPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"checkout"}) + `\b(cus_[0-9a-zA-Z]{26})\b`)
 )
@@ -53,10 +53,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Checkout,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
-				//Used the app's sandbox environment for this case since I can't create a live account.
+				// Used the app's sandbox environment for this case since I can't create a live account.
 				req, err := http.NewRequestWithContext(ctx, "GET", "https://api.sandbox.checkout.com/customers/"+resIdMatch, nil)
 				if err != nil {
 					continue
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Checkvist,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resEmailMatch),
 			}

 			if verify {
@@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_ClickHelp,
 					Raw:          []byte(resServer),
+					RawV2:        []byte(resServer + resEmail),
 				}

 				if verify {
@@ -50,6 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_ClickSendsms,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_ClockworkSMS,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + tokenRes),
 			}

 			if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_CloudElements,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resOrgMatch),
 			}

 			if verify {
@@ -60,6 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_CloudflareGlobalApiKey,
 				Redacted:     emailRes,
 				Raw:          []byte(apiKeyRes),
+				RawV2:        []byte(apiKeyRes + emailRes),
 			}

 			if verify {
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_CompanyHub,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Confluent,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Copper,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_CrossBrowserTesting,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_CustomerGuru,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_CustomerIO,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -29,6 +29,9 @@ type Result struct {
 	Verified     bool
 	// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
 	Raw []byte
+	// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
+	// This is used for secrets that are multi part and could have the same ID. Ex: AWS credentials
+	RawV2 []byte
 	// Redacted contains the redacted version of the raw secret identification data for display purposes.
 	// A secret ID should be used if available.
 	Redacted       string
@@ -96,9 +99,9 @@ func PrefixRegex(keywords []string) string {
 	return pre + middle + post
 }

-//KeyIsRandom is a Low cost check to make sure that 'keys' include a number to reduce FPs.
-//Golang doesnt support regex lookaheads, so must be done in separate calls.
-//TODO improve checks. Shannon entropy did not work well.
+// KeyIsRandom is a Low cost check to make sure that 'keys' include a number to reduce FPs.
+// Golang doesnt support regex lookaheads, so must be done in separate calls.
+// TODO improve checks. Shannon entropy did not work well.
 func KeyIsRandom(key string) bool {
 	for _, ch := range key {
 		if unicode.IsDigit(ch) {
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_DiscordBotToken,
 				Redacted:     resId,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resId),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Dnscheck,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -56,6 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Dwolla,
 				Raw:          []byte(idMatch),
+				RawV2:        []byte(idMatch + secretMatch),
 			}

 			if verify {
@@ -2,13 +2,12 @@ package easyinsight

 import (
 	"context"
+	b64 "encoding/base64"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strings"

-	b64 "encoding/base64"
-	"net/http"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -55,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_EasyInsight,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Edamam,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resId),
 			}

 			if verify {
@@ -55,6 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_EightxEight,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resIdMatch),
 			}

 			if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_FacebookOAuth,
 				Redacted:     apiIdRes,
 				Raw:          []byte(apiSecretRes),
+				RawV2:        []byte(apiIdRes + apiSecretRes),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_FacePlusPlus,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_FourSquare,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -28,7 +28,7 @@ var (
 	client = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	//Removed bounds since there are some cases where the start and end of the token is a special character
+	// Removed bounds since there are some cases where the start and end of the token is a special character
 	keyPat    = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`)
 	secretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gengo"}) + `([ ]{0,1}[0-9a-zA-Z\[\]\-\(\)\{\}|_^@$=~]{64}[ \r\n]{1})`)
 )
@@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Gengo,
 				Raw:          []byte(resSecretMatch),
+				RawV2:        []byte(resMatch + resSecretMatch),
 			}

 			if verify {
@@ -80,7 +81,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result

 					if errBody == nil {
 						var response Response
-						json.Unmarshal(body, &response)
+						if err := json.Unmarshal(body, &response); err != nil {
+							continue
+						}

 						if res.StatusCode >= 200 && res.StatusCode < 300 && response.OpStat == "ok" {
 							s1.Verified = true
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Geocodio,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSearchMatch),
 			}

 			if verify {
@@ -76,7 +76,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result

 					if errBody == nil {
 						response := Response{}
-						xml.Unmarshal(body, &response)
+						if err := xml.Unmarshal(body, &response); err != nil {
+							continue
+						}

 						if res.StatusCode >= 200 && res.StatusCode < 300 && response.Error == nil {
 							s1.Verified = true
@@ -2,6 +2,10 @@ package kraken

 import (
 	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/base64"
 	"io"
 	"net/http"
 	"net/url"
@@ -10,11 +14,6 @@ import (
 	"strings"
 	"time"

-	"crypto/hmac"
-	"crypto/sha256"
-	"crypto/sha512"
-	"encoding/base64"
-
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -29,7 +28,7 @@ var (
 	client = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	//Bounds have been removed because there are some cases that tokens have trailing frontslash(/) or plus sign (+)
+	// Bounds have been removed because there are some cases that tokens have trailing frontslash(/) or plus sign (+)
 	keyPat     = regexp.MustCompile(detectors.PrefixRegex([]string{"kraken"}) + `\b([0-9A-Za-z\/\+=]{56}[ "'\r\n]{1})`)
 	privKeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kraken"}) + `\b([0-9A-Za-z\/\+=]{86,88}[ "'\r\n]{1})`)
 )
@@ -62,11 +61,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Kraken,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resPrivKeyMatch),
 			}

 			if verify {

-				//Increasing 64-bit integer, for each request that is made with a particular API key.
+				// Increasing 64-bit integer, for each request that is made with a particular API key.
 				apiNonce := strconv.FormatInt(time.Now().Unix(), 10)

 				payload := url.Values{}
@@ -106,7 +106,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 	return detectors.CleanResults(results), nil
 }

-//Code from https://docs.kraken.com/rest/#section/Authentication/Headers-and-Signature
+// Code from https://docs.kraken.com/rest/#section/Authentication/Headers-and-Signature
 func getKrakenSignature(url_path string, values url.Values, secret []byte) string {

 	sha := sha256.New()
@@ -65,6 +65,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_KuCoin,
 					Raw:          []byte(resKeyMatch),
+					RawV2:        []byte(resKeyMatch + resPassphraseMatch),
 				}

 				if verify {
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_MattermostPersonalToken,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + serverRes),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Mux,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecretMatch),
 			}

 			if verify {
@@ -51,6 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_NexmoApiKey,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -52,6 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				DetectorType: detectorspb.DetectorType_Plivo,
 				Redacted:     id,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + id),
 			}
 			stringResMatch := fmt.Sprintf("%s:%s", id, resMatch)
 			decodeSecret := b64.StdEncoding.EncodeToString([]byte(stringResMatch))
@@ -58,6 +58,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Poloniex,
 				Raw:          []byte(resSecretMatch),
+				RawV2:        []byte(resMatch + resSecretMatch),
 			}

 			if verify {
@@ -72,6 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_PusherChannelKey,
 					Raw:          []byte(resappMatch),
+					RawV2:        []byte(resappMatch + reskeyMatch),
 				}

 				if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Roaring,
 				Raw:          []byte(resClient),
+				RawV2:        []byte(resClient + resSecret),
 			}

 			if verify {
@@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_Rownd,
 					Raw:          []byte(keyMatch),
+					RawV2:        []byte(keyMatch + secretMatch),
 				}

 				if verify {
@@ -62,6 +62,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_SatismeterProjectkey,
 					Raw:          []byte(resMatch),
+					RawV2:        []byte(resMatch + resPassMatch),
 				}

 				if verify {
@@ -69,7 +69,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if err == nil {
 					defer res.Body.Close()
 					var authResponse authRes
-					json.NewDecoder(res.Body).Decode(&authResponse)
+					if err := json.NewDecoder(res.Body).Decode(&authResponse); err != nil {
+						continue
+					}
 					s.Verified = authResponse.Ok
 				}
 			}
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Smooch,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -60,6 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1 := detectors.Result{
 					DetectorType: detectorspb.DetectorType_Strava,
 					Raw:          []byte(resId),
+					RawV2:        []byte(resId + resSecret),
 				}

 				if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Textmagic,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resUser),
 			}

 			if verify {
@@ -53,6 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Tru,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + resSecret),
 			}

 			if verify {
@@ -54,6 +54,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_Vouchery,
 				Raw:          []byte(resMatch),
+				RawV2:        []byte(resMatch + subMatch),
 			}

 			if verify {
@@ -6,15 +6,18 @@ import (

 	"github.com/go-errors/errors"
 	"github.com/sirupsen/logrus"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/filesystem"
 )

-func (e *Engine) ScanFileSystem(ctx context.Context, directories []string) error {
+// ScanFileSystem scans a given file system.
+func (e *Engine) ScanFileSystem(ctx context.Context, c sources.Config) error {
 	connection := &sourcespb.Filesystem{
-		Directories: directories,
+		Directories: c.Directories,
 	}
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
@@ -6,39 +6,40 @@ import (
 	"runtime"

 	"github.com/go-errors/errors"
-	"github.com/go-git/go-git/v5/plumbing/object"
-
 	gogit "github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/sirupsen/logrus"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )

-func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string, maxDepth int, filter *common.Filter) error {
+// ScanGit scans any git source.
+func (e *Engine) ScanGit(ctx context.Context, c sources.Config) error {
 	logOptions := &gogit.LogOptions{}
 	opts := []git.ScanOption{
-		git.ScanOptionFilter(filter),
+		git.ScanOptionFilter(c.Filter),
 		git.ScanOptionLogOptions(logOptions),
 	}

-	repo, err := gogit.PlainOpenWithOptions(repoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
+	repo, err := gogit.PlainOpenWithOptions(c.RepoPath, &gogit.PlainOpenOptions{DetectDotGit: true})
 	if err != nil {
-		return fmt.Errorf("could open repo: %s: %w", repoPath, err)
+		return fmt.Errorf("could open repo: %s: %w", c.RepoPath, err)
 	}

 	var baseCommit *object.Commit
-	if len(baseRef) > 0 {
-		baseHash := plumbing.NewHash(baseRef)
-		if !plumbing.IsHash(baseRef) {
-			base, err := git.TryAdditionalBaseRefs(repo, baseRef)
+	if len(c.BaseRef) > 0 {
+		baseHash := plumbing.NewHash(c.BaseRef)
+		if !plumbing.IsHash(c.BaseRef) {
+			base, err := git.TryAdditionalBaseRefs(repo, c.BaseRef)
 			if err != nil {
 				return errors.WrapPrefix(err, "unable to resolve base ref", 0)
 			} else {
-				baseRef = base.String()
-				baseCommit, _ = repo.CommitObject(plumbing.NewHash(baseRef))
+				c.BaseRef = base.String()
+				baseCommit, _ = repo.CommitObject(plumbing.NewHash(c.BaseRef))
 			}
 		} else {
 			baseCommit, err = repo.CommitObject(baseHash)
@@ -49,15 +50,15 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
 	}

 	var headCommit *object.Commit
-	if len(headRef) > 0 {
-		headHash := plumbing.NewHash(headRef)
-		if !plumbing.IsHash(headRef) {
-			head, err := git.TryAdditionalBaseRefs(repo, headRef)
+	if len(c.HeadRef) > 0 {
+		headHash := plumbing.NewHash(c.HeadRef)
+		if !plumbing.IsHash(c.HeadRef) {
+			head, err := git.TryAdditionalBaseRefs(repo, c.HeadRef)
 			if err != nil {
 				return errors.WrapPrefix(err, "unable to resolve head ref", 0)
 			} else {
-				headRef = head.String()
-				headCommit, _ = repo.CommitObject(plumbing.NewHash(baseRef))
+				c.HeadRef = head.String()
+				headCommit, _ = repo.CommitObject(plumbing.NewHash(c.BaseRef))
 			}
 		} else {
 			headCommit, err = repo.CommitObject(headHash)
@@ -67,23 +68,23 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
 		}
 	}

-	// If baseCommit is an ancestor of headCommit, update baseRef to be the common ancestor.
+	// If baseCommit is an ancestor of headCommit, update c.BaseRef to be the common ancestor.
 	if headCommit != nil && baseCommit != nil {
 		mergeBase, err := headCommit.MergeBase(baseCommit)
 		if err != nil || len(mergeBase) < 1 {
 			return errors.WrapPrefix(err, "could not find common base between the given references", 0)
 		}
-		baseRef = mergeBase[0].Hash.String()
+		c.BaseRef = mergeBase[0].Hash.String()
 	}

-	if maxDepth != 0 {
-		opts = append(opts, git.ScanOptionMaxDepth(int64(maxDepth)))
+	if c.MaxDepth != 0 {
+		opts = append(opts, git.ScanOptionMaxDepth(int64(c.MaxDepth)))
 	}
-	if baseRef != "" {
-		opts = append(opts, git.ScanOptionBaseHash(baseRef))
+	if c.BaseRef != "" {
+		opts = append(opts, git.ScanOptionBaseHash(c.BaseRef))
 	}
-	if headRef != "" {
-		opts = append(opts, git.ScanOptionHeadCommit(headRef))
+	if c.HeadRef != "" {
+		opts = append(opts, git.ScanOptionHeadCommit(c.HeadRef))
 	}
 	scanOptions := git.NewScanOptions(opts...)

@@ -106,7 +107,7 @@ func (e *Engine) ScanGit(ctx context.Context, repoPath, headRef, baseRef string,
 	e.sourcesWg.Add(1)
 	go func() {
 		defer e.sourcesWg.Done()
-		err := gitSource.ScanRepo(ctx, repo, repoPath, scanOptions, e.ChunksChan())
+		err := gitSource.ScanRepo(ctx, repo, c.RepoPath, scanOptions, e.ChunksChan())
 		if err != nil {
 			logrus.WithError(err).Fatal("could not scan repo")
 		}
@@ -8,6 +8,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )

@@ -56,7 +57,16 @@ func TestGitEngine(t *testing.T) {
 			WithDecoders(decoders.DefaultDecoders()...),
 			WithDetectors(false, DefaultDetectors()...),
 		)
-		e.ScanGit(ctx, path, tTest.branch, tTest.base, tTest.maxDepth, tTest.filter)
+		cfg := sources.Config{
+			RepoPath: path,
+			HeadRef:  tTest.branch,
+			BaseRef:  tTest.base,
+			MaxDepth: tTest.maxDepth,
+			Filter:   tTest.filter,
+		}
+		if err := e.ScanGit(ctx, cfg); err != nil {
+			return
+		}
 		go e.Finish()
 		resultCount := 0
 		for result := range e.ResultsChan() {
@@ -104,7 +114,13 @@ func BenchmarkGitEngine(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		// TODO: this is measuring the time it takes to initialize the source
 		// and not to do the full scan
-		e.ScanGit(ctx, path, "", "", 0, common.FilterEmpty())
+		cfg := sources.Config{
+			RepoPath: path,
+			Filter:   common.FilterEmpty(),
+		}
+		if err := e.ScanGit(ctx, cfg); err != nil {
+			return
+		}
 	}
 	e.Finish()
 }
@@ -7,34 +7,35 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"

-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
 )

-func (e *Engine) ScanGitHub(ctx context.Context, endpoint string, repos, orgs []string, token string, includeForks bool, filter *common.Filter, concurrency int, includeMembers bool) error {
+// ScanGitHub scans Github with the provided options.
+func (e *Engine) ScanGitHub(ctx context.Context, c sources.Config) error {
 	source := github.Source{}
 	connection := sourcespb.GitHub{
-		Endpoint:      endpoint,
-		Organizations: orgs,
-		Repositories:  repos,
-		ScanUsers:     includeMembers,
+		Endpoint:      c.Endpoint,
+		Organizations: c.Orgs,
+		Repositories:  c.Repos,
+		ScanUsers:     c.IncludeMembers,
 	}
-	if len(token) > 0 {
+	if len(c.Token) > 0 {
 		connection.Credential = &sourcespb.GitHub_Token{
-			Token: token,
+			Token: c.Token,
 		}
 	} else {
 		connection.Credential = &sourcespb.GitHub_Unauthenticated{}
 	}
-	connection.IncludeForks = includeForks
+	connection.IncludeForks = c.IncludeForks
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, &connection, proto.MarshalOptions{})
 	if err != nil {
 		logrus.WithError(err).Error("failed to marshal github connection")
 		return err
 	}
-	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, concurrency)
+	err = source.Init(ctx, "trufflehog - github", 0, 0, false, &conn, c.Concurrency)
 	if err != nil {
 		logrus.WithError(err).Error("failed to initialize github source")
 		return err
@@ -6,31 +6,34 @@ import (

 	"github.com/go-errors/errors"
 	"github.com/sirupsen/logrus"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
 	"golang.org/x/net/context"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/gitlab"
 )

-func (e *Engine) ScanGitLab(ctx context.Context, endpoint, token string, repositories []string) error {
+// ScanGitLab scans GitLab with the provided configuration.
+func (e *Engine) ScanGitLab(ctx context.Context, c sources.Config) error {
 	connection := &sourcespb.GitLab{}

 	switch {
-	case len(token) > 0:
+	case len(c.Token) > 0:
 		connection.Credential = &sourcespb.GitLab_Token{
-			Token: token,
+			Token: c.Token,
 		}
 	default:
 		return fmt.Errorf("must provide token")
 	}

-	if len(endpoint) > 0 {
-		connection.Endpoint = endpoint
+	if len(c.Endpoint) > 0 {
+		connection.Endpoint = c.Endpoint
 	}

-	if len(repositories) > 0 {
-		connection.Repositories = repositories
+	if len(c.Repos) > 0 {
+		connection.Repositories = c.Repos
 	}

 	var conn anypb.Any
@@ -7,33 +7,36 @@ import (

 	"github.com/go-errors/errors"
 	"github.com/sirupsen/logrus"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"
 )

-func (e *Engine) ScanS3(ctx context.Context, key, secret string, cloudCred bool, buckets []string) error {
+// ScanS3 scans S3 buckets.
+func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error {
 	connection := &sourcespb.S3{
 		Credential: &sourcespb.S3_Unauthenticated{},
 	}
-	if cloudCred {
-		if len(key) > 0 || len(secret) > 0 {
+	if c.CloudCred {
+		if len(c.Key) > 0 || len(c.Secret) > 0 {
 			return fmt.Errorf("cannot use cloud credentials and basic auth together")
 		}
 		connection.Credential = &sourcespb.S3_CloudEnvironment{}
 	}
-	if len(key) > 0 && len(secret) > 0 {
+	if len(c.Key) > 0 && len(c.Secret) > 0 {
 		connection.Credential = &sourcespb.S3_AccessKey{
 			AccessKey: &credentialspb.KeySecret{
-				Key:    key,
-				Secret: secret,
+				Key:    c.Key,
+				Secret: c.Secret,
 			},
 		}
 	}
-	if len(buckets) > 0 {
-		connection.Buckets = buckets
+	if len(c.Buckets) > 0 {
+		connection.Buckets = c.Buckets
 	}
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
@@ -10,24 +10,26 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/syslog"
 )

-func (e *Engine) ScanSyslog(ctx context.Context, address, protocol, certPath, keyPath, format string, concurrency int) error {
+// ScanSyslog is a source that scans syslog files.
+func (e *Engine) ScanSyslog(ctx context.Context, c sources.Config) error {
 	connection := &sourcespb.Syslog{
-		Protocol:      protocol,
-		ListenAddress: address,
-		Format:        format,
+		Protocol:      c.Protocol,
+		ListenAddress: c.Address,
+		Format:        c.Format,
 	}

-	if certPath != "" && keyPath != "" {
-		cert, err := os.ReadFile(certPath)
+	if c.CertPath != "" && c.KeyPath != "" {
+		cert, err := os.ReadFile(c.CertPath)
 		if err != nil {
 			return errors.WrapPrefix(err, "could not open TLS cert file", 0)
 		}
 		connection.TlsCert = string(cert)

-		key, err := os.ReadFile(keyPath)
+		key, err := os.ReadFile(c.KeyPath)
 		if err != nil {
 			return errors.WrapPrefix(err, "could not open TLS key file", 0)
 		}
@@ -40,7 +42,7 @@ func (e *Engine) ScanSyslog(ctx context.Context, address, protocol, certPath, ke
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 	source := syslog.Source{}
-	err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, concurrency)
+	err = source.Init(ctx, "trufflehog - syslog", 0, 0, false, &conn, c.Concurrency)
 	source.InjectConnection(connection)
 	if err != nil {
 		logrus.WithError(err).Error("failed to initialize syslog source")
@@ -0,0 +1,240 @@
+//package main
+package gitparse
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"log"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// DateFormat is the standard date format for git.
+const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"
+
+// Commit contains commit header info and diffs.
+type Commit struct {
+	Hash    string
+	Author  string
+	Date    time.Time
+	Message strings.Builder
+	Diffs   []Diff
+}
+
+// Diff contains the info about a file diff in a commit.
+type Diff struct {
+	PathA     string
+	PathB     string
+	LineStart int
+	Content   bytes.Buffer
+	IsBinary  bool
+}
+
+/*
+func main() {
+	source := "/home/hrich/go/src/github.com/hashicorp/terraform"
+	start := time.Now()
+	fileChan := BillParser(source)
+	for range fileChan {
+	}
+	newTime := time.Now().Sub(start)
+
+	start = time.Now()
+	errChan := make(chan error)
+	fileChan2, _ := glgo.GitLog(source, glgo.LogOpts{}, errChan)
+	for range fileChan2 {
+	}
+	oldTime := time.Now().Sub(start)
+
+	fmt.Printf("Gitleaks parser: %d seconds\nNew parser: %d seconds", oldTime.Seconds(), newTime.Seconds())
+
+}
+*/
+
+func BillParser(source string) chan Commit {
+	cmd := exec.Command("git", "-C", source, "log", "-p", "-U0", "--full-history", "--all", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z")
+
+	stdOut, err := cmd.StdoutPipe()
+	if err != nil {
+		errs(err)
+	}
+	// TODO: Handle errors
+	// stdErr, err := cmd.StderrPipe()
+	// if err != nil {
+	// 	errs(err)
+	// }
+
+	err = cmd.Start()
+	if err != nil {
+		errs(err)
+	}
+
+	outReader := bufio.NewReader(stdOut)
+	var currentCommit *Commit
+	var currentDiff *Diff
+
+	commitChan := make(chan Commit)
+
+	// TODO: Something about nil handling maybe? I don't remember
+	go func() {
+		for {
+			line, _, err := outReader.ReadLine()
+			if err != nil {
+				fmt.Println(err)
+				break
+			}
+			logrus.Debugf("Line: %s", line)
+			switch {
+			case isCommitLine(line):
+				logrus.Debugf("commitline: %s", line)
+				if currentCommit != nil {
+					commitChan <- *currentCommit
+				}
+				currentCommit = &Commit{
+					Message: strings.Builder{},
+				}
+				currentCommit.Hash = string(line[7:])
+			case isAuthorLine(line):
+				currentCommit.Author = string(line[8:])
+			case isDateLine(line):
+				date, err := time.Parse(DateFormat, strings.TrimSpace(string(line[6:])))
+				if err != nil {
+					errs(err)
+				}
+				currentCommit.Date = date
+			case isDiffLine(line):
+				if currentDiff != nil {
+					currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+				}
+				currentDiff = &Diff{}
+			case isModeLine(line):
+			case isIndexLine(line):
+			case isPlusFileLine(line):
+				currentDiff.PathB = string(line[4:])
+			case isMinusFileLine(line):
+				currentDiff.PathA = string(line[4:])
+			case isPlusDiffLine(line):
+				currentDiff.Content.Write(line[1:])
+				currentDiff.Content.Write([]byte("\n"))
+			case isMinusDiffLine(line):
+			case isMessageLine(line):
+				currentCommit.Message.Write(line[4:])
+			case isBinaryLine(line):
+				currentDiff.IsBinary = true
+				currentDiff.PathB = pathFromBinaryLine(line, source)
+			}
+
+		}
+		if currentDiff != nil && currentDiff.Content.Len() > 0 {
+			currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+		}
+		if currentCommit != nil {
+			commitChan <- *currentCommit
+		}
+		cmd.Wait()
+		close(commitChan)
+	}()
+	return commitChan
+}
+
+func isDateLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:5], []byte("Date:")) {
+		return true
+	}
+	return false
+}
+
+func isAuthorLine(line []byte) bool {
+	if len(line) > 8 && bytes.Equal(line[:7], []byte("Author:")) {
+		return true
+	}
+	return false
+}
+
+func isCommitLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:6], []byte("commit")) {
+		return true
+	}
+	return false
+}
+
+func isDiffLine(line []byte) bool {
+	if len(line) > 5 && bytes.Equal(line[:4], []byte("diff")) {
+		return true
+	}
+	return false
+}
+
+func isIndexLine(line []byte) bool {
+	if len(line) > 6 && bytes.Equal(line[:5], []byte("index")) {
+		return true
+	}
+	return false
+}
+
+func isModeLine(line []byte) bool {
+	if len(line) > 8 && bytes.Equal(line[:8], []byte("new mode")) {
+		return true
+	}
+	return false
+}
+
+func isMinusFileLine(line []byte) bool {
+	if len(line) > 3 && bytes.Equal(line[:3], []byte("---")) {
+		return true
+	}
+	return false
+}
+
+func isPlusFileLine(line []byte) bool {
+	if len(line) > 3 && bytes.Equal(line[:3], []byte("+++")) {
+		return true
+	}
+	return false
+}
+
+func isPlusDiffLine(line []byte) bool {
+	if len(line) > 1 && bytes.Equal(line[:1], []byte("+")) {
+		return true
+	}
+	return false
+}
+
+func isMinusDiffLine(line []byte) bool {
+	if len(line) > 1 && bytes.Equal(line[:1], []byte("-")) {
+		return true
+	}
+	return false
+}
+
+func isMessageLine(line []byte) bool {
+	if len(line) > 4 && bytes.Equal(line[:4], []byte("    ")) {
+		return true
+	}
+	return false
+}
+
+func isBinaryLine(line []byte) bool {
+	if len(line) > 7 && bytes.Equal(line[:6], []byte("Binary")) {
+		return true
+	}
+	return false
+}
+
+func pathFromBinaryLine(line []byte, source string) string {
+	sbytes := bytes.Split(line, []byte(" and "))
+	if len(sbytes) != 2 {
+		logrus.Errorf("Expected binary line to be in 'Binary files a/filaA and b/fileB differ' format. Got: %s", line)
+		return ""
+	}
+	bRaw := sbytes[1]
+	return string(bRaw[2 : len(bRaw)-7]) // drop the "b/" and " differ"
+}
+
+func errs(err error) {
+	log.Println(err)
+}
@@ -0,0 +1,30 @@
+package gitparse
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	glgo "github.com/zricethezav/gitleaks/v8/detect/git"
+)
+
+func TestMain(t *testing.T) {
+	source := "/home/hrich/go/src/github.com/hashicorp/terraform"
+	start := time.Now()
+	commitChan := BillParser(source)
+	for _ = range commitChan {
+	}
+	fmt.Printf("BillParser took %f seconds.", time.Now().Sub(start).Seconds())
+	start = time.Now()
+	logOpts := glgo.LogOpts{
+		DisableSafeDir: true,
+	}
+	errChan := make(chan error)
+	fileChan, err := glgo.GitLog(source, logOpts, errChan)
+	if err != nil {
+		//errs(err)
+	}
+	for _ = range fileChan {
+	}
+	fmt.Printf("GitLeaksParser took %f seconds.", time.Now().Sub(start).Seconds())
+}
@@ -0,0 +1,159 @@
+package handlers
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/mholt/archiver/v4"
+	log "github.com/sirupsen/logrus"
+)
+
+type ctxKey int
+
+const (
+	depthKey ctxKey = iota
+)
+
+var (
+	maxDepth = 5
+)
+
+// Archive is a handler for extracting and decompressing archives.
+type Archive struct {
+	maxSize int
+	size    int
+}
+
+// New sets a default maximum size and current size counter.
+func (d *Archive) New() {
+	d.maxSize = 20 * 1024 * 1024 // 20MB
+	d.size = 0
+}
+
+// FromFile extracts the files from an archive.
+func (d *Archive) FromFile(data io.Reader) chan ([]byte) {
+	ctx := context.Background()
+	archiveChan := make(chan ([]byte), 512)
+	go func() {
+		defer close(archiveChan)
+		err := d.openArchive(ctx, 0, data, archiveChan)
+		if err != nil {
+			if errors.Is(err, archiver.ErrNoMatch) {
+				return
+			}
+			log.WithError(err).Debug("Error unarchiving chunk.")
+		}
+	}()
+	return archiveChan
+}
+
+// openArchive takes a reader and extracts the contents up to the maximum depth.
+func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan ([]byte)) error {
+	if depth >= maxDepth {
+		return fmt.Errorf("max archive depth reached")
+	}
+	format, reader, err := archiver.Identify("", reader)
+	if err != nil {
+		if errors.Is(err, archiver.ErrNoMatch) && depth > 0 {
+			chunkSize := 10 * 1024
+			for {
+				chunk := make([]byte, chunkSize)
+				n, _ := reader.Read(chunk)
+				archiveChan <- chunk
+				if n < chunkSize {
+					break
+				}
+			}
+			return nil
+		}
+		return err
+	}
+	switch archive := format.(type) {
+	case archiver.Extractor:
+		err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
+		if err != nil {
+			return err
+		}
+		return nil
+	case archiver.Decompressor:
+		compReader, err := archive.OpenReader(reader)
+		if err != nil {
+			return err
+		}
+		fileBytes, err := d.ReadToMax(compReader)
+		if err != nil {
+			return err
+		}
+		newReader := bytes.NewReader(fileBytes)
+		return d.openArchive(ctx, depth+1, newReader, archiveChan)
+	}
+	return fmt.Errorf("Unknown archive type: %s", format.Name())
+}
+
+// IsFiletype returns true if the provided reader is an archive.
+func (d *Archive) IsFiletype(reader io.Reader) (io.Reader, bool) {
+	format, readerB, err := archiver.Identify("", reader)
+	if err != nil {
+		return readerB, false
+	}
+	switch format.(type) {
+	case archiver.Extractor:
+		return readerB, true
+	case archiver.Decompressor:
+		return readerB, true
+	}
+	return readerB, false
+}
+
+// extractorHandler is applied to each file in an archiver.Extractor file.
+func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error {
+	return func(ctx context.Context, f archiver.File) error {
+		log.WithField("filename", f.Name()).Trace("Handling extracted file.")
+		depth := 0
+		if ctxDepth, ok := ctx.Value(depthKey).(int); ok {
+			depth = ctxDepth
+		}
+
+		fReader, err := f.Open()
+		if err != nil {
+			return err
+		}
+		fileBytes, err := d.ReadToMax(fReader)
+		if err != nil {
+			return err
+		}
+		fileContent := bytes.NewReader(fileBytes)
+
+		err = d.openArchive(ctx, depth, fileContent, archiveChan)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+}
+
+// ReadToMax reads up to the max size.
+func (d *Archive) ReadToMax(reader io.Reader) ([]byte, error) {
+	fileContent := bytes.Buffer{}
+	log.Tracef("Remaining buffer capacity: %d", d.maxSize-d.size)
+	for i := 0; i <= d.maxSize/512; i++ {
+		fileChunk := make([]byte, 512)
+		bRead, err := reader.Read(fileChunk)
+		if err != nil && !errors.Is(err, io.EOF) {
+			return []byte{}, err
+		}
+		d.size += bRead
+		fileContent.Write(fileChunk[0:bRead])
+		if bRead < 512 {
+			break
+		}
+		if d.size >= d.maxSize && bRead == 512 {
+			log.Debug("Max archive size reached.")
+			break
+		}
+	}
+	return fileContent.Bytes(), nil
+}
@@ -0,0 +1,85 @@
+package handlers
+
+import (
+	"net/http"
+	"regexp"
+	"testing"
+)
+
+func TestArchiveHandler(t *testing.T) {
+	tests := map[string]struct {
+		archiveURL     string
+		expectedChunks int
+		matchString    string
+	}{
+		"gzip-single": {
+			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/one-zip.gz",
+			1,
+			"AKIAYVP4CIPPH5TNP3SW",
+		},
+		"gzip-nested": {
+			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/double-zip.gz",
+			1,
+			"AKIAYVP4CIPPH5TNP3SW",
+		},
+		"gzip-too-deep": {
+			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/six-zip.gz",
+			0,
+			"",
+		},
+		"tar-single": {
+			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/one.tar",
+			1,
+			"AKIAYVP4CIPPH5TNP3SW",
+		},
+		"tar-nested": {
+			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/two.tar",
+			1,
+			"AKIAYVP4CIPPH5TNP3SW",
+		},
+		"tar-too-deep": {
+			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/six.tar",
+			0,
+			"",
+		},
+		"targz-single": {
+			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/tar-archive.tar.gz",
+			1,
+			"AKIAYVP4CIPPH5TNP3SW",
+		},
+		"gzip-large": {
+			"https://raw.githubusercontent.com/bill-rich/bad-secrets/master/FifteenMB.gz",
+			1543,
+			"AKIAYVP4CIPPH5TNP3SW",
+		},
+	}
+
+	for name, testCase := range tests {
+		resp, err := http.Get(testCase.archiveURL)
+		if err != nil || resp.StatusCode != http.StatusOK {
+			t.Error(err)
+		}
+		defer resp.Body.Close()
+
+		archive := Archive{}
+		archive.New()
+
+		archiveChan := archive.FromFile(resp.Body)
+
+		count := 0
+		re := regexp.MustCompile(testCase.matchString)
+		matched := false
+		for chunk := range archiveChan {
+			count++
+			if re.Match(chunk) {
+				matched = true
+			}
+		}
+		if !matched && len(testCase.matchString) > 0 {
+			t.Errorf("%s: Expected string not found in archive.", name)
+		}
+		if count != testCase.expectedChunks {
+			t.Errorf("%s: Unexpected number of chunks. Got %d, expected: %d", name, count, testCase.expectedChunks)
+		}
+	}
+}
@@ -0,0 +1,38 @@
+package handlers
+
+import (
+	"io"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func DefaultHandlers() []Handler {
+	return []Handler{
+		&Archive{},
+	}
+}
+
+type Handler interface {
+	FromFile(io.Reader) chan ([]byte)
+	IsFiletype(io.Reader) (io.Reader, bool)
+	New()
+}
+
+func HandleFile(file io.Reader, chunkSkel *sources.Chunk, chunksChan chan (*sources.Chunk)) bool {
+	for _, handler := range DefaultHandlers() {
+		handler.New()
+		var isType bool
+		file, isType = handler.IsFiletype(file)
+		if !isType {
+			continue
+		}
+		handlerChan := handler.FromFile(file)
+		for data := range handlerChan {
+			chunk := *chunkSkel
+			chunk.Data = data
+			chunksChan <- &chunk
+		}
+		return true
+	}
+	return false
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Bill Rich	0a3679671a	junk	2022-08-22 08:46:44 -07:00
Bill Rich	3b49183caf	wip	2022-08-16 08:04:17 -07:00
Bill Rich	809b35a843	wip	2022-08-16 07:54:10 -07:00
Bill Rich	6dd13a0b7c	wip	2022-08-16 07:36:03 -07:00
Bill Rich	bd1539d22a	wip	2022-08-15 13:12:59 -07:00
ahrav	2cab951ee5	[Thog-628] update detector results hash v2 (#710 ) * Start updating detectors that have two part creds to record the raw result as ID + secret. * Add more detectors. * More detectors. * More detectors. * remove comment out imports.	2022-08-12 14:53:37 -07:00
ahrav	73f9d3f0a0	[chore] - Use config struct instead of pointer for engine scans. (#709 ) * Use a config struct instead of pointer when scanning engine sources. * use config.	2022-08-12 09:56:24 -07:00
Miccah	2bd17052bb	[OC-40] Document pre-commit hook (#708 )	2022-08-11 22:19:15 -05:00
Bill Rich	4a93e49eea	Support scanning binary files in git sources (#684 ) * Scan binary files for git sources * Create data chunks in for loop * Linter feedback and newline commit result * Use disk buffered reader and chunker function	2022-08-10 16:10:45 -07:00
Bill Rich	a473b9aa99	Use re-readable reader and common chunker (#703 ) * Use re-readable reader and common chunker * Linter feedback * Break on error	2022-08-10 15:32:49 -07:00
ahrav	dcc102a81c	[Thog-371] Utilize config struct for engine scans (#700 ) * Use a config struct when scanning and engine source. * fix tests. * Move test_helpers to the sources pkg. * Handle ScanGit error in tests. * adderss comments. * Use functional options. * Remove temp var. * Add better var names for the setup functions for each config. * Remove unused var. * fix error logs. * fix error logs. * single line. * remove blank lines.	2022-08-10 10:11:13 -07:00
ahrav	30ebe84e3e	[THOG-608] - Fix linter errors. (#701 ) * Fix linter errors. * Fix gist adding test. * Update test string for mock JSON reply. * Remove if.	2022-08-09 19:20:02 -07:00
Dustin Decker	8826e369cf	AWS detector should detect with leading '+' (#698 )	2022-08-08 09:30:08 -07:00
ahrav	dfb7e9a405	Add hash v2 to results proto. (#693 )	2022-08-04 16:37:15 -07:00
Dustin Decker	d8ed02643a	Fix goreleaser dockerfile (#691 ) * copy entrypoint into goreleaser build * add test release make target	2022-08-03 13:56:44 -07:00
Sergej	b081766d9b	Include `extra_args` action args in readme. (#689 )	2022-08-03 08:52:41 -07:00
Max Thomson	b0a58fcf72	Update "Total Dectectors" Badge (#688 )	2022-08-02 21:14:36 -07:00
Bill Rich	7273dc9058	Archive decoder (#683 ) * Archive decoder * Fix reader handling * Seek error handling * Add tests * Fix extra empty chunk * Sync chunk size	2022-08-02 20:36:21 -07:00
Dustin Decker	047e2b4607	Copy entrypoint changes into goreleaser Dockerfile build to fix github action (#686 )	2022-08-02 12:50:26 -07:00
dependabot[bot]	29f12d2d35	Bump google.golang.org/protobuf from 1.28.0 to 1.28.1 (#681 ) Bumps [google.golang.org/protobuf](https://github.com/protocolbuffers/protobuf-go) from 1.28.0 to 1.28.1. - [Release notes](https://github.com/protocolbuffers/protobuf-go/releases) - [Changelog](https://github.com/protocolbuffers/protobuf-go/blob/master/release.bash) - [Commits](https://github.com/protocolbuffers/protobuf-go/compare/v1.28.0...v1.28.1) --- updated-dependencies: - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2022-08-02 11:53:33 -07:00