Update README.md

Update README for GH Comments Scanning
add line to link for azure repos. (#1801 )
2023-09-22 14:25:39 -04:00 · 2023-09-22 14:24:20 -04:00 · 2023-09-21 16:07:11 -07:00 · 2023-09-21 15:16:00 -07:00 · 2023-09-21 10:32:40 -07:00 · 2023-09-21 13:26:10 -04:00
60 changed files with 1752 additions and 799 deletions
@@ -0,0 +1,46 @@
+name: detector test aggregation
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 8 * * *"
+
+jobs:
+  test-detectors:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: "read"
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - id: "auth"
+        uses: "google-github-actions/auth@v1"
+        with:
+          workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
+          service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
+      - name: Test
+        run: make test-integration
+      - name: Set up gotestsum
+        run: |
+          go install gotest.tools/gotestsum@latest
+          mkdir -p tmp/test-results
+      - name: Test
+        run: |
+          CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=detectors -timeout=15m $(go list ./... | grep pkg/detectors)
+      - name: Upload test results to BuildPulse for flaky test detection
+        if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
+        uses: buildpulse/buildpulse-action@main
+        with:
+          account: 79229934
+          repository: 694446374
+          path: |
+            tmp/test-results/*.xml
+          key: ${{ secrets.BUILDPULSE_DETECTORS_ACCESS_KEY_ID }}
+          secret: ${{ secrets.BUILDPULSE_DETECTORS_SECRET_ACCESS_KEY }}
+          tags: detectors
@@ -1,4 +1,3 @@
-
 name: Test

 on:
@@ -14,57 +13,57 @@ jobs:
    if: ${{ ! github.event.pull_request.head.repo.fork }}
    runs-on: ubuntu-latest
    permissions:
-      actions: 'read'
-      contents: 'read'
-      id-token: 'write'
+      actions: "read"
+      contents: "read"
+      id-token: "write"
    steps:
-    - name: Install Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.21'
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - id: 'auth'
-      uses: 'google-github-actions/auth@v1'
-      with:
-        workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
-        service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
-    - name: Test
-      run: make test-integration
-  test-detectors:
-    if: ${{ ! github.event.pull_request.head.repo.fork }}
-    runs-on: ubuntu-latest
-    permissions:
-      actions: 'read'
-      contents: 'read'
-      id-token: 'write'
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.21'
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - id: 'auth'
-      uses: 'google-github-actions/auth@v1'
-      with:
-        workload_identity_provider: 'projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider'
-        service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
-    - name: Test
-      run: make test-detectors
-      continue-on-error: true
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - id: "auth"
+        uses: "google-github-actions/auth@v1"
+        with:
+          workload_identity_provider: "projects/811013774421/locations/global/workloadIdentityPools/github-pool/providers/github-provider"
+          service_account: "github-ci-external@trufflehog-testing.iam.gserviceaccount.com"
+      - name: Set up gotestsum
+        run: |
+          go install gotest.tools/gotestsum@latest
+          mkdir -p tmp/test-results
+      - name: Test
+        run: |
+          CGO_ENABLED=1 gotestsum --junitfile tmp/test-results/test.xml --raw-command -- go test -json -tags=sources $(go list ./... | grep -v /vendor/ | grep -v pkg/detectors)
+        if: ${{ success() || failure() }} # always run this step, even if there were previous errors
+      - name: Upload test results to BuildPulse for flaky test detection
+        if: ${{ !cancelled() }} # Run this step even when the tests fail. Skip if the workflow is cancelled.
+        uses: buildpulse/buildpulse-action@main
+        with:
+          account: 79229934
+          repository: 77726177
+          path: |
+            tmp/test-results/*.xml
+          key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
+          secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
+          tags: integration
+      - name: Annotate test results
+        uses: mikepenz/action-junit-report@v3
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: "tmp/test-results/*.xml"
  test-community:
    if: ${{ github.event.pull_request.head.repo.fork }}
    runs-on: ubuntu-latest
    permissions:
-      actions: 'read'
-      contents: 'read'
+      actions: "read"
+      contents: "read"
    steps:
-    - name: Install Go
-      uses: actions/setup-go@v4
-      with:
-        go-version: '1.21'
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Test
-      run: make test-community
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Test
+        run: make test-community
@@ -110,37 +110,61 @@ Expected output:
 ...
 ```

-## 4: Scan an S3 bucket for verified keys
+## 4: Scan a GitHub Repo's Issues and PR Comments
+
+Command:
+
+```bash
+trufflehog github --repo https://github.com/trufflesecurity/test_keys  --only-verified --issue-comments --pr-comments
+```
+
+Expected output:
+```
+...
+2023-09-22T18:22:10Z	info-0	trufflehog	scanning github pull request comments	{"repository": "https://github.com/trufflesecurity/test_keys.git"}
+Found verified result 🐷🔑
+Detector Type: AWS
+Decoder Type: PLAIN
+Raw result: AKIAQYLPMN5HLU6OP3FX
+Line: 1
+Link: https://github.com/trufflesecurity/test_keys/issues/3#issuecomment-1731842331
+Repository: test_keys
+Timestamp: 2023-09-22 18:06:26 +0000 UTC
+...
+```
+
+
+## 5: Scan an S3 bucket for verified keys

 ```bash
 trufflehog s3 --bucket=<bucket name> --only-verified
 ```

-## 5: Scan S3 buckets using IAM Roles
+## 6: Scan S3 buckets using IAM Roles

 ```bash
 trufflehog s3 --role-arn=<iam role arn>
 ```

-## 6: Scan a Github Repo using SSH authentication in docker
+## 7: Scan a Github Repo using SSH authentication in docker

 ```bash
 docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
 ```

-## 7: Scan individual files or directories
+## 8: Scan individual files or directories

 ```bash
 trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
 ```

-## 8: Scan GCS buckets for verified secrets.
+## 9: Scan GCS buckets for verified secrets.

 ```bash
 trufflehog gcs --project-id=<project-ID> --cloud-environment --only-verified
 ```

-## 9: Scan a Docker image for verified secrets.
+## 10: Scan a Docker image for verified secrets.

 Use the `--image` flag multiple times to scan multiple images.

@@ -9,8 +9,9 @@ replace github.com/couchbase/gocbcore/v10 => github.com/couchbase/gocbcore/v10 v

 require (
 	cloud.google.com/go/secretmanager v1.11.1
-	cloud.google.com/go/storage v1.31.0
+	cloud.google.com/go/storage v1.33.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
+	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1
 	github.com/BobuSumisu/aho-corasick v1.0.3
 	github.com/TheZeroSlave/zapsentry v1.17.0
 	github.com/aws/aws-sdk-go v1.44.83
@@ -20,7 +21,7 @@ require (
 	github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
 	github.com/bradleyfalzon/ghinstallation/v2 v2.6.0
 	github.com/charmbracelet/bubbles v0.16.1
-	github.com/charmbracelet/bubbletea v0.24.1
+	github.com/charmbracelet/bubbletea v0.24.2
 	github.com/charmbracelet/glamour v0.6.0
 	github.com/charmbracelet/lipgloss v0.7.1
 	github.com/couchbase/gocb/v2 v2.6.3
@@ -29,10 +30,10 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v1.0.2
 	github.com/fatih/color v1.15.0
 	github.com/felixge/fgprof v0.9.3
-	github.com/getsentry/sentry-go v0.22.0
+	github.com/getsentry/sentry-go v0.24.1
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.8.1
-	github.com/go-ldap/ldap/v3 v3.4.5
+	github.com/go-ldap/ldap/v3 v3.4.6
 	github.com/go-logr/logr v1.2.4
 	github.com/go-logr/zapr v1.2.4
 	github.com/go-redis/redis v6.15.9+incompatible
@@ -53,7 +54,8 @@ require (
 	github.com/launchdarkly/go-server-sdk/v6 v6.1.0
 	github.com/lib/pq v1.10.9
 	github.com/lrstanley/bubblezone v0.0.0-20221222153816-e95291e2243e
-	github.com/mattn/go-isatty v0.0.18
+	github.com/marusama/semaphore/v2 v2.5.0
+	github.com/mattn/go-isatty v0.0.19
 	github.com/mattn/go-sqlite3 v1.14.17
 	github.com/mholt/archiver/v4 v4.0.0-alpha.8
 	github.com/muesli/reflow v0.3.0
@@ -70,12 +72,12 @@ require (
 	go.mongodb.org/mongo-driver v1.12.0
 	go.uber.org/mock v0.2.0
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.12.0
+	golang.org/x/crypto v0.13.0
 	golang.org/x/exp v0.0.0-20230206171751-46f607a40771
-	golang.org/x/oauth2 v0.10.0
+	golang.org/x/oauth2 v0.12.0
 	golang.org/x/sync v0.3.0
-	golang.org/x/text v0.12.0
-	google.golang.org/api v0.131.0
+	golang.org/x/text v0.13.0
+	google.golang.org/api v0.132.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
@@ -83,7 +85,7 @@ require (
 )

 require (
-	cloud.google.com/go v0.110.2 // indirect
+	cloud.google.com/go v0.110.4 // indirect
 	cloud.google.com/go/compute v1.20.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	cloud.google.com/go/iam v1.1.0 // indirect
@@ -151,7 +153,7 @@ require (
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.4.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
@@ -168,7 +170,7 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.3.1 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20171119193500-2bcd89a1743f // indirect
@@ -203,7 +205,7 @@ require (
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/montanaflynn/stats v0.6.6 // indirect
 	github.com/mtibben/percent v0.2.1 // indirect
 	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
 	github.com/muesli/cancelreader v0.2.2 // indirect
@@ -244,16 +246,16 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
 	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/term v0.11.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.12.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect
+	google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/grpc v1.56.2 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -7,8 +7,8 @@ cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTj
 cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
 cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
 cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
-cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
+cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk=
+cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
 cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
 cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
 cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZNbg=
@@ -25,8 +25,8 @@ cloud.google.com/go/secretmanager v1.11.1 h1:cLTCwAjFh9fKvU6F13Y4L9vPcx9yiWPyWXE
 cloud.google.com/go/secretmanager v1.11.1/go.mod h1:znq9JlXgTNdBeQk9TBW/FnR/W4uChEKGeqQWAJ8SXFw=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI=
-cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0=
+cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M=
+cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8=
 dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -178,8 +178,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY=
 github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc=
-github.com/charmbracelet/bubbletea v0.24.1 h1:LpdYfnu+Qc6XtvMz6d/6rRY71yttHTP5HtrjMgWvixc=
-github.com/charmbracelet/bubbletea v0.24.1/go.mod h1:rK3g/2+T8vOSEkNHvtq40umJpeVYDn6bLaqbgzhL/hg=
+github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY=
+github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg=
 github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc=
 github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc=
 github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E=
@@ -263,13 +263,13 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
-github.com/getsentry/sentry-go v0.22.0 h1:XNX9zKbv7baSEI65l+H1GEJgSeIC1c7EN5kluWaP6dM=
-github.com/getsentry/sentry-go v0.22.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/getsentry/sentry-go v0.24.1 h1:W6/0GyTy8J6ge6lVCc94WB6Gx2ZuLrgopnn9w8Hiwuk=
+github.com/getsentry/sentry-go v0.24.1/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
-github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A=
-github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
+github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA=
+github.com/go-asn1-ber/asn1-ber v1.5.5/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
@@ -282,8 +282,8 @@ github.com/go-git/go-git/v5 v5.8.1 h1:Zo79E4p7TRk0xoRgMq0RShiTHGKcKI4+DI6BfJc/Q+
 github.com/go-git/go-git/v5 v5.8.1/go.mod h1:FHFuoD6yGz5OSKEBK+aWN9Oah0q54Jxl0abmj6GnqAo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8=
-github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs=
+github.com/go-ldap/ldap/v3 v3.4.6 h1:ert95MdbiG7aWo/oPYp9btL3KJlMPKnP58r09rI8T+A=
+github.com/go-ldap/ldap/v3 v3.4.6/go.mod h1:IGMQANNtxpsOzj7uUAMjpGBaOVTC4DYyIy8VsTdxmtc=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo=
@@ -304,6 +304,7 @@ github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
 github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
 github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
+github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@@ -382,9 +383,11 @@ github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8I
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
 github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM=
 github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -489,13 +492,15 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/marusama/semaphore/v2 v2.5.0 h1:o/1QJD9DBYOWRnDhPwDVAXQn6mQYD0gZaS1Tpx6DJGM=
+github.com/marusama/semaphore/v2 v2.5.0/go.mod h1:z9nMiNUekt/LTpTUQdpp+4sJeYqUGpwMHfW0Z8V8fnQ=
 github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
 github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
-github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
 github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -518,8 +523,9 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
-github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v64GQ=
+github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
 github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
 github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
 github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 h1:kMlmsLSbjkikxQJ1IPwaM+7LJ9ltFu/fi8CRzvSnQmA=
@@ -565,6 +571,7 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
 github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
 github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
@@ -701,8 +708,8 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -767,16 +774,16 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
-golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
-golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
+golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
+golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -835,16 +842,16 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
-golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -857,8 +864,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@@ -910,8 +917,8 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.131.0 h1:AcgWS2edQ4chVEt/SxgDKubVu/9/idCJy00tBGuGB4M=
-google.golang.org/api v0.131.0/go.mod h1:7vtkbKv2REjJbxmHSkBTBQ5LUGvPdAqjjvt84XAfhpA=
+google.golang.org/api v0.132.0 h1:8t2/+qZ26kAOGSmOiHwVycqVaDg7q3JDILrNi/Z6rvc=
+google.golang.org/api v0.132.0/go.mod h1:AeTBC6GpJnJSRJjktDcPX0QwtS8pGYZOV6MSuSCusw0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -934,12 +941,12 @@ google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs=
-google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
-google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 h1:s5YSX+ZH5b5vS9rnpGymvIyMpLRJizowqDlOuyjXnTk=
-google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o=
+google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 h1:Au6te5hbKUV8pIYWHqOUZ1pva5qK/rwbIhoXEUB9Lu8=
+google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:O9kGHb51iE/nOGvQaDUuadVYqovW56s5emA88lQnj6Y=
+google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130 h1:XVeBY8d/FaK4848myy41HBqnDwvxeV3zMZhwN1TvAMU=
+google.golang.org/genproto/googleapis/api v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:mPBs5jNgx2GuQGvFwUvVKqtn6HsUw9nP64BedgvqEsQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -66,7 +66,7 @@ type ResultWithMetadata struct {
 	// SourceMetadata contains source-specific contextual information.
 	SourceMetadata *source_metadatapb.MetaData
 	// SourceID is the ID of the source that the API uses to map secrets to specific sources.
-	SourceID int64
+	SourceID sources.SourceID
 	// SourceType is the type of Source.
 	SourceType sourcespb.SourceType
 	// SourceName is the name of the Source.
@@ -96,7 +96,7 @@ func constructRequest(secret, keyID string) (*http.Request, error) {
 		return nil, err
 	}

-	params := map[string]interface{}{
+	params := map[string]any{
 		"request": accountDetail,
 		"nonce":   time.Now().UnixNano(),
 	}
@@ -83,8 +83,8 @@ func (s Scanner) Type() detectorspb.DetectorType {

 type typeRes struct {
 	Types []struct {
-		Name       string        `json:"name"`
-		Type       interface{}   `json:"type"`
-		Parameters []interface{} `json:"parameters"`
+		Name       string `json:"name"`
+		Type       any    `json:"type"`
+		Parameters []any  `json:"parameters"`
 	} `json:"types"`
 }
@@ -9,7 +9,7 @@ import (
 	"golang.org/x/crypto/ssh"
 )

-//go:embed "rockyou-15.txt"
+//go:embed "list.txt"
 var rawCrackList []byte
 var passphrases [][]byte

@@ -21,17 +21,17 @@ var (
 	ErrUncrackable = errors.New("unable to crack encryption")
 )

-func crack(in []byte) (interface{}, error) {
+func crack(in []byte) (any, string, error) {
 	for _, passphrase := range passphrases {
 		parsed, err := ssh.ParseRawPrivateKeyWithPassphrase(in, passphrase)
 		if err != nil {
 			if errors.Is(err, x509.IncorrectPasswordError) {
 				continue
 			} else {
-				return nil, err
+				return nil, "", err
 			}
 		}
-		return parsed, nil
+		return parsed, string(passphrase), nil
 	}
-	return nil, ErrUncrackable
+	return nil, "", ErrUncrackable
 }
@@ -56,23 +56,29 @@ Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=

 func Test_crack(t *testing.T) {
 	tests := []struct {
-		name    string
-		in      []byte
-		wantErr bool
+		name             string
+		in               []byte
+		wantedPassphrase string
+		wantErr          bool
 	}{
 		{
-			name:    "crackable",
-			wantErr: false,
-			in:      testEncryptedKey,
+			name:             "crackable",
+			wantErr:          false,
+			in:               testEncryptedKey,
+			wantedPassphrase: string(testEncryptedKeyCorrectPassword),
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := crack(tt.in)
+			_, passphrase, err := crack(tt.in)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("crack() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+
+			if passphrase != string(tt.wantedPassphrase) {
+				t.Errorf("crack() passphrase = %v, want %v", passphrase, string(testEncryptedKeyCorrectPassword))
+			}
 		})
 	}
 }
@@ -5,10 +5,12 @@ import (
 	"crypto/ed25519"
 	"crypto/rsa"
 	"crypto/sha1"
+	"crypto/sha256"
 	"crypto/x509"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
-	"strings"
+	"fmt"

 	"golang.org/x/crypto/ssh"
 )
@@ -18,18 +20,8 @@ var (
 	ErrEncryptedKey = errors.New("key is encrypted")
 )

-func FingerprintPEMKey(in []byte) (string, error) {
-	parsedKey, err := ssh.ParseRawPrivateKey(in)
-	if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
-		parsedKey, err = crack(in)
-		if err != nil {
-			return "", err
-		}
-	} else if err != nil {
-		return "", err
-	}
-
-	var pubKey interface{}
+func FingerprintPEMKey(parsedKey any) (string, error) {
+	var pubKey any
 	switch privateKey := parsedKey.(type) {
 	case *rsa.PrivateKey:
 		pubKey = &privateKey.PublicKey
@@ -44,6 +36,10 @@ func FingerprintPEMKey(in []byte) (string, error) {
 		return "", ErrNotSupported
 	}

+	return fingerprintPublicKey(pubKey)
+}
+
+func fingerprintPublicKey(pubKey any) (string, error) {
 	publickeyBytes, err := x509.MarshalPKIXPublicKey(pubKey)
 	if err != nil {
 		return "", err
@@ -53,3 +49,8 @@ func FingerprintPEMKey(in []byte) (string, error) {
 	publicKeyFingerprintInHex := hex.EncodeToString(publicKeyFingerprint[:])
 	return publicKeyFingerprintInHex, nil
 }
+
+func fingerprintSSHPublicKey(pubKey ssh.PublicKey) string {
+	publicKeyFingerprint := sha256.Sum256(pubKey.Marshal())
+	return fmt.Sprintf("SHA256:%s", base64.RawStdEncoding.EncodeToString(publicKeyFingerprint[:]))
+}
@@ -6,11 +6,14 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 	"time"

+	"github.com/AzureAD/microsoft-authentication-library-for-go/apps/errors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"golang.org/x/crypto/ssh"
 )

 type Scanner struct {
@@ -53,19 +56,67 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 			Redacted:     token[0:64],
 		}

-		fingerprint, err := FingerprintPEMKey([]byte(token))
+		secret.ExtraData = make(map[string]string)
+
+		var passphrase string
+		parsedKey, err := ssh.ParseRawPrivateKey([]byte(token))
+		if err != nil && strings.Contains(err.Error(), "private key is passphrase protected") {
+			secret.ExtraData["encrypted"] = "true"
+			parsedKey, passphrase, err = crack([]byte(token))
+			if err != nil {
+				secret.VerificationError = err
+				continue
+			}
+			if passphrase != "" {
+				secret.ExtraData["cracked_encryption_passphrase"] = "true"
+			}
+		} else if err != nil {
+			// couldn't parse key, probably invalid
+			continue
+		}
+
+		fingerprint, err := FingerprintPEMKey(parsedKey)
 		if err != nil {
 			continue
 		}

 		if verify {
+			verificationErrors := []string{}
 			data, err := lookupFingerprint(fingerprint, s.IncludeExpired)
 			if err == nil {
-				secret.StructuredData = data
 				if data != nil {
 					secret.Verified = true
+					secret.ExtraData["certificate_urls"] = strings.Join(data.CertificateURLs, ", ")
 				}
+			} else {
+				verificationErrors = append(verificationErrors, err.Error())
 			}
+
+			user, err := verifyGitHubUser(parsedKey)
+			if err != nil && !errors.Is(err, errPermissionDenied) {
+				verificationErrors = append(verificationErrors, err.Error())
+			}
+			if user != nil {
+				secret.Verified = true
+				secret.ExtraData["github_user"] = *user
+			}
+
+			user, err = verifyGitLabUser(parsedKey)
+			if err != nil && !errors.Is(err, errPermissionDenied) {
+				verificationErrors = append(verificationErrors, err.Error())
+			}
+			if user != nil {
+				secret.Verified = true
+				secret.ExtraData["gitlab_user"] = *user
+			}
+
+			if !secret.Verified && len(verificationErrors) > 0 {
+				secret.VerificationError = fmt.Errorf("verification failures: %s", strings.Join(verificationErrors, ", "))
+			}
+		}
+
+		if len(secret.ExtraData) == 0 {
+			secret.ExtraData = nil
 		}

 		results = append(results, secret)
@@ -74,23 +125,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]dete
 	return results, nil
 }

-func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (data *detectorspb.StructuredData, err error) {
+type result struct {
+	CertificateURLs []string
+	GitHubUsername  string
+}
+
+func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (*result, error) {
 	req, err := http.NewRequest("GET", fmt.Sprintf("https://keychecker.trufflesecurity.com/fingerprint/%s", publicKeyFingerprintInHex), nil)
 	if err != nil {
-		return
+		return nil, err
 	}
 	res, err := client.Do(req)
 	if err != nil {
-		return
+		return nil, err
 	}
 	defer res.Body.Close()

 	results := DriftwoodResult{}
 	err = json.NewDecoder(res.Body).Decode(&results)
 	if err != nil {
-		return
+		return nil, err
 	}

+	var data *result
+
 	seen := map[string]struct{}{}
 	for _, r := range results.CertificateResults {
 		if _, ok := seen[r.CertificateFingerprint]; ok {
@@ -100,36 +158,13 @@ func lookupFingerprint(publicKeyFingerprintInHex string, includeExpired bool) (d
 			continue
 		}
 		if data == nil {
-			data = &detectorspb.StructuredData{}
+			data = &result{}
 		}
-		if data.TlsPrivateKey == nil {
-			data.TlsPrivateKey = make([]*detectorspb.TlsPrivateKey, 0)
-		}
-		data.TlsPrivateKey = append(data.TlsPrivateKey, &detectorspb.TlsPrivateKey{
-			CertificateFingerprint: r.CertificateFingerprint,
-			ExpirationTimestamp:    r.ExpirationTimestamp.Unix(),
-			VerificationUrl:        fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint),
-		})
+		data.CertificateURLs = append(data.CertificateURLs, fmt.Sprintf("https://crt.sh/?q=%s", r.CertificateFingerprint))
 		seen[r.CertificateFingerprint] = struct{}{}
 	}

-	for _, r := range results.GitHubSSHResults {
-		if _, ok := seen[r.Username]; ok {
-			continue
-		}
-		if data == nil {
-			data = &detectorspb.StructuredData{}
-		}
-		if data.GithubSshKey == nil {
-			data.GithubSshKey = make([]*detectorspb.GitHubSSHKey, 0)
-		}
-		data.GithubSshKey = append(data.GithubSshKey, &detectorspb.GitHubSSHKey{
-			User: r.Username,
-		})
-		seen[r.Username] = struct{}{}
-	}
-
-	return
+	return data, nil
 }

 type DriftwoodResult struct {
@@ -20,13 +20,15 @@ import (
 func TestPrivatekey_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
 	secretTLS := testSecrets.MustGetField("PRIVATEKEY_TLS")
 	secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
+	secretGitHubEncrypted := testSecrets.MustGetField("PRIVATEKEY_GITHUB_ENCRYPTED")
 	secretInactive := testSecrets.MustGetField("PRIVATEKEY_UNVERIFIED")
+
 	type args struct {
 		ctx    context.Context
 		data   []byte
@@ -75,19 +77,8 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgw",
-					StructuredData: &detectorspb.StructuredData{
-						TlsPrivateKey: []*detectorspb.TlsPrivateKey{
-							{
-								CertificateFingerprint: "1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
-								VerificationUrl:        "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9",
-								ExpirationTimestamp:    1497337731,
-							},
-							{
-								CertificateFingerprint: "0e9de31fb2ee16465a4d5d93b227d54f870326d1",
-								VerificationUrl:        "https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
-								ExpirationTimestamp:    1528192832,
-							},
-						},
+					ExtraData: map[string]string{
+						"certificate_urls": "https://crt.sh/?q=1e20c40deb44a8539dd3ac3e8c53b72750cb19f9, https://crt.sh/?q=0e9de31fb2ee16465a4d5d93b227d54f870326d1",
 					},
 				},
 			},
@@ -106,12 +97,8 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5v",
-					StructuredData: &detectorspb.StructuredData{
-						GithubSshKey: []*detectorspb.GitHubSSHKey{
-							{
-								User: "thisisforgithub0",
-							},
-						},
+					ExtraData: map[string]string{
+						"github_user": "sirdetectsalot",
 					},
 				},
 			},
@@ -121,47 +108,8 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 			name: "found encrypted GitHub SSH private key, verified",
 			s:    Scanner{},
 			args: args{
-				ctx: context.Background(),
-				data: []byte(`-----BEGIN OPENSSH PRIVATE KEY-----
-						b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABAjNIZuun
-						xgLkM8KuzfmQuRAAAAEAAAAAEAAAGXAAAAB3NzaC1yc2EAAAADAQABAAABgQDe3Al0EMPz
-						utVNk5DixaYrGMK56RqUoqGBinke6SWVWmqom1lBcJWzor6HlnMRPPr7YCEsJKL4IpuVwu
-						inRa5kdtNTyM7yyQTSR2xXCS0fUItNuq8pUktsH8VUggpMeew8hJv7rFA7tnIg3UXCl6iF
-						OLZKbDA5aa24idpcD8b1I9/RzTOB1fu0of5xd9vgODzGw5JvHQSJ0FaA42aNBMGwrDhDB3
-						sgnRNdWf6NNIh8KpXXMKJADf3klsyn6He8L2bPMp8a4wwys2YB35p5zQ0JURovsdewlOxH
-						NT7eP19eVf4dCreibxUmRUaob5DEoHEk8WrxjKWIYUuLeD6AfcW6oXyRU2Yy8Vrt6SqFl5
-						WAi47VMFTkDZYS/eCvG53q9UBHpCj7Qvb0vSkCZXBvBIhlw193F3PX4WvO1IXsMwvQ1D1X
-						lmomsItbqM0cJyKw6LU18QWiBHvE7BqcphaoL5E08W2ATTSRIMCp6rt4rptM7KyGK8rc6W
-						UYrCnWt6KlCA8AAAWQXk+lVx6bH5itIKKYmQr6cR/5xtZ2GHAxnYtvlW3xnGhU0MHv+lJ2
-						uoWlT2RXE5pdMUQj7rNWAMqkwifSKZs9wBfYeo1TaFDmC3nW7yHSN3XTuO78mPIW5JyvmE
-						Rj5qjsUn7fNmzECoAxnVERhwnF3KqUBEPzIAc6/7v/na9NTiiGaJPco9lvCoPWbVLN08WG
-						SuyU+0x5zc3ebzuPcYqu5/c5nmiGxhALrIhjIS0OV1mtAAFhvdMjMIHOijOzSKVCC7rRk5
-						kG9EMLNvOn/DUVSRHamw5gs2V3V+Zq2g5nYWfgq8aDSTB8XlIzOj1cz3HwfN6pfSNQ/3Qe
-						wOQfWfTWdO+JSL8aoBN5Wg8tDbgmvmbFrINsJfFfSm0wZgcHhC7Ul4U3v4c8PoNdK9HXwi
-						TKKzJ9nxLYb+vDh50cnkseu2gt0KwVpjIorxEqeK755mKPao3JmOMr6uFTQsb+g+ZNgPwl
-						nRHA4Igx+zADFj3twldnKIiRpBQ5J4acur3uQ+saanBTXgul1TiFiUGT2cnz+IiCsdPovg
-						TAMt868W5LmzpfH4Cy54JtaRC4/UuMnkTGbWgutVDnWj2stOAzsQ1YmhH5igUmc94mUL+W
-						8vQDCKpeI8n+quDS9zxTvy4L4H5Iz7OZlh0h6N13BDvCYXKcNF/ugkfxZbu8mZsZQQzXNR
-						wOrEtKoHc4AnXYNzsuHEoEyLyJxGfFRDSTLbyN9wFOS/c0k9Gjte+kQRZjBVGORE5sN6X3
-						akUnTF76RhbEc+LamrwM1h5340bwosRbR8I+UrsQdFfJBEj1ZSyMRJlMkFUNi6blt7bhyx
-						ea+Pm2A614nlYUBjw2KKzzn8N/0H2NpJjIptvDsbrx3BS/rKwOeJwavRrGnIlEzuAag4vx
-						Zb2TPVta45uz7fQP5IBl83b0BJKI5Zv/fniUeLI78W/UsZqb64YQbfRyBzFtI1T/SsCi0B
-						e0EyKMzbxtSceT1Mb8eJiVIq04Xpwez9fIUt5rSedZD8KPq8P6s0cGsR7Qmw6eXZ/dBR/a
-						s5vPhfIUmQawmnwAVuWNRdQQ79jUBSn5M+ZRVVTgEG+vFyvxr/bZqOo1JCoq5BmQhLWGRJ
-						Dk9TolbeFIVFrkuXkcu99a079ux7XSkON64oPzHrcsEzjPA1GPqs9CGBSO16wq/nI3zg+E
-						kcOCaurc9yHJJPwduem0+8WLX3WoGNfQRKurtQze2ppy8KarEtDhDd96sKkhYaqOg3GOX8
-						Yx827L4vuWSJSIqKuO2kH6kOCMUNO16piv0z/8u3CJxOGh9+4FZIop81fiFTKLhV3/gwLm
-						fzFY++KIZrLfZcUjzd80NNEja69F452Eb9HrI5BurN/PznDEi9bzM598Y7beyl4/kd4R2e
-						S7SW9/LOrGw5UgxtiU+kV8nPz1PdgxO4sRlnntSBEwkQBzMkLOpq2h2BuJ2TlMP/TWuwLQ
-						sDkv1Yk1pD0roGmtMzbujnURGxqRJ8gUmuIot4hpfyRSssvnRQQZ3lQCQCwHiE+HJxXWf5
-						c58zOMjW7o21tI8e13uUnbRoQVJM9XYqk1usPXIkYPYL9uOw3AW/Zn+cnDrsXvTK9ZxgGD
-						/90b1BNwVqMlUK+QggHNwl5qD8eoXK5cDvav66te+E+V7FYFQ06w3tytRVz8SjoaiChN02
-						muIjvl6G7Hoj1hObM2t/ZheN1EShS11z868hhS6Mx7GvIdtkXuvdiBYMiBLOshJQxB8Mzx
-						iug9W+Di3upLf0UMC1TqADGphsIHRU7RbmHQ8Rwp7dogswmDfpRSapPt9p0D+6Ad5VBzi3
-						f3BPXj76UBLMEJCrZR1P28vnAA7AyNHaLvMPlWDMG5v3V/UV+ugyFcoBAOyjiQgYST8F3e
-						Hx7UPVlTK8dyvk1Z+Yw0nrfNClI=
-						-----END OPENSSH PRIVATE KEY-----
-		`),
+				ctx:    context.Background(),
+				data:   []byte(secretGitHubEncrypted),
 				verify: true,
 			},
 			want: []detectors.Result{
@@ -169,12 +117,10 @@ func TestPrivatekey_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_PrivateKey,
 					Verified:     true,
 					Redacted:     "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAACmFl",
-					StructuredData: &detectorspb.StructuredData{
-						GithubSshKey: []*detectorspb.GitHubSSHKey{
-							{
-								User: "thisisforgithub0",
-							},
-						},
+					ExtraData: map[string]string{
+						"github_user":                   "sirdetectsalot",
+						"encrypted":                     "true",
+						"cracked_encryption_passphrase": "true",
 					},
 				},
 			},
@@ -262,7 +208,7 @@ func Test_lookupFingerprint(t *testing.T) {
 				t.Errorf("lookupFingerprint() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.TlsPrivateKey) > 0, tt.wantFingerprints) {
+			if !reflect.DeepEqual(gotFingerprints != nil && len(gotFingerprints.CertificateURLs) > 0, tt.wantFingerprints) {
 				t.Errorf("lookupFingerprint() = %v, want %v", gotFingerprints, tt.wantFingerprints)
 			}
 		})
@@ -0,0 +1,125 @@
+package privatekey
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/githubs-ssh-key-fingerprints
+var githubFingerprints = map[string]string{
+	"SHA256:uNiVztksCsDhcc0u9e8BujQXVUpKZIDTMczCvj3tD2s": "RSA",
+	"SHA256:br9IjFspm1vxR3iA35FWE+4VTyz1hYVLIE2t1/CeyWQ": "DSA - deprecated",
+	"SHA256:p2QAMXNIC1TJYWeIOttrVc98/R1BUFWu3/LiyKgUfQM": "ECDSA",
+	"SHA256:+DiY3wvvV6TuJJhbpZisF/zLDA0zPMSvHdkr4UvCOqU": "ED25519",
+}
+
+// https://docs.gitlab.com/ee/user/gitlab_com/index.html#ssh-host-keys-fingerprints
+var gitlabFingerprints = map[string]string{
+	"SHA256:HbW3g8zUjNSksFbqTiUWPWg2Bq1x8xdGUrliXFzSnUw": "ECDSA",
+	"SHA256:eUXGGm1YGsMAS7vkcx6JOJdOGHPem5gQp4taiCfCLB8": "ED25519",
+	"SHA256:ROQFvPThGrW4RuWLoL9tq9I9zJ42fK4XywyRtbOz/EQ": "RSA",
+}
+
+func firstResponseFromSSH(parsedKey any, username, hostport string) (string, error) {
+	signer, err := ssh.NewSignerFromKey(parsedKey)
+	if err != nil {
+		return "", err
+	}
+
+	// Verify the server fingerprint to ensure that there is no MITM replay attack
+	config := &ssh.ClientConfig{
+		Timeout: 3 * time.Second,
+		User:    username,
+		Auth: []ssh.AuthMethod{
+			ssh.PublicKeys(signer),
+		},
+		HostKeyCallback: func(hostname string, _ net.Addr, key ssh.PublicKey) error {
+			switch hostname {
+			case "github.com:22":
+				fingerprint := fingerprintSSHPublicKey(key)
+				if _, ok := githubFingerprints[fingerprint]; !ok {
+					return fmt.Errorf("unknown host fingerprint for github.com, got %s", fingerprint)
+				}
+			case "gitlab.com:22":
+				fingerprint := fingerprintSSHPublicKey(key)
+				if _, ok := gitlabFingerprints[fingerprint]; !ok {
+					return fmt.Errorf("unknown host fingerprint for gitlab.com, got %s", fingerprint)
+				}
+			default:
+				return errors.New("unknown host in fingerprint db")
+			}
+			return nil
+		},
+	}
+
+	client, err := ssh.Dial("tcp", hostport, config)
+	if err != nil {
+		if strings.Contains(err.Error(), "unable to authenticate") {
+			return "", errPermissionDenied
+		}
+		return "", err
+	}
+	defer client.Close()
+
+	session, err := client.NewSession()
+	if err != nil {
+		return "", err
+	}
+	defer session.Close()
+
+	var output bytes.Buffer
+	session.Stderr = &output
+
+	err = session.Shell()
+	if err != nil {
+		return "", err
+	}
+	_ = session.Wait()
+
+	return output.String(), err
+}
+
+var errPermissionDenied = errors.New("permission denied")
+
+func verifyGitHubUser(parsedKey any) (*string, error) {
+	output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.Contains(output, "Permission denied") {
+		return nil, errPermissionDenied
+	}
+
+	if strings.Contains(output, "successfully authenticated") {
+		username := strings.TrimSuffix(strings.Split(output, " ")[1], "!")
+		return &username, nil
+	}
+
+	return nil, nil
+}
+
+func verifyGitLabUser(parsedKey any) (*string, error) {
+	output, err := firstResponseFromSSH(parsedKey, "git", "gitlab.com:22")
+	if err != nil {
+		return nil, err
+	}
+
+	if strings.Contains(output, "Permission denied") {
+		return nil, errPermissionDenied
+	}
+
+	if strings.Contains(output, "Welcome to GitLab") {
+		split := strings.Split(output, " ")
+		username := strings.TrimPrefix(strings.TrimSuffix(split[len(split)-1], "!"), "@")
+		return &username, nil
+	}
+
+	return nil, nil
+}
@@ -0,0 +1,34 @@
+package privatekey
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"golang.org/x/crypto/ssh"
+)
+
+func TestFirstResponseFromSSH(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors4")
+	if err != nil {
+		t.Fatalf("could not get test secrets from GCP: %s", err)
+	}
+	secretGitHub := testSecrets.MustGetField("PRIVATEKEY_GITHUB")
+
+	parsedKey, err := ssh.ParseRawPrivateKey([]byte(normalize(secretGitHub)))
+	if err != nil {
+		t.Fatalf("could not parse test secret: %s", err)
+	}
+
+	output, err := firstResponseFromSSH(parsedKey, "git", "github.com:22")
+	if err != nil {
+		t.Fail()
+	}
+
+	if len(output) == 0 {
+		t.Fail()
+	}
+}
@@ -47,7 +47,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}

 		if verify {
-			req, err := http.NewRequestWithContext(ctx, "GET", "https://rubygems.org/api/v1/gems", nil)
+			req, err := http.NewRequestWithContext(ctx, "GET", "https://rubygems.org/api/v1/gems.json", nil)
 			if err != nil {
 				continue
 			}
@@ -56,7 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 {
+				if res.StatusCode >= 200 && res.StatusCode < 300 || res.StatusCode == http.StatusForbidden {
 					s1.Verified = true
 				} else {
 					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
@@ -2,6 +2,7 @@ package sendbirdorganizationapi

 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -11,13 +12,15 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

-type Scanner struct{}
+type Scanner struct {
+	client *http.Client
+}

 // Ensure the Scanner satisfies the interface at compile time.
 var _ detectors.Detector = (*Scanner)(nil)

 var (
-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sendbird"}) + `\b([0-9a-f]{24})\b`)
@@ -47,25 +50,33 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}

 		if verify {
+			client := s.client
+			if client == nil {
+				client = defaultClient
+			}
+
 			req, err := http.NewRequestWithContext(ctx, "GET", "https://gate.sendbird.com/api/v2/applications", nil)
 			if err != nil {
-				continue
+				s1.VerificationError = err
 			}
 			req.Header.Add("SENDBIRDORGANIZATIONAPITOKEN", resMatch)
 			res, err := client.Do(req)
-			if err == nil {
+			if err != nil {
+				s1.VerificationError = err
+			} else {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
-				} else {
-					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
-					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
-						continue
-					}
+				} else if res.StatusCode != http.StatusForbidden {
+					s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
 				}
 			}
 		}

+		// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
+		if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+			continue
+		}
 		results = append(results, s1)
 	}

@@ -9,7 +9,8 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -19,7 +20,7 @@ import (
 func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
-	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+	testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
 	if err != nil {
 		t.Fatalf("could not get test secrets from GCP: %s", err)
 	}
@@ -32,11 +33,12 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, verified",
@@ -52,7 +54,8 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 					Verified:     true,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
 		},
 		{
 			name: "found, unverified",
@@ -68,7 +71,42 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 					Verified:     false,
 				},
 			},
-			wantErr: false,
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, would be verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
+		{
+			name: "found, verified but unexpected api surface",
+			s:    Scanner{client: common.ConstantResponseHttpClient(404, "")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a sendbird secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SendbirdOrganizationAPI,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
 		},
 		{
 			name: "not found",
@@ -84,20 +122,23 @@ func TestSendbirdOrganizationAPI_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("SendbirdOrganizationAPI.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
+
 			for i := range got {
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
-				got[i].Raw = nil
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
+				}
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("SendbirdOrganizationAPI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "VerificationError")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("Sendbird.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}
@@ -1,6 +1,7 @@
 package slackwebhook

 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -69,11 +70,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if err != nil {
 					continue
 				}
-				body := string(bodyBytes)

 				defer res.Body.Close()

-				if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (strings.Contains(body, "no_text") || strings.Contains(body, "missing_text"))) {
+				if res.StatusCode >= 200 && res.StatusCode < 300 || (res.StatusCode == 400 && (bytes.Equal(bodyBytes, []byte("no_text")) || bytes.Equal(bodyBytes, []byte("missing_text")))) {
 					s1.Verified = true
 				} else if res.StatusCode == 401 || res.StatusCode == 403 {
 					// The secret is determinately not verified (nothing to do)
@@ -6,14 +6,14 @@ package slackwebhook
 import (
 	"context"
 	"fmt"
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-cmp/cmp/cmpopts"
 	"testing"
 	"time"

-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )

@@ -57,6 +57,40 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 			wantErr:             false,
 			wantVerificationErr: false,
 		},
+		{
+			name: "found, verified, 400 no_text",
+			s:    Scanner{client: common.ConstantResponseHttpClient(400, "no_text")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
+		{
+			name: "found, verified, 400 missing_text",
+			s:    Scanner{client: common.ConstantResponseHttpClient(400, "missing_text")},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     true,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: false,
+		},
 		{
 			name: "found, unverified",
 			s:    Scanner{},
@@ -120,6 +154,23 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 			wantErr:             false,
 			wantVerificationErr: true,
 		},
+		{
+			name: "found, account disabled",
+			s:    Scanner{client: common.ConstantResponseHttpClient(400, disabledAccountResponse)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a slackwebhook secret %s within", secret)),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_SlackWebhook,
+					Verified:     false,
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -144,6 +195,8 @@ func TestSlackWebhook_FromChunk(t *testing.T) {
 	}
 }

+const disabledAccountResponse = `missing_text_or_fallback_or_attachments`
+
 func BenchmarkFromData(benchmark *testing.B) {
 	ctx := context.Background()
 	s := Scanner{}
@@ -21,13 +21,13 @@ var (
 	client = common.SaneHttpClient()

 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sonarcloud"}) + `\b([0-9a-z]{40})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sonar"}) + `\b([0-9a-z]{40})\b`)
 )

 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"sonarcloud"}
+	return []string{"sonar"}
 }

 // FromData will find and optionally verify SonarCloud secrets in a given set of bytes.
@@ -15,6 +15,7 @@ import (

 type Scanner struct {
 	allowKnownTestSites bool
+	client              *http.Client
 }

 // Ensure the Scanner satisfies the interface at compile time.
@@ -23,7 +24,7 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)

-	client = common.SaneHttpClient()
+	defaultClient = common.SaneHttpClient()
 )

 // Keywords are used for efficiently pre-filtering chunks.
@@ -71,7 +72,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		// Removing the path causes possible deduplication issues if some paths have basic auth and some do not.
 		rawURL.Path = ""

-		s := detectors.Result{
+		s1 := detectors.Result{
 			DetectorType: detectorspb.DetectorType_URI,
 			Raw:          []byte(rawURL.String()),
 			RawV2:        []byte(rawURLStr),
@@ -79,27 +80,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 		}

 		if verify {
-			s.Verified = verifyURL(ctx, parsedURL)
+			if s.client == nil {
+				s.client = defaultClient
+			}
+			s1.Verified, s1.VerificationError = verifyURL(ctx, s.client, parsedURL)
 		}

-		if !s.Verified {
+		if !s1.Verified {
 			// Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.
 			if strings.HasPrefix(password, "$") {
 				continue
 			}
 		}

-		if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
+		if !s1.Verified && !s.allowKnownTestSites && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, false) {
 			continue
 		}

-		results = append(results, s)
+		results = append(results, s1)
 	}

 	return results, nil
 }

-func verifyURL(ctx context.Context, u *url.URL) bool {
+func verifyURL(ctx context.Context, client *http.Client, u *url.URL) (bool, error) {
 	// defuse most SSRF payloads
 	u.Path = strings.TrimSuffix(u.Path, "/")
 	u.RawQuery = ""
@@ -112,41 +116,41 @@ func verifyURL(ctx context.Context, u *url.URL) bool {

 	req, err := http.NewRequest("GET", credentialedURL, nil)
 	if err != nil {
-		return false
+		return false, err
 	}
 	req = req.WithContext(ctx)
 	credentialedRes, err := client.Do(req)
 	if err != nil {
-		return false
+		return false, err
 	}
 	credentialedRes.Body.Close()

 	// If the credentialed URL returns a non 2XX code, we can assume it's a false positive.
 	if credentialedRes.StatusCode < 200 || credentialedRes.StatusCode > 299 {
-		return false
+		return false, nil
 	}

 	time.Sleep(time.Millisecond * 10)

 	req, err = http.NewRequest("GET", nonCredentialedURL, nil)
 	if err != nil {
-		return false
+		return false, err
 	}
 	req = req.WithContext(ctx)
 	nonCredentialedRes, err := client.Do(req)
 	if err != nil {
-		return false
+		return false, err
 	}
 	nonCredentialedRes.Body.Close()

 	// If the non-credentialed URL returns a non 400-428 code and basic auth header, we can assume it's verified now.
 	if nonCredentialedRes.StatusCode >= 400 && nonCredentialedRes.StatusCode < 429 {
 		if nonCredentialedRes.Header.Get("WWW-Authenticate") != "" {
-			return true
+			return true, nil
 		}
 	}

-	return false
+	return false, nil
 }

 func (s Scanner) Type() detectorspb.DetectorType {
@@ -7,8 +7,10 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"

 	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
@@ -20,11 +22,12 @@ func TestURI_FromChunk(t *testing.T) {
 		verify bool
 	}
 	tests := []struct {
-		name    string
-		s       Scanner
-		args    args
-		want    []detectors.Result
-		wantErr bool
+		name                string
+		s                   Scanner
+		args                args
+		want                []detectors.Result
+		wantErr             bool
+		wantVerificationErr bool
 	}{
 		{
 			name: "found, unverified, wrong username",
@@ -77,6 +80,24 @@ func TestURI_FromChunk(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "found, verified if not for timeout",
+			s:    Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+			args: args{
+				ctx:    context.Background(),
+				data:   []byte(fmt.Sprintf("You can find a uri secret %s within", "https://httpwatch:pass@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx")),
+				verify: true,
+			},
+			want: []detectors.Result{
+				{
+					DetectorType: detectorspb.DetectorType_URI,
+					Verified:     false,
+					Redacted:     "https://httpwatch:********@www.httpwatch.com",
+				},
+			},
+			wantErr:             false,
+			wantVerificationErr: true,
+		},
 		{
 			name: "bad scheme",
 			s:    Scanner{},
@@ -101,8 +122,8 @@ func TestURI_FromChunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			s := Scanner{allowKnownTestSites: true}
-			got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+			tt.s.allowKnownTestSites = true
+			got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)
 				return
@@ -111,8 +132,13 @@ func TestURI_FromChunk(t *testing.T) {
 			// 	return
 			// }
 			for i := range got {
+				if (got[i].VerificationError != nil) != tt.wantVerificationErr {
+					t.Errorf("URI.FromData() error = %v, wantVerificationErr %v", got[i].VerificationError, tt.wantErr)
+					return
+				}
 				got[i].Raw = nil
 				got[i].RawV2 = nil
+				got[i].VerificationError = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
@@ -8,7 +8,6 @@ import (

 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci"
 )

@@ -27,17 +26,13 @@ func (e *Engine) ScanCircleCI(ctx context.Context, token string) error {
 		return err
 	}

-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - Circle CI", new(circleci.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			circleSource := circleci.Source{}
-			if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return &circleSource, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - Circle CI"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, circleci.SourceType)
+
+	circleSource := &circleci.Source{}
+	if err := circleSource.Init(ctx, "trufflehog - Circle CI", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	_, err = e.sourceManager.Run(ctx, sourceName, circleSource)
 	return err
 }
@@ -6,23 +6,18 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
 )

 // ScanDocker scans a given docker connection.
 func (e *Engine) ScanDocker(ctx context.Context, conn *anypb.Any) error {
-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - docker", new(docker.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			dockerSource := docker.Source{}
-			if err := dockerSource.Init(ctx, "trufflehog - docker", jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return &dockerSource, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - docker"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, docker.SourceType)
+
+	dockerSource := &docker.Source{}
+	if err := dockerSource.Init(ctx, sourceName, jobID, sourceID, true, conn, runtime.NumCPU()); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	_, err := e.sourceManager.Run(ctx, sourceName, dockerSource)
 	return err
 }
@@ -19,6 +19,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/output"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -239,7 +240,7 @@ func (m *Metrics) getScanDuration() time.Duration {
 func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 	logger := context.Background().Logger()
 	avgTime := map[string][]time.Duration{}
-	e.metrics.detectorAvgTime.Range(func(k, v interface{}) bool {
+	e.metrics.detectorAvgTime.Range(func(k, v any) bool {
 		key, ok := k.(string)
 		if !ok {
 			logger.Info("expected detectorAvgTime key to be a string")
@@ -532,12 +533,12 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
 	}

 	for _, res := range results {
-		e.processResult(data, res)
+		e.processResult(ctx, data, res)
 	}
 	data.wgDoneFn()
 }

-func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
+func (e *Engine) processResult(ctx context.Context, data detectableChunk, res detectors.Result) {
 	ignoreLinePresent := false
 	if SupportsLineNumbers(data.chunk.SourceType) {
 		copyChunk := data.chunk
@@ -545,8 +546,12 @@ func (e *Engine) processResult(data detectableChunk, res detectors.Result) {
 		if copyMetaData, ok := copyMetaDataClone.(*source_metadatapb.MetaData); ok {
 			copyChunk.SourceMetadata = copyMetaData
 		}
-		fragStart, mdLine := FragmentFirstLine(&copyChunk)
+		fragStart, mdLine, link := FragmentFirstLineAndLink(&copyChunk)
 		ignoreLinePresent = SetResultLineNumber(&copyChunk, &res, fragStart, mdLine)
+		if err := UpdateLink(ctx, copyChunk.SourceMetadata, link, *mdLine); err != nil {
+			ctx.Logger().Error(err, "error setting link")
+			return
+		}
 		data.chunk = copyChunk
 	}
 	if ignoreLinePresent {
@@ -600,7 +605,8 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
 		sourcespb.SourceType_SOURCE_TYPE_GERRIT,
 		sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG,
 		sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT,
-		sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
+		sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM,
+		sourcespb.SourceType_SOURCE_TYPE_AZURE_REPOS:
 		return true
 	default:
 		return false
@@ -625,27 +631,45 @@ func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64,
 	return lineNumber, false
 }

-// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
-// chunk metadata.
-func FragmentFirstLine(chunk *sources.Chunk) (int64, *int64) {
-	var fragmentStart *int64
+// FragmentFirstLineAndLink extracts the first line number and the link from the chunk metadata.
+// It returns:
+//   - The first line number of the fragment.
+//   - A pointer to the line number, facilitating direct updates.
+//   - The link associated with the fragment. This link may be updated in the chunk metadata
+//     if there's a change in the line number.
+func FragmentFirstLineAndLink(chunk *sources.Chunk) (int64, *int64, string) {
+	if chunk.SourceMetadata == nil {
+		return 0, nil, ""
+	}
+
+	var (
+		fragmentStart *int64
+		link          string
+	)
 	switch metadata := chunk.SourceMetadata.GetData().(type) {
 	case *source_metadatapb.MetaData_Git:
 		fragmentStart = &metadata.Git.Line
 	case *source_metadatapb.MetaData_Github:
 		fragmentStart = &metadata.Github.Line
+		link = metadata.Github.Link
 	case *source_metadatapb.MetaData_Gitlab:
 		fragmentStart = &metadata.Gitlab.Line
+		link = metadata.Gitlab.Link
 	case *source_metadatapb.MetaData_Bitbucket:
 		fragmentStart = &metadata.Bitbucket.Line
+		link = metadata.Bitbucket.Link
 	case *source_metadatapb.MetaData_Gerrit:
 		fragmentStart = &metadata.Gerrit.Line
 	case *source_metadatapb.MetaData_Filesystem:
 		fragmentStart = &metadata.Filesystem.Line
+		link = metadata.Filesystem.Link
+	case *source_metadatapb.MetaData_AzureRepos:
+		fragmentStart = &metadata.AzureRepos.Line
+		link = metadata.AzureRepos.Link
 	default:
-		return 0, nil
+		return 0, nil, ""
 	}
-	return *fragmentStart, fragmentStart
+	return *fragmentStart, fragmentStart, link
 }

 // SetResultLineNumber sets the line number in the provided result.
@@ -654,3 +678,33 @@ func SetResultLineNumber(chunk *sources.Chunk, result *detectors.Result, fragSta
 	*mdLine = fragStart + offset
 	return skip
 }
+
+// UpdateLink updates the link of the provided source metadata.
+func UpdateLink(ctx context.Context, metadata *source_metadatapb.MetaData, link string, line int64) error {
+	if metadata == nil {
+		return fmt.Errorf("metadata is nil when setting the link")
+	}
+
+	if link == "" {
+		ctx.Logger().V(4).Info("link is empty, skipping update")
+		return nil
+	}
+
+	newLink := giturl.UpdateLinkLineNumber(ctx, link, line)
+
+	switch meta := metadata.GetData().(type) {
+	case *source_metadatapb.MetaData_Github:
+		meta.Github.Link = newLink
+	case *source_metadatapb.MetaData_Gitlab:
+		meta.Gitlab.Link = newLink
+	case *source_metadatapb.MetaData_Bitbucket:
+		meta.Bitbucket.Link = newLink
+	case *source_metadatapb.MetaData_Filesystem:
+		meta.Filesystem.Link = newLink
+	case *source_metadatapb.MetaData_AzureRepos:
+		meta.AzureRepos.Link = newLink
+	default:
+		return fmt.Errorf("unsupported metadata type")
+	}
+	return nil
+}
@@ -11,6 +11,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
@@ -151,21 +152,27 @@ func TestDefaultDecoders(t *testing.T) {
 }

 func TestSupportsLineNumbers(t *testing.T) {
-	testCases := []struct {
-		name     string
-		input    sourcespb.SourceType
-		expected bool
+	tests := []struct {
+		name          string
+		sourceType    sourcespb.SourceType
+		expectedValue bool
 	}{
-		{"Git source type", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
-		{"Github source type", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
-		{"Gitlab source type", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
+		{"Git source", sourcespb.SourceType_SOURCE_TYPE_GIT, true},
+		{"Github source", sourcespb.SourceType_SOURCE_TYPE_GITHUB, true},
+		{"Gitlab source", sourcespb.SourceType_SOURCE_TYPE_GITLAB, true},
+		{"Bitbucket source", sourcespb.SourceType_SOURCE_TYPE_BITBUCKET, true},
+		{"Gerrit source", sourcespb.SourceType_SOURCE_TYPE_GERRIT, true},
+		{"Github unauthenticated org source", sourcespb.SourceType_SOURCE_TYPE_GITHUB_UNAUTHENTICATED_ORG, true},
+		{"Public Git source", sourcespb.SourceType_SOURCE_TYPE_PUBLIC_GIT, true},
+		{"Filesystem source", sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM, true},
+		{"Azure Repos source", sourcespb.SourceType_SOURCE_TYPE_AZURE_REPOS, true},
+		{"Unsupported type", sourcespb.SourceType_SOURCE_TYPE_BUILDKITE, false},
 	}

-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			if result := SupportsLineNumbers(tc.input); result != tc.expected {
-				t.Errorf("Expected %v for input %v, got %v", tc.expected, tc.input, result)
-			}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := SupportsLineNumbers(tt.sourceType)
+			assert.Equal(t, tt.expectedValue, result)
 		})
 	}
 }
@@ -205,3 +212,188 @@ func TestEngine_DuplicatSecrets(t *testing.T) {
 	want := uint64(5)
 	assert.Equal(t, want, e.GetMetrics().UnverifiedSecretsFound)
 }
+
+func TestFragmentFirstLineAndLink(t *testing.T) {
+	tests := []struct {
+		name         string
+		chunk        *sources.Chunk
+		expectedLine int64
+		expectedLink string
+	}{
+		{
+			name: "Test Git Metadata",
+			chunk: &sources.Chunk{
+				SourceMetadata: &source_metadatapb.MetaData{
+					Data: &source_metadatapb.MetaData_Git{
+						Git: &source_metadatapb.Git{
+							Line: 10,
+						},
+					},
+				},
+			},
+			expectedLine: 10,
+			expectedLink: "", // Git doesn't support links
+		},
+		{
+			name: "Test Github Metadata",
+			chunk: &sources.Chunk{
+				SourceMetadata: &source_metadatapb.MetaData{
+					Data: &source_metadatapb.MetaData_Github{
+						Github: &source_metadatapb.Github{
+							Line: 5,
+							Link: "https://example.github.com",
+						},
+					},
+				},
+			},
+			expectedLine: 5,
+			expectedLink: "https://example.github.com",
+		},
+		{
+			name: "Test Azure Repos Metadata",
+			chunk: &sources.Chunk{
+				SourceMetadata: &source_metadatapb.MetaData{
+					Data: &source_metadatapb.MetaData_AzureRepos{
+						AzureRepos: &source_metadatapb.AzureRepos{
+							Line: 5,
+							Link: "https://example.azure.com",
+						},
+					},
+				},
+			},
+			expectedLine: 5,
+			expectedLink: "https://example.azure.com",
+		},
+		{
+			name:         "Unsupported Type",
+			chunk:        &sources.Chunk{},
+			expectedLine: 0,
+			expectedLink: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			line, linePtr, link := FragmentFirstLineAndLink(tt.chunk)
+			assert.Equal(t, tt.expectedLink, link, "Mismatch in link")
+			assert.Equal(t, tt.expectedLine, line, "Mismatch in line")
+
+			if linePtr != nil {
+				assert.Equal(t, tt.expectedLine, *linePtr, "Mismatch in linePtr value")
+			}
+		})
+	}
+}
+
+func TestSetLink(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    *source_metadatapb.MetaData
+		link     string
+		line     int64
+		wantLink string
+		wantErr  bool
+	}{
+		{
+			name: "Github link set",
+			input: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Github{
+					Github: &source_metadatapb.Github{},
+				},
+			},
+			link:     "https://github.com/example",
+			line:     42,
+			wantLink: "https://github.com/example#L42",
+		},
+		{
+			name: "Gitlab link set",
+			input: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Gitlab{
+					Gitlab: &source_metadatapb.Gitlab{},
+				},
+			},
+			link:     "https://gitlab.com/example",
+			line:     10,
+			wantLink: "https://gitlab.com/example#L10",
+		},
+		{
+			name: "Bitbucket link set",
+			input: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Bitbucket{
+					Bitbucket: &source_metadatapb.Bitbucket{},
+				},
+			},
+			link:     "https://bitbucket.com/example",
+			line:     8,
+			wantLink: "https://bitbucket.com/example#L8",
+		},
+		{
+			name: "Filesystem link set",
+			input: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Filesystem{
+					Filesystem: &source_metadatapb.Filesystem{},
+				},
+			},
+			link:     "file:///path/to/example",
+			line:     3,
+			wantLink: "file:///path/to/example#L3",
+		},
+		{
+			name: "Azure Repos link set",
+			input: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_AzureRepos{
+					AzureRepos: &source_metadatapb.AzureRepos{},
+				},
+			},
+			link:     "https://dev.azure.com/example",
+			line:     3,
+			wantLink: "https://dev.azure.com/example?line=3",
+		},
+		{
+			name: "Unsupported metadata type",
+			input: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Git{
+					Git: &source_metadatapb.Git{},
+				},
+			},
+			link:    "https://git.example.com/link",
+			line:    5,
+			wantErr: true,
+		},
+		{
+			name:    "Metadata nil",
+			input:   nil,
+			link:    "https://some.link",
+			line:    1,
+			wantErr: true,
+		},
+	}
+
+	ctx := context.Background()
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := UpdateLink(ctx, tt.input, tt.link, tt.line)
+			if err != nil && !tt.wantErr {
+				t.Errorf("Unexpected error: %v", err)
+			}
+
+			if tt.wantErr {
+				return
+			}
+
+			switch data := tt.input.GetData().(type) {
+			case *source_metadatapb.MetaData_Github:
+				assert.Equal(t, tt.wantLink, data.Github.Link, "Github link mismatch")
+			case *source_metadatapb.MetaData_Gitlab:
+				assert.Equal(t, tt.wantLink, data.Gitlab.Link, "Gitlab link mismatch")
+			case *source_metadatapb.MetaData_Bitbucket:
+				assert.Equal(t, tt.wantLink, data.Bitbucket.Link, "Bitbucket link mismatch")
+			case *source_metadatapb.MetaData_Filesystem:
+				assert.Equal(t, tt.wantLink, data.Filesystem.Link, "Filesystem link mismatch")
+			case *source_metadatapb.MetaData_AzureRepos:
+				assert.Equal(t, tt.wantLink, data.AzureRepos.Link, "Azure Repos link mismatch")
+			}
+		})
+	}
+}
@@ -24,18 +24,14 @@ func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig)
 		return err
 	}

-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - filesystem", new(filesystem.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			fileSystemSource := filesystem.Source{}
-			fileSystemSource.WithFilter(c.Filter)
-			if err := fileSystemSource.Init(ctx, "trufflehog - filesystem", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return &fileSystemSource, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - filesystem"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, filesystem.SourceType)
+
+	fileSystemSource := &filesystem.Source{}
+	fileSystemSource.WithFilter(c.Filter)
+	if err := fileSystemSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	_, err = e.sourceManager.Run(ctx, sourceName, fileSystemSource)
 	return err
 }
@@ -44,18 +44,14 @@ func (e *Engine) ScanGCS(ctx context.Context, c sources.GCSConfig) error {
 		return fmt.Errorf("failed to marshal GCS connection: %w", err)
 	}

-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gcs", new(gcs.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			gcsSource := gcs.Source{}
-			if err := gcsSource.Init(ctx, "trufflehog - gcs", jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
-				return nil, err
-			}
-			return &gcsSource, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - gcs"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gcs.SourceType)
+
+	gcsSource := &gcs.Source{}
+	if err := gcsSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, int(c.Concurrency)); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	_, err = e.sourceManager.Run(ctx, sourceName, gcsSource)
 	return err
 }

@@ -51,21 +51,18 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) error {
 		return err
 	}

-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - git", new(git.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			gitSource := git.Source{}
-			if err := gitSource.Init(ctx, "trufflehog - git", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			gitSource.WithScanOptions(scanOptions)
-			// Don't try to clean up the provided directory. That's handled by the
-			// caller of ScanGit.
-			gitSource.WithPreserveTempDirs(true)
-			return &gitSource, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - git"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, git.SourceType)
+
+	gitSource := &git.Source{}
+	if err := gitSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	gitSource.WithScanOptions(scanOptions)
+	// Don't try to clean up the provided directory. That's handled by the
+	// caller of ScanGit.
+	gitSource.WithPreserveTempDirs(true)
+
+	_, err := e.sourceManager.Run(ctx, sourceName, gitSource)
 	return err
 }
@@ -47,18 +47,14 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) error {
 	}
 	scanOptions := git.NewScanOptions(opts...)

-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - github", new(github.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			githubSource := github.Source{}
-			if err := githubSource.Init(ctx, "trufflehog - github", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
-				return nil, err
-			}
-			githubSource.WithScanOptions(scanOptions)
-			return &githubSource, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - github"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, github.SourceType)
+
+	githubSource := &github.Source{}
+	if err := githubSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	githubSource.WithScanOptions(scanOptions)
+	_, err = e.sourceManager.Run(ctx, sourceName, githubSource)
 	return err
 }
@@ -50,18 +50,14 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) error {
 		return err
 	}

-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - gitlab", new(gitlab.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			gitlabSource := gitlab.Source{}
-			if err := gitlabSource.Init(ctx, "trufflehog - gitlab", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			gitlabSource.WithScanOptions(scanOptions)
-			return &gitlabSource, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - gitlab"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, gitlab.SourceType)
+
+	gitlabSource := &gitlab.Source{}
+	if err := gitlabSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	gitlabSource.WithScanOptions(scanOptions)
+	_, err = e.sourceManager.Run(ctx, sourceName, gitlabSource)
 	return err
 }
@@ -58,17 +58,13 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.S3Config) error {
 		return err
 	}

-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - s3", new(s3.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			s3Source := s3.Source{}
-			if err := s3Source.Init(ctx, "trufflehog - s3", jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
-				return nil, err
-			}
-			return &s3Source, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - s3"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, s3.SourceType)
+
+	s3Source := &s3.Source{}
+	if err := s3Source.Init(ctx, sourceName, jobID, sourceID, true, &conn, runtime.NumCPU()); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	_, err = e.sourceManager.Run(ctx, sourceName, s3Source)
 	return err
 }
@@ -41,18 +41,14 @@ func (e *Engine) ScanSyslog(ctx context.Context, c sources.SyslogConfig) error {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}

-	handle, err := e.sourceManager.Enroll(ctx, "trufflehog - syslog", new(syslog.Source).Type(),
-		func(ctx context.Context, jobID, sourceID int64) (sources.Source, error) {
-			syslogSource := syslog.Source{}
-			if err := syslogSource.Init(ctx, "trufflehog - syslog", jobID, sourceID, true, &conn, c.Concurrency); err != nil {
-				return nil, err
-			}
-			syslogSource.InjectConnection(connection)
-			return &syslogSource, nil
-		})
-	if err != nil {
+	sourceName := "trufflehog - syslog"
+	sourceID, jobID, _ := e.sourceManager.GetIDs(ctx, sourceName, syslog.SourceType)
+	syslogSource := &syslog.Source{}
+	if err := syslogSource.Init(ctx, sourceName, jobID, sourceID, true, &conn, c.Concurrency); err != nil {
 		return err
 	}
-	_, err = e.sourceManager.ScheduleRun(ctx, handle)
+	syslogSource.InjectConnection(connection)
+
+	_, err = e.sourceManager.Run(ctx, sourceName, syslogSource)
 	return err
 }
@@ -2,10 +2,13 @@ package giturl

 import (
 	"net/url"
+	"regexp"
 	"strconv"
 	"strings"

 	"github.com/pkg/errors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )

 type provider string
@@ -143,3 +146,43 @@ func GenerateLink(repo, commit, file string, line int64) string {
 		return baseLink
 	}
 }
+
+var linePattern = regexp.MustCompile(`L\d+`)
+
+// UpdateLinkLineNumber updates the line number in a repository link.
+// Used post-link generation to refine reported issue locations within large scanned blocks.
+func UpdateLinkLineNumber(ctx context.Context, link string, newLine int64) string {
+	parsedURL, err := url.Parse(link)
+	if err != nil {
+		ctx.Logger().Error(err, "unable to parse link to update line number", "link", link)
+		return link
+	}
+
+	switch determineProvider(link) {
+	case providerBitbucket:
+		// For Bitbucket, it doesn't support line links (based on the GenerateLink function).
+		// So we don't need to change anything.
+		return link
+
+	case providerAzure:
+		// For Azure, line numbers are appended as ?line=<number>.
+		query := parsedURL.Query()
+		query.Set("line", strconv.FormatInt(newLine, 10))
+		parsedURL.RawQuery = query.Encode()
+
+	case providerGithub, providerGitlab:
+		// If the provider name isn't one of the cloud defaults, it is probably an on-prem github or gitlab.
+		// So do the same thing.
+		fallthrough
+	default:
+		// Assumed format: .../blob/<commit>/file.go#L<number>
+		fragment := "L" + strconv.FormatInt(newLine, 10)
+		if linePattern.MatchString(parsedURL.Fragment) {
+			parsedURL.Fragment = linePattern.ReplaceAllString(parsedURL.Fragment, fragment)
+		} else {
+			parsedURL.Fragment += fragment
+		}
+	}
+
+	return parsedURL.String()
+}
@@ -4,6 +4,8 @@ import (
 	"testing"

 	"github.com/pkg/errors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 )

 func Test_NormalizeOrgRepoURL(t *testing.T) {
@@ -187,3 +189,84 @@ func TestGenerateLink(t *testing.T) {
 		})
 	}
 }
+
+func TestUpdateLinkLineNumber(t *testing.T) {
+	t.Parallel()
+
+	type args struct {
+		link    string
+		newLine int64
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "Update bitbucket, no line number supported",
+			args: args{
+				link:    "https://bitbucket.org/org/repo/blob/xyz123/main.go",
+				newLine: int64(10),
+			},
+			want: "https://bitbucket.org/org/repo/blob/xyz123/main.go",
+		},
+		{
+			name: "Update github link with line",
+			args: args{
+				link:    "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L4",
+				newLine: int64(10),
+			},
+			want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L10",
+		},
+		{
+			name: "Update Azure link with line",
+			args: args{
+				link:    "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=20",
+				newLine: int64(40),
+			},
+			want: "https://dev.azure.com/org/project/_git/repo/commit/abcdef/main.go?line=40",
+		},
+		{
+			name: "Add line to github link without line",
+			args: args{
+				link:    "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore",
+				newLine: int64(7),
+			},
+			want: "https://github.com/trufflesec-julian/confluence-go-api/blob/047b4a2ba42fc5b6c0bd535c5307434a666db5ec/.gitignore#L7",
+		},
+		{
+			name: "Update Unknown provider on-prem instance with line",
+			args: args{
+				link:    "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L30",
+				newLine: int64(50),
+			},
+			want: "https://onprem.customdomain.com/org/repo/blob/xyz123/main.go#L50",
+		},
+		{
+			name: "Update Unknown provider on-prem instance without line",
+			args: args{
+				link:    "https://onprem.customdomain.com/org/repo/commit/xyz123",
+				newLine: int64(50),
+			},
+			want: "https://onprem.customdomain.com/org/repo/commit/xyz123#L50",
+		},
+		{
+			name: "Invalid link",
+			args: args{
+				link:    "definitely not a link",
+				newLine: int64(50),
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := UpdateLinkLineNumber(context.Background(), tt.args.link, tt.args.newLine)
+			if got != tt.want && !tt.wantErr {
+				t.Errorf("UpdateLinkLineNumber() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
@@ -78,7 +78,7 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
 			if errors.Is(err, archiver.ErrNoMatch) {
 				return
 			}
-			logger.V(2).Info("Error unarchiving chunk.")
+			logger.Error(err, "error unarchiving chunk.")
 		}
 	}()
 	return archiveChan
@@ -102,6 +102,10 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
 		return a.handleNonArchiveContent(ctx, arReader, archiveChan)
 	}

+	if err != nil {
+		return err
+	}
+
 	switch archive := format.(type) {
 	case archiver.Decompressor:
 		info := decompressorInfo{depth: depth, reader: arReader, archiveChan: archiveChan, archiver: archive}
@@ -9,6 +9,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"

 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
 	"github.com/stretchr/testify/assert"
@@ -236,3 +237,37 @@ func TestExtractRPMContent(t *testing.T) {
 	expectedLength := 1822720
 	assert.Equal(t, expectedLength, len(string(content)))
 }
+
+func TestOpenInvalidArchive(t *testing.T) {
+	reader := strings.NewReader("invalid archive")
+
+	ctx := logContext.AddLogger(context.Background())
+	a := &Archive{}
+
+	archiveChan := make(chan []byte)
+
+	err := a.openArchive(ctx, 0, reader, archiveChan)
+	assert.Error(t, err)
+}
+
+func TestNestedDirArchive(t *testing.T) {
+	file, err := os.Open("testdata/dir-archive.zip")
+	assert.Nil(t, err)
+	defer file.Close()
+
+	ctx, cancel := logContext.WithTimeout(logContext.Background(), 5*time.Second)
+	defer cancel()
+	sourceChan := make(chan *sources.Chunk, 1)
+
+	go func() {
+		defer close(sourceChan)
+		HandleFile(ctx, file, &sources.Chunk{}, sourceChan)
+	}()
+
+	count := 0
+	want := 4
+	for range sourceChan {
+		count++
+	}
+	assert.Equal(t, want, count)
+}
@@ -47,42 +47,44 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
 		// an io.MultiReader, which is used by the SpecializedHandler.
 		reReader, err := diskbufferreader.New(file)
 		if err != nil {
-			aCtx.Logger().Error(err, "error creating re-reader reader")
+			aCtx.Logger().Error(err, "error creating reusable reader")
 			return false
 		}
-		defer reReader.Close()

-		// Check if the handler implements SpecializedHandler and process accordingly.
-		if specialHandler, ok := h.(SpecializedHandler); ok {
-			file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
-			if isSpecial {
-				return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
-			}
-
-			if err != nil {
-				aCtx.Logger().Error(err, "error handling file")
-			}
+		if success := processHandler(aCtx, h, reReader, chunkSkel, chunksChan); success {
+			return true
 		}
-
-		if err := reReader.Reset(); err != nil {
-			aCtx.Logger().Error(err, "error resetting re-reader")
-			return false
-		}
-		if _, isType := h.IsFiletype(aCtx, reReader); !isType {
-			continue
-		}
-
-		if err := reReader.Reset(); err != nil {
-			aCtx.Logger().Error(err, "error resetting re-reader")
-			return false
-		}
-		reReader.Stop()
-		return handleChunks(aCtx, h.FromFile(ctx, reReader), chunkSkel, chunksChan)
 	}

 	return false
 }

+func processHandler(ctx logContext.Context, h Handler, reReader *diskbufferreader.DiskBufferReader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
+	defer reReader.Close()
+	defer reReader.Stop()
+
+	if specialHandler, ok := h.(SpecializedHandler); ok {
+		file, isSpecial, err := specialHandler.HandleSpecialized(ctx, reReader)
+		if isSpecial {
+			return handleChunks(ctx, h.FromFile(ctx, file), chunkSkel, chunksChan)
+		}
+		if err != nil {
+			ctx.Logger().Error(err, "error handling file")
+		}
+	}
+
+	if _, err := reReader.Seek(0, io.SeekStart); err != nil {
+		ctx.Logger().Error(err, "error seeking to start of file")
+		return false
+	}
+
+	if _, isType := h.IsFiletype(ctx, reReader); !isType {
+		return false
+	}
+
+	return handleChunks(ctx, h.FromFile(ctx, reReader), chunkSkel, chunksChan)
+}
+
 func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
 	for {
 		select {
@@ -10,6 +10,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

 // JSONPrinter is a printer that prints results in JSON format.
@@ -20,7 +21,7 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 		// SourceMetadata contains source-specific contextual information.
 		SourceMetadata *source_metadatapb.MetaData
 		// SourceID is the ID of the source that the API uses to map secrets to specific sources.
-		SourceID int64
+		SourceID sources.SourceID
 		// SourceType is the type of Source.
 		SourceType sourcespb.SourceType
 		// SourceName is the name of the Source.
@@ -76,7 +76,7 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 		}
 	}

-	aggregateData := make(map[string]interface{})
+	aggregateData := make(map[string]any)
 	var aggregateDataKeys []string

 	for _, data := range meta {
@@ -93,7 +93,7 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
 	return nil
 }

-func structToMap(obj interface{}) (m map[string]map[string]interface{}, err error) {
+func structToMap(obj any) (m map[string]map[string]any, err error) {
 	data, err := json.Marshal(obj)
 	if err != nil {
 		return
@@ -19,13 +19,17 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

-const baseURL = "https://circleci.com/api/v1.1/"
+const (
+	SourceType = sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
+
+	baseURL = "https://circleci.com/api/v1.1/"
+)

 type Source struct {
 	name     string
 	token    string
-	sourceId int64
-	jobId    int64
+	sourceId sources.SourceID
+	jobId    sources.JobID
 	verify   bool
 	jobPool  *errgroup.Group
 	sources.Progress
@@ -40,19 +44,19 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI
+	return SourceType
 }

-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceId
 }

-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobId
 }

 // Init returns an initialized CircleCI source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId
@@ -24,10 +24,12 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

+const SourceType = sourcespb.SourceType_SOURCE_TYPE_DOCKER
+
 type Source struct {
 	name        string
-	sourceId    int64
-	jobId       int64
+	sourceId    sources.SourceID
+	jobId       sources.JobID
 	verify      bool
 	concurrency int
 	conn        sourcespb.Docker
@@ -42,19 +44,19 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_DOCKER
+	return SourceType
 }

-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceId
 }

-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobId
 }

 // Init initializes the source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId
@@ -22,10 +22,12 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

+const SourceType = sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
+
 type Source struct {
 	name     string
-	sourceId int64
-	jobId    int64
+	sourceId sources.SourceID
+	jobId    sources.JobID
 	verify   bool
 	paths    []string
 	log      logr.Logger
@@ -43,19 +45,19 @@ var _ sources.SourceUnitChunker = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM
+	return SourceType
 }

-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceId
 }

-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobId
 }

 // Init returns an initialized Filesystem source.
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, _ int) error {
 	s.log = aCtx.Logger()

 	s.name = name
@@ -28,7 +28,11 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

-const defaultCachePersistIncrement = 2500
+const (
+	SourceType = sourcespb.SourceType_SOURCE_TYPE_GCS
+
+	defaultCachePersistIncrement = 2500
+)

 // Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
@@ -37,16 +41,16 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_GCS
+	return SourceType
 }

 // SourceID number for GCS Source.
-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceId
 }

 // JobID number for GCS Source.
-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobId
 }

@@ -58,8 +62,8 @@ type objectManager interface {
 // Source represents a GCS source.
 type Source struct {
 	name        string
-	jobId       int64
-	sourceId    int64
+	jobId       sources.JobID
+	sourceId    sources.SourceID
 	concurrency int
 	verify      bool

@@ -107,7 +111,7 @@ func (c *persistableCache) shouldPersist() (bool, string) {
 }

 // Init returns an initialized GCS source.
-func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, id sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	s.log = aCtx.Logger()

 	s.name = name
@@ -34,10 +34,12 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

+const SourceType = sourcespb.SourceType_SOURCE_TYPE_GIT
+
 type Source struct {
 	name     string
-	sourceId int64
-	jobId    int64
+	sourceId sources.SourceID
+	jobId    sources.JobID
 	verify   bool
 	git      *Git
 	sources.Progress
@@ -51,8 +53,8 @@ type Source struct {
 type Git struct {
 	sourceType         sourcespb.SourceType
 	sourceName         string
-	sourceID           int64
-	jobID              int64
+	sourceID           sources.SourceID
+	jobID              sources.JobID
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData
 	verify             bool
 	metrics            metrics
@@ -63,7 +65,7 @@ type metrics struct {
 	commitsScanned uint64
 }

-func NewGit(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
+func NewGit(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(file, email, commit, timestamp, repository string, line int64) *source_metadatapb.MetaData,
 ) *Git {
 	return &Git{
@@ -84,14 +86,14 @@ var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_GIT
+	return SourceType
 }

-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceId
 }

-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobId
 }

@@ -109,7 +111,7 @@ func (s *Source) WithPreserveTempDirs(preserve bool) {
 }

 // Init returns an initialized GitHub source.
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId
@@ -370,6 +372,32 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
 	return clonePath, repo, nil
 }

+// PingRepoUsingToken executes git ls-remote on a repo and returns any error that occurs. It can be used to validate
+// that a repo actually exists and is reachable.
+//
+// Pinging using other authentication methods is only unimplemented because there's been no pressing need for it yet.
+func PingRepoUsingToken(ctx context.Context, token, gitUrl, user string) error {
+	if err := GitCmdCheck(); err != nil {
+		return err
+	}
+	lsUrl, err := GitURLParse(gitUrl)
+	if err != nil {
+		return err
+	}
+	if lsUrl.User == nil {
+		lsUrl.User = url.UserPassword(user, token)
+	}
+
+	// We don't actually care about any refs on the remote, we just care whether can can list them at all. So we query
+	// only for a ref that we know won't exist to minimize the search time on the remote. (By default, ls-remote exits
+	// with 0 even if it doesn't find any matching refs.)
+	fakeRef := "TRUFFLEHOG_CHECK_GIT_REMOTE_URL_REACHABILITY"
+	gitArgs := []string{"ls-remote", lsUrl.String(), "--quiet", fakeRef}
+	cmd := exec.Command("git", gitArgs...)
+	_, err = cmd.CombinedOutput()
+	return err
+}
+
 // CloneRepoUsingToken clones a repo using a provided token.
 func CloneRepoUsingToken(ctx context.Context, token, gitUrl, user string, args ...string) (string, *git.Repository, error) {
 	userInfo := url.UserPassword(user, token)
@@ -39,6 +39,8 @@ import (
 )

 const (
+	SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB
+
 	unauthGithubOrgRateLimt = 30
 	defaultPagination       = 100
 	membersAppPagination    = 500
@@ -51,8 +53,8 @@ type Source struct {
 	githubUser  string
 	githubToken string

-	sourceID          int64
-	jobID             int64
+	sourceID          sources.SourceID
+	jobID             sources.JobID
 	verify            bool
 	repos             []string
 	members           []string
@@ -104,14 +106,14 @@ var endsWithGithub = regexp.MustCompile(`github\.com/?$`)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_GITHUB
+	return SourceType
 }

-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceID
 }

-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobID
 }

@@ -200,7 +202,7 @@ func (c *filteredRepoCache) includeRepo(s string) bool {
 }

 // Init returns an initialized GitHub source.
-func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, sourceID sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	s.log = aCtx.Logger()

 	s.name = name
@@ -190,13 +190,13 @@ func TestAddMembersByApp(t *testing.T) {
 	gock.New("https://api.github.com").
 		Get("/app/installations").
 		Reply(200).
-		JSON([]map[string]interface{}{
+		JSON([]map[string]any{
 			{"account": map[string]string{"login": "super-secret-org", "type": "Organization"}},
 		})
 	gock.New("https://api.github.com").
 		Get("/orgs/super-secret-org/members").
 		Reply(200).
-		JSON([]map[string]interface{}{
+		JSON([]map[string]any{
 			{"login": "ssm1"},
 			{"login": "ssm2"},
 			{"login": "ssm3"},
@@ -221,7 +221,7 @@ func TestAddReposByApp(t *testing.T) {
 	gock.New("https://api.github.com").
 		Get("/installation/repositories").
 		Reply(200).
-		JSON(map[string]interface{}{
+		JSON(map[string]any{
 			"repositories": []map[string]string{
 				{"clone_url": "https://github/ssr1.git", "full_name": "ssr1"},
 				{"clone_url": "https://github/ssr2.git", "full_name": "ssr2"},
@@ -247,7 +247,7 @@ func TestAddOrgsByUser(t *testing.T) {
 	gock.New("https://api.github.com").
 		Get("/user/orgs").
 		Reply(200).
-		JSON([]map[string]interface{}{
+		JSON([]map[string]any{
 			{"login": "sso2"},
 		})

@@ -29,10 +29,12 @@ import (
 	"google.golang.org/protobuf/types/known/anypb"
 )

+const SourceType = sourcespb.SourceType_SOURCE_TYPE_GITLAB
+
 type Source struct {
 	name            string
-	sourceId        int64
-	jobId           int64
+	sourceId        sources.SourceID
+	jobId           sources.JobID
 	verify          bool
 	authMethod      string
 	user            string
@@ -53,23 +55,24 @@ type Source struct {
 // Ensure the Source satisfies the interfaces at compile time.
 var _ sources.Source = (*Source)(nil)
 var _ sources.SourceUnitUnmarshaller = (*Source)(nil)
+var _ sources.Validator = (*Source)(nil)

 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_GITLAB
+	return SourceType
 }

-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceId
 }

-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobId
 }

 // Init returns an initialized Gitlab source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId
@@ -146,9 +149,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 		return errors.New(err)
 	}
 	// Get repo within target.
-	repos, errs := s.getRepos()
+	repos, errs := normalizeRepos(s.repos)
 	for _, repoErr := range errs {
-		ctx.Logger().Info("error getting repo", "error", repoErr)
+		ctx.Logger().Info("error normalizing repo", "error", repoErr)
 	}

 	// End early if we had errors getting specified repos but none were validated.
@@ -158,26 +161,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .

 	// Get all repos if not specified.
 	if len(repos) == 0 {
-		projects, err := s.getAllProjects(ctx, apiClient)
-		if err != nil {
-			return fmt.Errorf("error getting all projects: %v", err)
-		}
-		// Turn projects into URLs for Git cloner.
-		for _, prj := range projects {
-			if s.ignoreRepo(ctx, prj.PathWithNamespace) {
-				continue
-			}
-
-			// Ensure the urls are valid before adding them to the repo list.
-			_, err := url.Parse(prj.HTTPURLToRepo)
-			if err != nil {
-				fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
-			}
-			repos = append(repos, prj.HTTPURLToRepo)
-		}
-		if len(repos) == 0 {
-			return errors.Errorf("unable to discover any repos")
+		ignoreRepo := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {
+			ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", pattern)
+		})
+		gitlabRepos, err2, done := s.getReposFromGitlab(ctx, apiClient, ignoreRepo)
+		if done {
+			return err2
 		}
+		repos = gitlabRepos
 	}

 	s.repos = repos
@@ -187,6 +178,69 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 	return s.scanRepos(ctx, chunksChan)
 }

+func (s *Source) Validate(ctx context.Context) []error {
+	// The client is only used to query Gitlab for a repo list - it's not used to actually clone anything. Thus, we
+	// don't use it if there is a list of explicitly configured repos. However, constructing it validates that the
+	// configured authentication method is sensible, so we'll do it here.
+	apiClient, err := s.newClient()
+	if err != nil {
+		return []error{err}
+	}
+
+	_, _, err = apiClient.Users.CurrentUser()
+	if err != nil {
+		msg := fmt.Sprintf("gitlab authentication failed using method %v", s.authMethod)
+		return []error{errors.WrapPrefix(err, msg, 0)}
+	}
+
+	explicitlyConfiguredRepos, errs := normalizeRepos(s.repos)
+
+	if len(explicitlyConfiguredRepos) > 0 {
+		user := s.user
+		if user == "" {
+			user = "placeholder"
+		}
+
+		// We only check reachability for explicitly configured repositories. The purpose of source validation is to
+		// help users validate their configuration files, and Gitlab telling us about repositories that it won't let us
+		// access isn't a local configuration issue.
+		for _, r := range explicitlyConfiguredRepos {
+			if err := git.PingRepoUsingToken(ctx, s.token, r, user); err != nil {
+				msg := fmt.Sprintf("could not reach git repository %q", r)
+				err = errors.WrapPrefix(err, msg, 0)
+				errs = append(errs, err)
+			}
+		}
+
+		if len(s.ignoreRepos) > 0 {
+			errs = append(errs, fmt.Errorf("both repositories and ignore patterns were explicitly configured; ignore patterns will not be used"))
+		}
+	}
+
+	if len(explicitlyConfiguredRepos) > 0 || len(errs) > 0 {
+		return errs
+	}
+
+	ignoreProject := buildIgnorer(s.ignoreRepos, func(err error, pattern string) {
+		msg := fmt.Sprintf("could not compile ignore repo pattern %q", pattern)
+		errs = append(errs, errors.WrapPrefix(err, msg, 0))
+	})
+
+	projects, err := s.getAllProjects(ctx, apiClient)
+	if err != nil {
+		errs = append(errs, err)
+		return errs
+	}
+
+	for _, p := range projects {
+		if !ignoreProject(p.PathWithNamespace) {
+			return errs
+		}
+	}
+
+	return append(errs, fmt.Errorf("ignore patterns excluded all projects"))
+}
+
 func (s *Source) newClient() (*gitlab.Client, error) {
 	// Initialize a new api instance.
 	switch s.authMethod {
@@ -322,23 +376,30 @@ func (s *Source) getAllProjects(ctx context.Context, apiClient *gitlab.Client) (
 	return projectList, nil
 }

-func (s *Source) getRepos() ([]string, []error) {
-	if len(s.repos) == 0 {
-		return nil, nil
+func (s *Source) getReposFromGitlab(ctx context.Context, apiClient *gitlab.Client, ignoreRepo func(repo string) bool) ([]string, error, bool) {
+	projects, err := s.getAllProjects(ctx, apiClient)
+	if err != nil {
+		return nil, fmt.Errorf("error getting all projects: %v", err), true
 	}

-	var validRepos []string
-	var errs []error
-	for _, prj := range s.repos {
-		repo, err := giturl.NormalizeGitlabRepo(prj)
-		if err != nil {
-			errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
+	// Turn projects into URLs for Git cloner.
+	var repos []string
+	for _, prj := range projects {
+		if ignoreRepo(prj.PathWithNamespace) {
 			continue
 		}

-		validRepos = append(validRepos, repo)
+		// Ensure the urls are valid before adding them to the repo list.
+		_, err := url.Parse(prj.HTTPURLToRepo)
+		if err != nil {
+			fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
+		}
+		repos = append(repos, prj.HTTPURLToRepo)
 	}
-	return validRepos, errs
+	if len(repos) == 0 {
+		return nil, errors.Errorf("unable to discover any repos"), true
+	}
+	return repos, nil, false
 }

 func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) error {
@@ -411,21 +472,6 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 	return nil
 }

-func (s *Source) ignoreRepo(ctx context.Context, r string) bool {
-	for _, ignore := range s.ignoreRepos {
-		g, err := glob.Compile(ignore)
-		if err != nil {
-			ctx.Logger().Error(err, "could not compile ignore repo glob", "glob", ignore)
-			continue
-		}
-		if g.Match(r) {
-			ctx.Logger().V(2).Info("Ignoring repo", "repo", r)
-			return true
-		}
-	}
-	return false
-}
-
 // setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
 func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
 	s.resumeInfoMutex.Lock()
@@ -445,3 +491,42 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 func (s *Source) WithScanOptions(scanOptions *git.ScanOptions) {
 	s.scanOptions = scanOptions
 }
+
+func buildIgnorer(patterns []string, onCompileErr func(err error, pattern string)) func(repo string) bool {
+	var globs []glob.Glob
+
+	for _, pattern := range patterns {
+		g, err := glob.Compile(pattern)
+		if err != nil {
+			onCompileErr(err, pattern)
+			continue
+		}
+		globs = append(globs, g)
+	}
+
+	f := func(repo string) bool {
+		for _, g := range globs {
+			if g.Match(repo) {
+				return true
+			}
+		}
+		return false
+	}
+
+	return f
+}
+
+func normalizeRepos(repos []string) ([]string, []error) {
+	var validRepos []string
+	var errs []error
+	for _, prj := range repos {
+		repo, err := giturl.NormalizeGitlabRepo(prj)
+		if err != nil {
+			errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
+			continue
+		}
+
+		validRepos = append(validRepos, repo)
+	}
+	return validRepos, errs
+}
@@ -177,6 +177,150 @@ func TestSource_Scan(t *testing.T) {
 	}
 }

+func TestSource_Validate(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	secret, err := common.GetTestSecret(ctx)
+	if err != nil {
+		t.Fatal(fmt.Errorf("failed to access secret: %v", err))
+	}
+	token := secret.MustGetField("GITLAB_TOKEN")
+	tokenWrongScope := secret.MustGetField("GITLAB_TOKEN_WRONG_SCOPE")
+
+	tests := []struct {
+		name         string
+		connection   *sourcespb.GitLab
+		wantErrCount int
+		wantErrs     []string
+	}{
+		{
+			name: "basic auth did not authenticate",
+			connection: &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_BasicAuth{
+					BasicAuth: &credentialspb.BasicAuth{
+						Username: "bad-user",
+						Password: "bad-password",
+					},
+				},
+			},
+			wantErrCount: 1,
+		},
+		{
+			name: "token did not authenticate",
+			connection: &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_Token{
+					Token: "bad-token",
+				},
+			},
+			wantErrCount: 1,
+		},
+		{
+			name: "bad repo urls",
+			connection: &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_Token{
+					Token: token,
+				},
+				Repositories: []string{
+					"https://gitlab.com/testermctestface/testy",  // valid
+					"https://gitlab.com/testermctestface/testy/", // trailing slash
+					"ssh:git@gitlab.com/testermctestface/testy",  // bad protocol
+					"https://gitlab.com",                         // no path
+					"https://gitlab.com/",                        // no org name
+					"https://gitlab.com//testy",                  // no org name
+					"https://gitlab.com/testermctestface/",       // no repo name
+				},
+			},
+			wantErrCount: 6,
+		},
+		{
+			name: "token does not have permission to list projects",
+			connection: &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_Token{
+					Token: tokenWrongScope,
+				},
+			},
+			wantErrCount: 1,
+		},
+		{
+			name: "repositories and ignore globs both configured",
+			connection: &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_Token{
+					Token: token,
+				},
+				Repositories: []string{
+					"https://gitlab.com/testermctestface/testy", // valid
+				},
+				IgnoreRepos: []string{
+					"tes1188/*-gitlab",
+					"[", // glob doesn't compile, but this won't be checked
+				},
+			},
+			wantErrCount: 1,
+		},
+		{
+			name: "could not compile ignore glob(s)",
+			connection: &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_Token{
+					Token: token,
+				},
+				IgnoreRepos: []string{
+					"tes1188/*-gitlab",
+					"[",    // glob doesn't compile
+					"[a-]", // glob doesn't compile
+				},
+			},
+			wantErrCount: 2,
+		},
+		{
+			name: "repositories do not exist or are not accessible",
+			connection: &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_Token{
+					Token: token,
+				},
+				Repositories: []string{
+					"https://gitlab.com/testermctestface/testy",
+					"https://gitlab.com/testermctestface/doesn't-exist",
+					"https://gitlab.com/testermctestface/also-doesn't-exist",
+				},
+			},
+			wantErrCount: 2,
+		},
+		{
+			name: "ignore globs exclude all repos",
+			connection: &sourcespb.GitLab{
+				Credential: &sourcespb.GitLab_Token{
+					Token: token,
+				},
+				IgnoreRepos: []string{
+					"*",
+				},
+			},
+			wantErrCount: 1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Source{}
+
+			conn, err := anypb.New(tt.connection)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			err = s.Init(ctx, tt.name, 0, 0, false, conn, 1)
+			if err != nil {
+				t.Fatalf("Source.Init() error: %v", err)
+			}
+
+			errs := s.Validate(ctx)
+
+			assert.Equal(t, tt.wantErrCount, len(errs))
+		})
+	}
+}
+
 func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
 	tests := []struct {
 		startingResumeInfoSlice []string
@@ -43,8 +43,8 @@ type JobProgressHook interface {
 // If the job supports it, the reference can also be used to cancel running via
 // CancelRun.
 type JobProgressRef struct {
-	JobID       int64
-	SourceID    int64
+	JobID       JobID
+	SourceID    SourceID
 	SourceName  string
 	jobProgress *JobProgress
 }
@@ -101,8 +101,8 @@ func (f ChunkError) Unwrap() error { return f.err }
 // JobProgress aggregates information about a run of a Source.
 type JobProgress struct {
 	// Unique identifiers for this job.
-	JobID      int64
-	SourceID   int64
+	JobID      JobID
+	SourceID   SourceID
 	SourceName string
 	// Tracks whether the job is finished or not.
 	ctx    context.Context
@@ -155,7 +155,7 @@ func WithCancel(cancel context.CancelCauseFunc) func(*JobProgress) {
 }

 // NewJobProgress creates a new job report for the given source and job ID.
-func NewJobProgress(jobID, sourceID int64, sourceName string, opts ...func(*JobProgress)) *JobProgress {
+func NewJobProgress(jobID JobID, sourceID SourceID, sourceName string, opts ...func(*JobProgress)) *JobProgress {
 	ctx, cancel := context.WithCancel(context.Background())
 	jp := &JobProgress{
 		JobID:      jobID,
@@ -38,8 +38,8 @@ func TestJobProgressFatalErrors(t *testing.T) {
 func TestJobProgressRef(t *testing.T) {
 	jp := NewJobProgress(123, 456, "source name")
 	ref := jp.Ref()
-	assert.Equal(t, int64(123), ref.JobID)
-	assert.Equal(t, int64(456), ref.SourceID)
+	assert.Equal(t, JobID(123), ref.JobID)
+	assert.Equal(t, SourceID(456), ref.SourceID)

 	// Test Done() blocks until Finish() is called.
 	select {
@@ -41,7 +41,7 @@ func (m *MockJobProgressHook) End(arg0 JobProgressRef, arg1 time.Time) {
 }

 // End indicates an expected call of End.
-func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) End(arg0, arg1 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "End", reflect.TypeOf((*MockJobProgressHook)(nil).End), arg0, arg1)
 }
@@ -53,7 +53,7 @@ func (m *MockJobProgressHook) EndEnumerating(arg0 JobProgressRef, arg1 time.Time
 }

 // EndEnumerating indicates an expected call of EndEnumerating.
-func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) EndEnumerating(arg0, arg1 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).EndEnumerating), arg0, arg1)
 }
@@ -65,7 +65,7 @@ func (m *MockJobProgressHook) EndUnitChunking(arg0 JobProgressRef, arg1 SourceUn
 }

 // EndUnitChunking indicates an expected call of EndUnitChunking.
-func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) EndUnitChunking(arg0, arg1, arg2 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EndUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).EndUnitChunking), arg0, arg1, arg2)
 }
@@ -77,7 +77,7 @@ func (m *MockJobProgressHook) Finish(arg0 JobProgressRef) {
 }

 // Finish indicates an expected call of Finish.
-func (mr *MockJobProgressHookMockRecorder) Finish(arg0 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) Finish(arg0 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockJobProgressHook)(nil).Finish), arg0)
 }
@@ -89,7 +89,7 @@ func (m *MockJobProgressHook) ReportChunk(arg0 JobProgressRef, arg1 SourceUnit,
 }

 // ReportChunk indicates an expected call of ReportChunk.
-func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) ReportChunk(arg0, arg1, arg2 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportChunk", reflect.TypeOf((*MockJobProgressHook)(nil).ReportChunk), arg0, arg1, arg2)
 }
@@ -101,7 +101,7 @@ func (m *MockJobProgressHook) ReportError(arg0 JobProgressRef, arg1 error) {
 }

 // ReportError indicates an expected call of ReportError.
-func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) ReportError(arg0, arg1 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportError", reflect.TypeOf((*MockJobProgressHook)(nil).ReportError), arg0, arg1)
 }
@@ -113,7 +113,7 @@ func (m *MockJobProgressHook) ReportUnit(arg0 JobProgressRef, arg1 SourceUnit) {
 }

 // ReportUnit indicates an expected call of ReportUnit.
-func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) ReportUnit(arg0, arg1 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportUnit", reflect.TypeOf((*MockJobProgressHook)(nil).ReportUnit), arg0, arg1)
 }
@@ -125,7 +125,7 @@ func (m *MockJobProgressHook) Start(arg0 JobProgressRef, arg1 time.Time) {
 }

 // Start indicates an expected call of Start.
-func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) Start(arg0, arg1 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockJobProgressHook)(nil).Start), arg0, arg1)
 }
@@ -137,7 +137,7 @@ func (m *MockJobProgressHook) StartEnumerating(arg0 JobProgressRef, arg1 time.Ti
 }

 // StartEnumerating indicates an expected call of StartEnumerating.
-func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) StartEnumerating(arg0, arg1 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartEnumerating", reflect.TypeOf((*MockJobProgressHook)(nil).StartEnumerating), arg0, arg1)
 }
@@ -149,7 +149,7 @@ func (m *MockJobProgressHook) StartUnitChunking(arg0 JobProgressRef, arg1 Source
 }

 // StartUnitChunking indicates an expected call of StartUnitChunking.
-func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 interface{}) *gomock.Call {
+func (mr *MockJobProgressHookMockRecorder) StartUnitChunking(arg0, arg1, arg2 any) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartUnitChunking", reflect.TypeOf((*MockJobProgressHook)(nil).StartUnitChunking), arg0, arg1, arg2)
 }
@@ -31,6 +31,8 @@ import (
 )

 const (
+	SourceType = sourcespb.SourceType_SOURCE_TYPE_S3
+
 	defaultAWSRegion     = "us-east-1"
 	defaultMaxObjectSize = 250 * 1024 * 1024 // 250 MiB
 	maxObjectSizeLimit   = 250 * 1024 * 1024 // 250 MiB
@@ -38,8 +40,8 @@ const (

 type Source struct {
 	name        string
-	sourceId    int64
-	jobId       int64
+	sourceId    sources.SourceID
+	jobId       sources.JobID
 	verify      bool
 	concurrency int
 	log         logr.Logger
@@ -58,19 +60,19 @@ var _ sources.Validator = (*Source)(nil)

 // Type returns the type of source
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_S3
+	return SourceType
 }

-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceId
 }

-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobId
 }

 // Init returns an initialized AWS source
-func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(aCtx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	s.log = context.WithValues(aCtx, "source", s.Type(), "name", name).Logger()

 	s.name = name
@@ -198,7 +200,7 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu
 		}

 		s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket), "")
-		logger.Info("Scanning bucket")
+		logger.V(3).Info("Scanning bucket")

 		regionalClient, err := s.getRegionalClientForBucket(ctx, client, role, bucket)
 		if err != nil {
@@ -278,7 +280,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan

 		// ignore large files
 		if *obj.Size > s.maxObjectSize {
-			s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
+			s.log.V(5).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
 			continue
 		}

@@ -2,10 +2,12 @@ package sources

 import (
 	"fmt"
+	"runtime"
 	"sync"
 	"sync/atomic"
 	"time"

+	"github.com/marusama/semaphore/v2"
 	"golang.org/x/sync/errgroup"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -13,32 +15,12 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 )

-// handle uniquely identifies a Source given to the manager to manage. If the
-// SourceManager is connected to the API, it will be equivalent to the unique
-// source ID, otherwise it behaves as a counter.
-type handle int64
-
-// SourceInitFunc is a function that takes a source and job ID and returns an
-// initialized Source.
-type SourceInitFunc func(ctx context.Context, jobID, sourceID int64) (Source, error)
-
-// sourceInfo is an aggregate struct to store source information provided on
-// initialization.
-type sourceInfo struct {
-	initFunc SourceInitFunc
-	name     string
-}
-
 type SourceManager struct {
 	api   apiClient
 	hooks []JobProgressHook
-	// Map of handle to source initializer.
-	handles     map[handle]sourceInfo
-	handlesLock sync.Mutex
 	// Pool limiting the amount of concurrent sources running.
-	pool                errgroup.Group
-	poolLimit           int
-	currentRunningCount int32
+	sem semaphore.Semaphore
+	wg  sync.WaitGroup
 	// Max number of units to scan concurrently per source.
 	concurrentUnits int
 	// Run the sources using source unit enumeration / chunking if available.
@@ -46,15 +28,16 @@ type SourceManager struct {
 	// Downstream chunks channel to be scanned.
 	outputChunks chan *Chunk
 	// Set when Wait() returns.
-	done bool
+	firstErr chan error
+	waitErr  error
+	done     bool
 }

 // apiClient is an interface for optionally communicating with an external API.
 type apiClient interface {
-	// RegisterSource lets the API know we have a source and it returns a unique source ID for it.
-	RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error)
-	// GetJobID queries the API for an existing job or new job ID.
-	GetJobID(ctx context.Context, id int64) (int64, error)
+	// GetIDs informs the API of the source that's about to run and returns
+	// two identifiers used during source initialization.
+	GetIDs(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, JobID, error)
 }

 // WithAPI adds an API client to the manager for tracking jobs and progress. If
@@ -72,8 +55,7 @@ func WithReportHook(hook JobProgressHook) func(*SourceManager) {
 // WithConcurrentSources limits the concurrent number of sources a manager can run.
 func WithConcurrentSources(concurrency int) func(*SourceManager) {
 	return func(mgr *SourceManager) {
-		mgr.pool.SetLimit(concurrency)
-		mgr.poolLimit = concurrency
+		mgr.sem.SetLimit(concurrency)
 	}
 }

@@ -99,8 +81,9 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
 	mgr := SourceManager{
 		// Default to the headless API. Can be overwritten by the WithAPI option.
 		api:          &headlessAPI{},
-		handles:      make(map[handle]sourceInfo),
+		sem:          semaphore.New(runtime.NumCPU()),
 		outputChunks: make(chan *Chunk),
+		firstErr:     make(chan error, 1),
 	}
 	for _, opt := range opts {
 		opt(&mgr)
@@ -108,81 +91,53 @@ func NewManager(opts ...func(*SourceManager)) *SourceManager {
 	return &mgr
 }

-// Enroll informs the SourceManager to track and manage a Source.
-func (s *SourceManager) Enroll(ctx context.Context, name string, kind sourcespb.SourceType, f SourceInitFunc) (handle, error) {
-	if s.done {
-		return 0, fmt.Errorf("manager is done")
-	}
-	id, err := s.api.RegisterSource(ctx, name, kind)
-	if err != nil {
-		return 0, err
-	}
-	handleID := handle(id)
-	s.handlesLock.Lock()
-	defer s.handlesLock.Unlock()
-	if _, ok := s.handles[handleID]; ok {
-		// TODO: smartly handle this?
-		return 0, fmt.Errorf("handle ID '%d' already in use", handleID)
-	}
-	s.handles[handleID] = sourceInfo{
-		initFunc: f,
-		name:     name,
-	}
-	return handleID, nil
+func (s *SourceManager) GetIDs(ctx context.Context, sourceName string, kind sourcespb.SourceType) (SourceID, JobID, error) {
+	return s.api.GetIDs(ctx, sourceName, kind)
 }

 // Run blocks until a resource is available to run the source, then
-// synchronously runs it. The first fatal error, if any, will be returned.
-func (s *SourceManager) Run(ctx context.Context, handle handle) (JobProgressRef, error) {
-	progress, err := s.asyncRun(ctx, handle)
-	if err != nil {
-		return progress, err
-	}
-	<-progress.Done()
-	return progress, progress.Snapshot().FatalError()
-}
-
-// ScheduleRun blocks until a resource is available to run the source, then
 // asynchronously runs it. Error information is stored and accessible via the
 // JobProgressRef as it becomes available.
-func (s *SourceManager) ScheduleRun(ctx context.Context, handle handle) (JobProgressRef, error) {
-	return s.asyncRun(ctx, handle)
+func (s *SourceManager) Run(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
+	return s.asyncRun(ctx, sourceName, source)
 }

-// asyncRun is a helper method to asynchronously run the Source. It calls out
-// to the API to get a job ID for this run, creates a JobProgress object, then
-// waits for an available goroutine to asynchronously run it.
-func (s *SourceManager) asyncRun(ctx context.Context, handle handle) (JobProgressRef, error) {
+// asyncRun is a helper method to asynchronously run the Source.
+func (s *SourceManager) asyncRun(ctx context.Context, sourceName string, source Source) (JobProgressRef, error) {
+	sourceID, jobID := source.SourceID(), source.JobID()
 	// Do preflight checks before waiting on the pool.
-	if err := s.preflightChecks(ctx, handle); err != nil {
-		return JobProgressRef{}, err
-	}
-	// Get the name. Should never fail due to preflight checks.
-	sourceInfo, ok := s.getSourceInfo(handle)
-	if !ok {
-		return JobProgressRef{SourceID: int64(handle)}, fmt.Errorf("unrecognized handle")
-	}
-	sourceName := sourceInfo.name
-	// Get a Job ID.
-	ctx = context.WithValue(ctx, "source_id", int64(handle))
-	jobID, err := s.api.GetJobID(ctx, int64(handle))
-	if err != nil {
-		return JobProgressRef{SourceID: int64(handle), SourceName: sourceName}, err
+	if err := s.preflightChecks(ctx); err != nil {
+		return JobProgressRef{
+			SourceName: sourceName,
+			SourceID:   sourceID,
+			JobID:      jobID,
+		}, err
 	}
 	// Create a JobProgress object for tracking progress.
 	ctx, cancel := context.WithCancelCause(ctx)
-	progress := NewJobProgress(jobID, int64(handle), sourceName, WithHooks(s.hooks...), WithCancel(cancel))
-	s.pool.Go(func() error {
-		atomic.AddInt32(&s.currentRunningCount, 1)
-		defer atomic.AddInt32(&s.currentRunningCount, -1)
+	progress := NewJobProgress(jobID, sourceID, sourceName, WithHooks(s.hooks...), WithCancel(cancel))
+	if err := s.sem.Acquire(ctx, 1); err != nil {
+		// Context cancelled.
+		progress.ReportError(Fatal{err})
+		return progress.Ref(), Fatal{err}
+	}
+	s.wg.Add(1)
+	go func() {
+		defer s.sem.Release(1)
+		defer s.wg.Done()
 		ctx := context.WithValues(ctx,
 			"job_id", jobID,
 			"source_manager_worker_id", common.RandomID(5),
 		)
 		defer common.Recover(ctx)
 		defer cancel(nil)
-		return s.run(ctx, handle, jobID, progress)
-	})
+		if err := s.run(ctx, source, progress); err != nil {
+			select {
+			case s.firstErr <- err:
+			default:
+			}
+		}
+	}()
 	return progress.Ref(), nil
 }

@@ -199,13 +154,19 @@ func (s *SourceManager) Chunks() <-chan *Chunk {
 func (s *SourceManager) Wait() error {
 	// Check if the manager has been Waited.
 	if s.done {
-		return s.pool.Wait()
+		return s.waitErr
 	}
-	defer close(s.outputChunks)
-	defer func() { s.done = true }()
+	s.done = true

 	// Return the first error returned by run.
-	return s.pool.Wait()
+	s.wg.Wait()
+	select {
+	case s.waitErr = <-s.firstErr:
+	default:
+	}
+	close(s.outputChunks)
+	close(s.firstErr)
+	return s.waitErr
 }

 // ScanChunk injects a chunk into the output stream of chunks to be scanned.
@@ -216,33 +177,45 @@ func (s *SourceManager) ScanChunk(chunk *Chunk) {
 }

 // AvailableCapacity returns the number of concurrent jobs the manager can
-// accommodate at this time. If there is no limit, -1 is returned.
+// accommodate at this time.
 func (s *SourceManager) AvailableCapacity() int {
-	if s.poolLimit == 0 {
-		return -1
-	}
-	runCount := atomic.LoadInt32(&s.currentRunningCount)
-	return s.poolLimit - int(runCount)
+	return s.sem.GetLimit() - s.sem.GetCount()
+}
+
+// MaxConcurrentSources returns the maximum configured limit of concurrent
+// sources the manager will run.
+func (s *SourceManager) MaxConcurrentSources() int {
+	return s.sem.GetLimit()
+}
+
+// ConcurrentSources returns the current number of concurrently running
+// sources.
+func (s *SourceManager) ConcurrentSources() int {
+	return s.sem.GetCount()
+}
+
+// SetMaxConcurrentSources sets the maximum number of concurrently running
+// sources. If the count is lower than the already existing number of
+// concurrently running sources, no sources will be scheduled to run until the
+// existing sources complete.
+func (s *SourceManager) SetMaxConcurrentSources(maxRunCount int) {
+	s.sem.SetLimit(maxRunCount)
 }

 // preflightChecks is a helper method to check the Manager or the context isn't
-// done and that the handle is valid.
-func (s *SourceManager) preflightChecks(ctx context.Context, handle handle) error {
+// done.
+func (s *SourceManager) preflightChecks(ctx context.Context) error {
 	// Check if the manager has been Waited.
 	if s.done {
 		return fmt.Errorf("manager is done")
 	}
-	// Check the handle is valid.
-	if _, ok := s.getSourceInfo(handle); !ok {
-		return fmt.Errorf("unrecognized handle")
-	}
 	return ctx.Err()
 }

 // run is a helper method to sychronously run the source. It does not check for
 // acquired resources. An error is returned if there was a fatal error during
 // the run. This information is also recorded in the JobProgress.
-func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, report *JobProgress) error {
+func (s *SourceManager) run(ctx context.Context, source Source, report *JobProgress) error {
 	defer report.Finish()
 	report.Start(time.Now())
 	defer func() { report.End(time.Now()) }()
@@ -253,34 +226,21 @@ func (s *SourceManager) run(ctx context.Context, handle handle, jobID int64, rep
 		}
 	}()

-	// Initialize the source.
-	sourceInfo, ok := s.getSourceInfo(handle)
-	if !ok {
-		// Shouldn't happen due to preflight checks.
-		err := fmt.Errorf("unrecognized handle")
-		report.ReportError(Fatal{err})
-		return Fatal{err}
-	}
-	source, err := sourceInfo.initFunc(ctx, jobID, int64(handle))
-	if err != nil {
-		report.ReportError(Fatal{err})
-		return Fatal{err}
-	}
 	report.TrackProgress(source.GetProgress())
 	ctx = context.WithValues(ctx,
 		"source_type", source.Type().String(),
-		"source_name", sourceInfo.name,
+		"source_name", report.SourceName,
 	)
 	// Check for the preferred method of tracking source units.
 	if enumChunker, ok := source.(SourceUnitEnumChunker); ok && s.useSourceUnits {
-		return s.runWithUnits(ctx, handle, enumChunker, report)
+		return s.runWithUnits(ctx, enumChunker, report)
 	}
-	return s.runWithoutUnits(ctx, handle, source, report)
+	return s.runWithoutUnits(ctx, source, report)
 }

 // runWithoutUnits is a helper method to run a Source. It has coarse-grained
 // job reporting.
-func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, source Source, report *JobProgress) error {
+func (s *SourceManager) runWithoutUnits(ctx context.Context, source Source, report *JobProgress) error {
 	// Introspect on the chunks we get from the Chunks method.
 	ch := make(chan *Chunk, 1)
 	var wg sync.WaitGroup
@@ -310,7 +270,7 @@ func (s *SourceManager) runWithoutUnits(ctx context.Context, handle handle, sour
 // runWithUnits is a helper method to run a Source that is also a
 // SourceUnitEnumChunker. This allows better introspection of what is getting
 // scanned and any errors encountered.
-func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source SourceUnitEnumChunker, report *JobProgress) error {
+func (s *SourceManager) runWithUnits(ctx context.Context, source SourceUnitEnumChunker, report *JobProgress) error {
 	unitReporter := &mgrUnitReporter{
 		unitCh: make(chan SourceUnit, 1),
 		report: report,
@@ -385,28 +345,15 @@ func (s *SourceManager) runWithUnits(ctx context.Context, handle handle, source
 	}
 }

-// getSourceInfo is a helper method for safe concurrent access to the
-// map[handle]SourceInitFunc map.
-func (s *SourceManager) getSourceInfo(handle handle) (sourceInfo, bool) {
-	s.handlesLock.Lock()
-	defer s.handlesLock.Unlock()
-	f, ok := s.handles[handle]
-	return f, ok
-}
-
 // headlessAPI implements the apiClient interface locally.
 type headlessAPI struct {
-	// Counters for assigning handle and job IDs.
+	// Counters for assigning source and job IDs.
 	sourceIDCounter int64
 	jobIDCounter    int64
 }

-func (api *headlessAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
-	return atomic.AddInt64(&api.sourceIDCounter, 1), nil
-}
-
-func (api *headlessAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
-	return atomic.AddInt64(&api.jobIDCounter, 1), nil
+func (api *headlessAPI) GetIDs(context.Context, string, sourcespb.SourceType) (SourceID, JobID, error) {
+	return SourceID(atomic.AddInt64(&api.sourceIDCounter, 1)), JobID(atomic.AddInt64(&api.jobIDCounter, 1)), nil
 }

 // mgrUnitReporter implements the UnitReporter interface.
@@ -15,15 +15,15 @@ import (

 // DummySource implements Source and is used for testing a SourceManager.
 type DummySource struct {
-	sourceID int64
-	jobID    int64
+	sourceID SourceID
+	jobID    JobID
 	chunker
 }

 func (d *DummySource) Type() sourcespb.SourceType { return 1337 }
-func (d *DummySource) SourceID() int64            { return d.sourceID }
-func (d *DummySource) JobID() int64               { return d.jobID }
-func (d *DummySource) Init(_ context.Context, _ string, jobID, sourceID int64, _ bool, _ *anypb.Any, _ int) error {
+func (d *DummySource) SourceID() SourceID         { return d.sourceID }
+func (d *DummySource) JobID() JobID               { return d.jobID }
+func (d *DummySource) Init(_ context.Context, _ string, jobID JobID, sourceID SourceID, _ bool, _ *anypb.Any, _ int) error {
 	d.sourceID = sourceID
 	d.jobID = jobID
 	return nil
@@ -80,16 +80,13 @@ func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) er
 func (c errorChunker) Enumerate(context.Context, UnitReporter) error                { return c }
 func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error   { return c }

-// enrollDummy is a helper function to enroll a DummySource with a SourceManager.
-func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {
-	return mgr.Enroll(context.Background(), "dummy", 1337,
-		func(ctx context.Context, jobID, sourceID int64) (Source, error) {
-			source := &DummySource{chunker: chunkMethod}
-			if err := source.Init(ctx, "dummy", jobID, sourceID, true, nil, 42); err != nil {
-				return nil, err
-			}
-			return source, nil
-		})
+// buildDummy is a helper function to enroll a DummySource with a SourceManager.
+func buildDummy(chunkMethod chunker) (Source, error) {
+	source := &DummySource{chunker: chunkMethod}
+	if err := source.Init(context.Background(), "dummy", 123, 456, true, nil, 42); err != nil {
+		return nil, err
+	}
+	return source, nil
 }

 // tryRead is a helper function that will try to read from a channel and return
@@ -105,11 +102,13 @@ func tryRead(ch <-chan *Chunk) (*Chunk, error) {

 func TestSourceManagerRun(t *testing.T) {
 	mgr := NewManager(WithBufferedOutput(8))
-	handle, err := enrollDummy(mgr, &counterChunker{count: 1})
+	source, err := buildDummy(&counterChunker{count: 1})
 	assert.NoError(t, err)
 	for i := 0; i < 3; i++ {
-		_, err = mgr.Run(context.Background(), handle)
+		ref, err := mgr.Run(context.Background(), "dummy", source)
+		<-ref.Done()
 		assert.NoError(t, err)
+		assert.NoError(t, ref.Snapshot().FatalError())
 		chunk, err := tryRead(mgr.Chunks())
 		assert.NoError(t, err)
 		assert.Equal(t, []byte{byte(i)}, chunk.Data)
@@ -121,35 +120,31 @@ func TestSourceManagerRun(t *testing.T) {

 func TestSourceManagerWait(t *testing.T) {
 	mgr := NewManager()
-	handle, err := enrollDummy(mgr, &counterChunker{count: 1})
+	source, err := buildDummy(&counterChunker{count: 1})
 	assert.NoError(t, err)
 	// Asynchronously run the source.
-	_, err = mgr.ScheduleRun(context.Background(), handle)
+	_, err = mgr.Run(context.Background(), "dummy", source)
 	assert.NoError(t, err)
 	// Read the 1 chunk we're expecting so Waiting completes.
 	<-mgr.Chunks()
 	// Wait for all resources to complete.
 	assert.NoError(t, mgr.Wait())
-	// Enroll and run should return an error now.
-	_, err = enrollDummy(mgr, &counterChunker{count: 1})
-	assert.Error(t, err)
-	_, err = mgr.ScheduleRun(context.Background(), handle)
+	// Run should return an error now.
+	_, err = buildDummy(&counterChunker{count: 1})
+	assert.NoError(t, err)
+	_, err = mgr.Run(context.Background(), "dummy", source)
 	assert.Error(t, err)
 }

 func TestSourceManagerError(t *testing.T) {
 	mgr := NewManager()
-	handle, err := enrollDummy(mgr, errorChunker{fmt.Errorf("oops")})
+	source, err := buildDummy(errorChunker{fmt.Errorf("oops")})
 	assert.NoError(t, err)
-	// A synchronous run should fail.
-	_, err = mgr.Run(context.Background(), handle)
-	assert.Error(t, err)
-	// Scheduling a run should not fail, but the error should surface in
-	// Wait().
-	ref, err := mgr.ScheduleRun(context.Background(), handle)
+	ref, err := mgr.Run(context.Background(), "dummy", source)
 	assert.NoError(t, err)
-	assert.Error(t, mgr.Wait())
+	<-ref.Done()
 	assert.Error(t, ref.Snapshot().FatalError())
+	assert.Error(t, mgr.Wait())
 }

 func TestSourceManagerReport(t *testing.T) {
@@ -159,11 +154,11 @@ func TestSourceManagerReport(t *testing.T) {
 		{WithBufferedOutput(8), WithSourceUnits(), WithConcurrentUnits(1)},
 	} {
 		mgr := NewManager(opts...)
-		handle, err := enrollDummy(mgr, &counterChunker{count: 4})
+		source, err := buildDummy(&counterChunker{count: 4})
 		assert.NoError(t, err)
-		// Synchronously run the source.
-		ref, err := mgr.Run(context.Background(), handle)
+		ref, err := mgr.Run(context.Background(), "dummy", source)
 		assert.NoError(t, err)
+		<-ref.Done()
 		assert.Equal(t, 0, len(ref.Snapshot().Errors))
 		assert.Equal(t, uint64(4), ref.Snapshot().TotalChunks)
 	}
@@ -221,10 +216,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {
 		{unit: "three", err: "not again"},
 	}
 	mgr := NewManager(WithBufferedOutput(8), WithSourceUnits())
-	handle, err := enrollDummy(mgr, &unitChunker{input})
+	source, err := buildDummy(&unitChunker{input})
 	assert.NoError(t, err)
-	ref, err := mgr.Run(context.Background(), handle)
+	ref, err := mgr.Run(context.Background(), "dummy", source)
 	assert.NoError(t, err)
+	<-ref.Done()
 	report := ref.Snapshot()
 	assert.Equal(t, len(input), int(report.TotalUnits))
 	assert.Equal(t, len(input), int(report.FinishedUnits))
@@ -235,11 +231,11 @@ func TestSourceManagerNonFatalError(t *testing.T) {

 func TestSourceManagerContextCancelled(t *testing.T) {
 	mgr := NewManager(WithBufferedOutput(8))
-	handle, err := enrollDummy(mgr, &counterChunker{count: 100})
+	source, err := buildDummy(&counterChunker{count: 100})
 	assert.NoError(t, err)

 	ctx, cancel := context.WithCancel(context.Background())
-	ref, err := mgr.ScheduleRun(ctx, handle)
+	ref, err := mgr.Run(ctx, "dummy", source)
 	assert.NoError(t, err)

 	cancel()
@@ -249,47 +245,18 @@ func TestSourceManagerContextCancelled(t *testing.T) {
 }

 type DummyAPI struct {
-	registerSource func(context.Context, string, sourcespb.SourceType) (int64, error)
-	getJobID       func(context.Context, int64) (int64, error)
+	registerSource func(context.Context, string, sourcespb.SourceType) (SourceID, error)
+	getJobID       func(context.Context, SourceID) (JobID, error)
 }

-func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (int64, error) {
+func (api DummyAPI) RegisterSource(ctx context.Context, name string, kind sourcespb.SourceType) (SourceID, error) {
 	return api.registerSource(ctx, name, kind)
 }

-func (api DummyAPI) GetJobID(ctx context.Context, id int64) (int64, error) {
+func (api DummyAPI) GetJobID(ctx context.Context, id SourceID) (JobID, error) {
 	return api.getJobID(ctx, id)
 }

-func TestSourceManagerJobAndSourceIDs(t *testing.T) {
-	mgr := NewManager(WithAPI(DummyAPI{
-		registerSource: func(context.Context, string, sourcespb.SourceType) (int64, error) {
-			return 1337, nil
-		},
-		getJobID: func(context.Context, int64) (int64, error) {
-			return 9001, nil
-		},
-	}))
-	var (
-		initializedJobID    int64
-		initializedSourceID int64
-	)
-	handle, err := mgr.Enroll(context.Background(), "dummy", 1337,
-		func(ctx context.Context, jobID, sourceID int64) (Source, error) {
-			initializedJobID = jobID
-			initializedSourceID = sourceID
-			return nil, fmt.Errorf("ignore")
-		})
-	assert.NoError(t, err)
-
-	ref, _ := mgr.Run(context.Background(), handle)
-	assert.Equal(t, int64(1337), initializedSourceID)
-	assert.Equal(t, int64(1337), ref.SourceID)
-	assert.Equal(t, int64(9001), initializedJobID)
-	assert.Equal(t, int64(9001), ref.JobID)
-	assert.Equal(t, "dummy", ref.SourceName)
-}
-
 // Chunk method that has a custom callback for the Chunks method.
 type callbackChunker struct {
 	cb func(context.Context, chan *Chunk) error
@@ -304,7 +271,7 @@ func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) e
 func TestSourceManagerCancelRun(t *testing.T) {
 	mgr := NewManager(WithBufferedOutput(8))
 	var returnedErr error
-	handle, err := enrollDummy(mgr, callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
+	source, err := buildDummy(callbackChunker{func(ctx context.Context, _ chan *Chunk) error {
 		// The context passed to Chunks should get cancelled when ref.CancelRun() is called.
 		<-ctx.Done()
 		returnedErr = fmt.Errorf("oh no: %w", ctx.Err())
@@ -312,7 +279,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
 	}})
 	assert.NoError(t, err)

-	ref, err := mgr.ScheduleRun(context.Background(), handle)
+	ref, err := mgr.Run(context.Background(), "dummy", source)
 	assert.NoError(t, err)

 	cancelErr := fmt.Errorf("abort! abort!")
@@ -326,7 +293,7 @@ func TestSourceManagerCancelRun(t *testing.T) {
 func TestSourceManagerAvailableCapacity(t *testing.T) {
 	mgr := NewManager(WithConcurrentSources(1337))
 	start, end := make(chan struct{}), make(chan struct{})
-	handle, err := enrollDummy(mgr, callbackChunker{func(context.Context, chan *Chunk) error {
+	source, err := buildDummy(callbackChunker{func(context.Context, chan *Chunk) error {
 		start <- struct{}{} // Send start signal.
 		<-end               // Wait for end signal.
 		return nil
@@ -334,7 +301,7 @@ func TestSourceManagerAvailableCapacity(t *testing.T) {
 	assert.NoError(t, err)

 	assert.Equal(t, 1337, mgr.AvailableCapacity())
-	ref, err := mgr.ScheduleRun(context.Background(), handle)
+	ref, err := mgr.Run(context.Background(), "dummy", source)
 	assert.NoError(t, err)

 	<-start // Wait for start signal.
@@ -11,14 +11,19 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 )

+type (
+	SourceID int64
+	JobID    int64
+)
+
 // Chunk contains data to be decoded and scanned along with context on where it came from.
 type Chunk struct {
 	// SourceName is the name of the Source that produced the chunk.
 	SourceName string
 	// SourceID is the ID of the source that the Chunk originated from.
-	SourceID int64
+	SourceID SourceID
 	// JobID is the ID of the job that the Chunk originated from.
-	JobID int64
+	JobID JobID
 	// SourceType is the type of Source that produced the chunk.
 	SourceType sourcespb.SourceType
 	// SourceMetadata holds the context of where the Chunk was found.
@@ -45,11 +50,11 @@ type Source interface {
 	// Type returns the source type, used for matching against configuration and jobs.
 	Type() sourcespb.SourceType
 	// SourceID returns the initialized source ID used for tracking relationships in the DB.
-	SourceID() int64
+	SourceID() SourceID
 	// JobID returns the initialized job ID used for tracking relationships in the DB.
-	JobID() int64
+	JobID() JobID
 	// Init initializes the source.
-	Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error
+	Init(aCtx context.Context, name string, jobId JobID, sourceId SourceID, verify bool, connection *anypb.Any, concurrency int) error
 	// Chunks emits data over a channel which is then decoded and scanned for secrets.
 	// By default, data is obtained indiscriminately. However, by providing one or more
 	// ChunkingTarget parameters, the caller can direct the function to retrieve
@@ -272,6 +277,9 @@ type Validator interface {
 // scope should be the len(scopedItems)
 // message is the public facing user information about the current progress
 // encodedResumeInfo is an optional string representing any information necessary to resume the job if interrupted
+//
+//	NOTE: SetProgressOngoing should be used when source does not yet know how many items it is scanning (scope)
+//	and does not want to display a percentage complete
 func (p *Progress) SetProgressComplete(i, scope int, message, encodedResumeInfo string) {
 	p.mut.Lock()
 	defer p.mut.Unlock()
@@ -290,6 +298,23 @@ func (p *Progress) SetProgressComplete(i, scope int, message, encodedResumeInfo
 	p.PercentComplete = int64((float64(i) / float64(scope)) * 100)
 }

+// SetProgressOngoing sets information about the current running job based on
+// the highest level objects in the source.
+// message is the public facing user information about the current progress
+// encodedResumeInfo is an optional string representing any information necessary to resume the job if interrupted
+//
+//	NOTE: This method should be used over SetProgressComplete when the source does
+//	not yet know how many items it is scanning and does not want to display a percentage complete.
+func (p *Progress) SetProgressOngoing(message string, encodedResumeInfo string) {
+	p.mut.Lock()
+	defer p.mut.Unlock()
+
+	p.Message = message
+	p.EncodedResumeInfo = encodedResumeInfo
+	// Explicitly set SectionsRemaining to 0 so the frontend does not display a percent.
+	p.SectionsRemaining = 0
+}
+
 // GetProgress gets job completion percentage for metrics reporting.
 func (p *Progress) GetProgress() *Progress {
 	p.mut.Lock()
@@ -23,12 +23,16 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

-const nilString = ""
+const (
+	SourceType = sourcespb.SourceType_SOURCE_TYPE_SYSLOG
+
+	nilString = ""
+)

 type Source struct {
 	name     string
-	sourceId int64
-	jobId    int64
+	sourceId sources.SourceID
+	jobId    sources.JobID
 	verify   bool
 	syslog   *Syslog
 	sources.Progress
@@ -38,14 +42,14 @@ type Source struct {
 type Syslog struct {
 	sourceType         sourcespb.SourceType
 	sourceName         string
-	sourceID           int64
-	jobID              int64
+	sourceID           sources.SourceID
+	jobID              sources.JobID
 	sourceMetadataFunc func(hostname, appname, procid, timestamp, facility, client string) *source_metadatapb.MetaData
 	verify             bool
 	concurrency        *semaphore.Weighted
 }

-func NewSyslog(sourceType sourcespb.SourceType, jobID, sourceID int64, sourceName string, verify bool, concurrency int,
+func NewSyslog(sourceType sourcespb.SourceType, jobID sources.JobID, sourceID sources.SourceID, sourceName string, verify bool, concurrency int,
 	sourceMetadataFunc func(hostname, appname, procid, timestamp, facility, client string) *source_metadatapb.MetaData,
 ) *Syslog {
 	return &Syslog{
@@ -103,14 +107,14 @@ var _ sources.Source = (*Source)(nil)
 // Type returns the type of source.
 // It is used for matching source types in configuration and job input.
 func (s *Source) Type() sourcespb.SourceType {
-	return sourcespb.SourceType_SOURCE_TYPE_SYSLOG
+	return SourceType
 }

-func (s *Source) SourceID() int64 {
+func (s *Source) SourceID() sources.SourceID {
 	return s.sourceId
 }

-func (s *Source) JobID() int64 {
+func (s *Source) JobID() sources.JobID {
 	return s.jobId
 }

@@ -119,7 +123,7 @@ func (s *Source) InjectConnection(conn *sourcespb.Syslog) {
 }

 // Init returns an initialized Syslog source.
-func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {

 	s.name = name
 	s.sourceId = sourceId
Author	SHA1	Message	Date
joeleonjr	ba75163d88	Update README.md	2023-09-22 14:25:39 -04:00
joeleonjr	1edefe4f73	Update README for GH Comments Scanning	2023-09-22 14:24:20 -04:00
ahrav	6affc903e1	add line to link for azure repos. (#1801 )	2023-09-21 16:07:11 -07:00
Dustin Decker	5afc7a32ca	fix detector test action (#1805 )	2023-09-21 15:16:00 -07:00
Dustin Decker	b66bd9544c	aggregate detector tests daily (#1800 ) * aggregate detector tests daily * add manual dispatch * fix test	2023-09-21 10:32:40 -07:00
āh̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d	62b2195502	Adding new function SetProgressOngoing to be used when the source does not yet know how many items it is scanning and does not want to display a percentage complete. (#1802 ) Co-Authored-By: @mcastorina	2023-09-21 13:26:10 -04:00
āh̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d	1a1b2ca51a	updating uri detector to use tri-state verification (#1791 )	2023-09-21 11:20:40 -04:00
dependabot[bot]	590115bca4	Bump golang.org/x/oauth2 from 0.10.0 to 0.12.0 (#1799 ) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.10.0 to 0.12.0. - [Commits](https://github.com/golang/oauth2/compare/v0.10.0...v0.12.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-09-20 19:49:12 -07:00
dependabot[bot]	9b01c3d71c	Bump github.com/go-ldap/ldap/v3 from 3.4.5 to 3.4.6 (#1796 ) Bumps [github.com/go-ldap/ldap/v3](https://github.com/go-ldap/ldap) from 3.4.5 to 3.4.6. - [Release notes](https://github.com/go-ldap/ldap/releases) - [Commits](https://github.com/go-ldap/ldap/compare/v3.4.5...v3.4.6) --- updated-dependencies: - dependency-name: github.com/go-ldap/ldap/v3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-09-20 17:58:21 -07:00
dependabot[bot]	16f885b2fc	Bump github.com/charmbracelet/bubbletea from 0.24.1 to 0.24.2 (#1798 ) Bumps [github.com/charmbracelet/bubbletea](https://github.com/charmbracelet/bubbletea) from 0.24.1 to 0.24.2. - [Release notes](https://github.com/charmbracelet/bubbletea/releases) - [Commits](https://github.com/charmbracelet/bubbletea/compare/v0.24.1...v0.24.2) --- updated-dependencies: - dependency-name: github.com/charmbracelet/bubbletea dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-09-20 17:22:52 -07:00
dependabot[bot]	7ac4be481c	Bump github.com/getsentry/sentry-go from 0.22.0 to 0.24.1 (#1797 ) Bumps [github.com/getsentry/sentry-go](https://github.com/getsentry/sentry-go) from 0.22.0 to 0.24.1. - [Release notes](https://github.com/getsentry/sentry-go/releases) - [Changelog](https://github.com/getsentry/sentry-go/blob/master/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-go/compare/v0.22.0...v0.24.1) --- updated-dependencies: - dependency-name: github.com/getsentry/sentry-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-09-20 17:21:43 -07:00
dependabot[bot]	f2a528e32b	Bump cloud.google.com/go/storage from 1.31.0 to 1.33.0 (#1795 ) Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.31.0 to 1.33.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.31.0...pubsub/v1.33.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/storage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-09-20 17:20:27 -07:00
dependabot[bot]	b9ae7014b0	Bump github.com/mattn/go-isatty from 0.0.18 to 0.0.19 (#1794 ) Bumps [github.com/mattn/go-isatty](https://github.com/mattn/go-isatty) from 0.0.18 to 0.0.19. - [Commits](https://github.com/mattn/go-isatty/compare/v0.0.18...v0.0.19) --- updated-dependencies: - dependency-name: github.com/mattn/go-isatty dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-09-20 16:53:24 -07:00
Miccah	efa404942a	Add ability to dynamically scale concurrently running sources (#1790 ) * Add ability to dynamically scale concurrently running sources Refactor SourceManager to use a counting semaphore to allow for dymanically changing limits. This complicated `Wait() error` which needs to return the first error encountered. We previously got that for free using `errgroup.Group`, however now we need to handle that ourselves. `Wait()` needs to return an error for use in the engine to set the correct exit code. * Group third party imports together	2023-09-20 16:49:56 -07:00
ahrav	a8c89c59b9	[bug] - fix link line (#1793 ) * fix link line. * rename.	2023-09-20 14:46:00 -07:00
ahrav	47d5ddebf2	Ability to update line number in link (#1788 ) * Add functionality to update a source's link in the metadata with the updated line number. * update comment. * add logic to engine. * only update link for non empty links. * add tests for bb.	2023-09-19 15:39:13 -07:00
Ankush Goel	63eaccd208	fixed rubygems detector (#1781 ) Co-authored-by: dsingdev-rocketx <bughunter00@protonmail.com>	2023-09-19 06:59:30 -07:00
Ankush Goel	f9ea22f72b	Update sonarcloud.go (#1784 )	2023-09-18 08:24:51 -07:00
ahrav	d2676618c0	[bug] - correclty handle nested archived directories (#1778 )	2023-09-15 04:37:15 -07:00
ahrav	22876f8381	replace interface{} with any. (#1771 )	2023-09-15 04:35:15 -07:00
Miccah	dbcb888063	Update Source interface to use SourceID and JobID types (#1774 ) The previous implementation used int64 for both, which can be mixed up easily. Using distinct types adds a layer of type safety checked by the compiler.	2023-09-14 11:28:24 -07:00
Mike Vanbuskirk	bbc3be3b6c	migrate buildpulse to integration test suite (#1775 )	2023-09-13 15:25:12 -05:00
Mike Vanbuskirk	b4329e0825	add buildpulse config to sources (#1764 ) * add buildpulse config to sources * remove tab indentation * add correct repo ID * wrap test command in gotestsum * exclusion should be detectors * fix exclusions to match community-test * update tag to reflect comm. tests	2023-09-13 11:34:53 -05:00
Cody Rose	1155ee2736	Implement Gitlab source validation (#1765 ) This PR implements validation of Gitlab source configuration. I was hoping to be able to unify more of the implementation of Validate and Chunks, but there was more divergence than I expected. Specifically, Chunks handles a fair number of Gitlab errors that aren't configuration errors (e.g. "Gitlab returned a repo with an unparseable URL"). Accommodating these in the Validate code path felt wrong, and I wasn't able to create a common code path that could accommodate both Validate and Chunks without looking awful.	2023-09-13 11:51:12 -04:00
martinohmann	31d17c4f93	fix: add missing error check in archive handler (#1770 ) Fixes #1769 The existing error check `errors.Is(err, archiver.ErrNoMatch) && depth > 0` only conditionally handled a specific error. Any other error case was not short circuited and ended up causing a nil-pointer dereference further down the method when `format.Name()` was invoked.	2023-09-13 07:07:40 -07:00
Miccah	72b6a9ec6b	Add a SourceType constant to all source packages (#1768 )	2023-09-12 17:23:25 -07:00
Miccah	be4d0bcb41	Refactor SourceManager to remove Enrollment (#1740 ) * Refactor SourceManager to remove Enrollment Initializing the Source will be the responsibility of the caller. The SourceManager exposes a GetIDs method for getting a source and job ID. * Update tests * Update engine usage * Update apiClient interface to have one GetIDs method * Update SourceManager usage in engine	2023-09-12 16:58:38 -07:00
âh̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d	91340c1e00	updating sendbirdorganizationapi detector to use tri-state verification (#1763 )	2023-09-12 10:48:11 -04:00
Mike Vanbuskirk	de540652cb	verbosity updates to s3 source (#1750 )	2023-09-11 14:53:43 -05:00
ahrav	90a07f0352	[chore] - fix slackwebhook detector (#1761 ) * fix slackwebhook detector. * sort imports. * add test for 400.	2023-09-11 12:48:40 -07:00
Dustin Decker	72b3fa31a3	Improve private key detector (#1760 ) * Surface extra data and check private keys directly against gitlab and github * fix encrpypted private key test * implement feedback * mod tidy * fix change * Set timeout for SSH connections	2023-09-11 12:05:27 -07:00