Use structpb for Extra Data to support other types

update aws test
update extra data type
2023-03-20 10:17:05 -07:00 · 2023-03-20 09:57:08 -07:00 · 2023-03-20 09:56:58 -07:00
30 changed files with 1475 additions and 1659 deletions
@@ -17,7 +17,7 @@ jobs:
    name: lint
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v3
        with:
          go-version: '1.18'
      - uses: actions/checkout@v3
@@ -33,7 +33,7 @@ jobs:
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v3
        with:
          go-version: '1.18'
      - name: Run GoReleaser
@@ -14,7 +14,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
    - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v3
      with:
        go-version: '1.18'
    - name: Checkout code
@@ -9,7 +9,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
    - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v3
      with:
        go-version: '1.18'
    - name: Checkout code
@@ -16,7 +16,7 @@ jobs:
    name: Run Snifftest
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v3
        with:
          go-version: '1.18'
      - uses: actions/checkout@v3
@@ -17,7 +17,7 @@ jobs:
      id-token: 'write'
    steps:
    - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v3
      with:
        go-version: '1.18'
    - name: Checkout code
@@ -36,7 +36,7 @@ jobs:
      id-token: 'write'
    steps:
    - name: Install Go
-      uses: actions/setup-go@v4
+      uses: actions/setup-go@v3
      with:
        go-version: '1.18'
    - name: Checkout code
@@ -10,7 +10,7 @@ replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff

 require (
 	cloud.google.com/go/secretmanager v1.10.0
-	cloud.google.com/go/storage v1.30.0
+	cloud.google.com/go/storage v1.29.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
 	github.com/TheZeroSlave/zapsentry v1.14.0
 	github.com/aws/aws-sdk-go v1.44.83
@@ -25,7 +25,7 @@ require (
 	github.com/felixge/fgprof v0.9.3
 	github.com/getsentry/sentry-go v0.19.0
 	github.com/go-errors/errors v1.4.2
-	github.com/go-git/go-git/v5 v5.6.1
+	github.com/go-git/go-git/v5 v5.6.0
 	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/go-logr/logr v1.2.3
 	github.com/go-logr/zapr v1.2.3
@@ -35,7 +35,7 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v42 v42.0.0
-	github.com/googleapis/gax-go/v2 v2.8.0
+	github.com/googleapis/gax-go/v2 v2.7.1
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-retryablehttp v0.7.2
 	github.com/jlaffaye/ftp v0.1.0
@@ -45,7 +45,6 @@ require (
 	github.com/lib/pq v1.10.7
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/mholt/archiver/v4 v4.0.0-alpha.7
-	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
 	github.com/pkg/errors v0.9.1
@@ -54,7 +53,7 @@ require (
 	github.com/stretchr/testify v1.8.2
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/xanzy/go-gitlab v0.81.0
-	go.mongodb.org/mongo-driver v1.11.3
+	go.mongodb.org/mongo-driver v1.11.2
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.7.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
@@ -62,7 +61,7 @@ require (
 	golang.org/x/sync v0.1.0
 	golang.org/x/text v0.8.0
 	google.golang.org/api v0.111.0
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/protobuf v1.29.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
 	sigs.k8s.io/yaml v1.3.0
@@ -82,8 +81,8 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
-	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
@@ -95,7 +94,7 @@ require (
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
-	github.com/go-git/go-billy/v5 v5.4.1 // indirect
+	github.com/go-git/go-billy/v5 v5.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
@@ -10,8 +10,8 @@ cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB
 cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
 cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
 cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
-cloud.google.com/go/storage v1.30.0 h1:g1yrbxAWOrvg/594228pETWkOi00MLTrOWfh56veU5o=
-cloud.google.com/go/storage v1.30.0/go.mod h1:xAVretHSROm1BQX4IIsoVgJqw0LqOyX+I/O2GzRAzdE=
+cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
+cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
@@ -38,12 +38,12 @@ github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzS
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
-github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
-github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
+github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
+github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
 github.com/TheZeroSlave/zapsentry v1.14.0 h1:z6H0lIyKVpdXsP5K8J11b4/+V+QCDLDslZmtXsnduMk=
 github.com/TheZeroSlave/zapsentry v1.14.0/go.mod h1:0hUR1Et5tSroEYhWOZcxE2iBqcFkuG7jGGlUmOA7XAs=
-github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
-github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
+github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
@@ -115,12 +115,12 @@ github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
 github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
 github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
-github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
+github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE=
+github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
 github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
 github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
-github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk=
-github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8=
+github.com/go-git/go-git/v5 v5.6.0 h1:JvBdYfcttd+0kdpuWO7KTu0FYgCf5W0t5VwkWGobaa4=
+github.com/go-git/go-git/v5 v5.6.0/go.mod h1:6nmJ0tJ3N4noMV1Omv7rC5FG3/o8Cm51TB4CJp7mRmE=
 github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
 github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -195,8 +195,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc=
-github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
+github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A=
+github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -283,8 +283,6 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
 github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
 github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
@@ -359,8 +357,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y=
-go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
+go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
@@ -385,7 +383,7 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -421,8 +419,7 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -466,15 +463,15 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
 golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -482,7 +479,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@@ -536,8 +532,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
+google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1,20 +0,0 @@
-// Package cache provides an interface which can be implemented by different cache types.
-package cache
-
-// Cache is used to store key/value pairs.
-type Cache interface {
-	// Set stores the given key/value pair.
-	Set(string, string)
-	// Get returns the value for the given key and a boolean indicating if the key was found.
-	Get(string) (string, bool)
-	// Exists returns true if the given key exists in the cache.
-	Exists(string) bool
-	// Delete the given key from the cache.
-	Delete(string)
-	// Clear all key/value pairs from the cache.
-	Clear()
-	// Count the number of key/value pairs in the cache.
-	Count() int
-	// Contents returns all keys in the cache encoded as a string.
-	Contents() string
-}
@@ -1,81 +0,0 @@
-package memory
-
-import (
-	"strings"
-	"time"
-
-	"github.com/patrickmn/go-cache"
-)
-
-const (
-	expirationInterval = 12 * time.Hour
-	purgeInterval      = 13 * time.Hour
-	defaultExpiration  = cache.DefaultExpiration
-)
-
-// Cache is a wrapper around the go-cache library.
-type Cache struct {
-	c *cache.Cache
-}
-
-// New constructs a new in-memory cache.
-func New() *Cache {
-	c := cache.New(expirationInterval, purgeInterval)
-	return &Cache{c: c}
-}
-
-// NewWithData constructs a new in-memory cache with existing data.
-func NewWithData(data []string) *Cache {
-	items := make(map[string]cache.Item, len(data))
-	for _, d := range data {
-		items[d] = cache.Item{Object: d, Expiration: int64(defaultExpiration)}
-	}
-
-	c := cache.NewFrom(expirationInterval, purgeInterval, items)
-	return &Cache{c: c}
-}
-
-// Set adds a key-value pair to the cache.
-func (c *Cache) Set(key, value string) {
-	c.c.Set(key, value, defaultExpiration)
-}
-
-// Get returns the value for the given key.
-func (c *Cache) Get(key string) (string, bool) {
-	res, ok := c.c.Get(key)
-	if !ok {
-		return "", ok
-	}
-	return res.(string), ok
-}
-
-// Exists returns true if the given key exists in the cache.
-func (c *Cache) Exists(key string) bool {
-	_, ok := c.c.Get(key)
-	return ok
-}
-
-// Delete removes the key-value pair from the cache.
-func (c *Cache) Delete(key string) {
-	c.c.Delete(key)
-}
-
-// Clear removes all key-value pairs from the cache.
-func (c *Cache) Clear() {
-	c.c.Flush()
-}
-
-// Count returns the number of key-value pairs in the cache.
-func (c *Cache) Count() int {
-	return c.c.ItemCount()
-}
-
-// Contents returns all key-value pairs in the cache encodes as a string.
-func (c *Cache) Contents() string {
-	items := c.c.Items()
-	res := make([]string, 0, len(items))
-	for k := range items {
-		res = append(res, k)
-	}
-	return strings.Join(res, ",")
-}
@@ -1,151 +0,0 @@
-package memory
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-)
-
-func TestCache(t *testing.T) {
-	c := New()
-
-	// Test set and get.
-	c.Set("key1", "key1")
-	v, ok := c.Get("key1")
-	if !ok || v != "key1" {
-		t.Fatalf("Unexpected value for key1: %v, %v", v, ok)
-	}
-
-	// Test exists.
-	if !c.Exists("key1") {
-		t.Fatalf("Expected key1 to exist")
-	}
-
-	// Test the count.
-	if c.Count() != 1 {
-		t.Fatalf("Unexpected count: %d", c.Count())
-	}
-
-	// Test delete.
-	c.Delete("key1")
-	v, ok = c.Get("key1")
-	if ok || v != "" {
-		t.Fatalf("Unexpected value for key1 after delete: %v, %v", v, ok)
-	}
-
-	// Test clear.
-	c.Set("key10", "key10")
-	c.Clear()
-	v, ok = c.Get("key10")
-	if ok || v != "" {
-		t.Fatalf("Unexpected value for key10 after clear: %v, %v", v, ok)
-	}
-
-	// Test contents.
-	keys := []string{"key1", "key2", "key3"}
-	for _, k := range keys {
-		c.Set(k, k)
-	}
-
-	items := c.Contents()
-	sort.Strings(keys)
-	res := strings.Split(items, ",")
-	sort.Strings(res)
-
-	if len(keys) != len(res) {
-		t.Fatalf("Unexpected length of items: %d", len(res))
-	}
-	if !cmp.Equal(keys, res) {
-		t.Fatalf("Unexpected items: %v", res)
-	}
-}
-
-func TestCache_NewWithData(t *testing.T) {
-	c := NewWithData([]string{"key1", "key2", "key3"})
-
-	// Test the count.
-	if c.Count() != 3 {
-		t.Fatalf("Unexpected count: %d", c.Count())
-	}
-
-	// Test contents.
-	keys := []string{"key1", "key2", "key3"}
-	items := c.Contents()
-	sort.Strings(keys)
-	res := strings.Split(items, ",")
-	sort.Strings(res)
-
-	if len(keys) != len(res) {
-		t.Fatalf("Unexpected length of items: %d", len(res))
-	}
-	if !cmp.Equal(keys, res) {
-		t.Fatalf("Unexpected items: %v", res)
-	}
-}
-
-func setupBenchmarks(b *testing.B) *Cache {
-	b.Helper()
-
-	c := New()
-
-	for i := 0; i < 500_000; i++ {
-		key := fmt.Sprintf("key%d", i)
-		c.Set(key, key)
-	}
-
-	return c
-}
-
-func BenchmarkSet(b *testing.B) {
-	c := New()
-
-	for i := 0; i < b.N; i++ {
-		key := fmt.Sprintf("key%d", i)
-		c.Set(key, key)
-	}
-}
-
-func BenchmarkGet(b *testing.B) {
-	c := setupBenchmarks(b)
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		key := fmt.Sprintf("key%d", i)
-		c.Get(key)
-	}
-}
-
-func BenchmarkDelete(b *testing.B) {
-	c := setupBenchmarks(b)
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		key := fmt.Sprintf("key%d", i)
-		c.Delete(key)
-	}
-}
-
-func BenchmarkCount(b *testing.B) {
-	c := setupBenchmarks(b)
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		c.Count()
-	}
-}
-
-func BenchmarkContents(b *testing.B) {
-	c := setupBenchmarks(b)
-	b.ResetTimer()
-
-	var s string
-
-	for i := 0; i < b.N; i++ {
-		s = c.Contents()
-	}
-
-	_ = s
-}
@@ -15,6 +15,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"
 )

 type scanner struct {
@@ -179,10 +180,12 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						err := json.NewDecoder(res.Body).Decode(&identityInfo)
 						if err == nil {
 							s1.Verified = true
-							s1.ExtraData = map[string]string{
-								"account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
-								"user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
-								"arn":     identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
+							s1.ExtraData = &structpb.Struct{
+								Fields: map[string]*structpb.Value{
+									"account": structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account),
+									"user_id": structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID),
+									"arn":     structpb.NewStringValue(identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn),
+								},
 							}
 						}
 						res.Body.Close()
@@ -15,6 +15,8 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
 )

 func TestAWS_FromChunk(t *testing.T) {
@@ -56,11 +58,13 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
-					Redacted:     "AKIAWARWQKZNHMZBLY4I",
-					ExtraData: map[string]string{
-						"account": "413504919130",
-						"arn":     "arn:aws:iam::413504919130:root",
-						"user_id": "413504919130",
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
+					ExtraData: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							"account": structpb.NewStringValue("171436882533"),
+							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
+							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
+						},
 					},
 				},
 			},
@@ -78,7 +82,7 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
-					Redacted:     "AKIAWARWQKZNHMZBLY4I",
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
 					ExtraData:    nil,
 				},
 			},
@@ -108,10 +112,12 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
 					Redacted:     id,
-					ExtraData: map[string]string{
-						"account": "413504919130",
-						"arn":     "arn:aws:iam::413504919130:root",
-						"user_id": "413504919130",
+					ExtraData: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							"account": structpb.NewStringValue("171436882533"),
+							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
+							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
+						},
 					},
 				},
 				{
@@ -146,10 +152,12 @@ func TestAWS_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
 					Redacted:     id,
-					ExtraData: map[string]string{
-						"account": "413504919130",
-						"arn":     "arn:aws:iam::413504919130:root",
-						"user_id": "413504919130",
+					ExtraData: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							"account": structpb.NewStringValue("171436882533"),
+							"arn":     structpb.NewStringValue("arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793"),
+							"user_id": structpb.NewStringValue("AIDASP2TPHJSUFRSTTZX4"),
+						},
 					},
 				},
 				{
@@ -172,7 +180,7 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
-					Redacted:     "AKIAWARWQKZNHMZBLY4I",
+					Redacted:     "AKIASP2TPHJSQH3FJRUX",
 				},
 			},
 			wantErr: false,
@@ -181,7 +189,7 @@ func TestAWS_FromChunk(t *testing.T) {
 			name: "skipped",
 			s: scanner{
 				skipIDs: map[string]struct{}{
-					"AKIAWARWQKZNHMZBLY4I": {},
+					"AKIASP2TPHJSQH3FJRUX": {},
 				},
 			},
 			args: args{
@@ -201,11 +209,16 @@ func TestAWS_FromChunk(t *testing.T) {
 				return
 			}
 			for i := range got {
+				if tt.want[i].ExtraData != nil {
+					if !proto.Equal(got[i].ExtraData, tt.want[i].ExtraData) {
+						t.Errorf("AWS.FromData() %s extra data not equal: got %+v, want %+v", tt.name, got[i].ExtraData, tt.want[i].ExtraData)
+					}
+				}
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 			}
-			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw")
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "ExtraData")
 			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
 				t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
@@ -12,6 +12,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"google.golang.org/protobuf/types/known/structpb"
 )

 // Detector defines an interface for scanning for and verifying secrets.
@@ -45,7 +46,7 @@ type Result struct {
 	// Redacted contains the redacted version of the raw secret identification data for display purposes.
 	// A secret ID should be used if available.
 	Redacted       string
-	ExtraData      map[string]string
+	ExtraData      *structpb.Struct
 	StructuredData *detectorspb.StructuredData
 }

@@ -3,7 +3,6 @@ package fastlypersonaltoken
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
 	"net/http"
 	"regexp"
@@ -12,6 +11,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"
 )

 type Scanner struct{}
@@ -76,12 +76,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						continue
 					}
 					s1.Verified = true
-					s1.ExtraData = map[string]string{
-						"username":                userRes.Login,
-						"name":                    userRes.Name,
-						"role":                    userRes.Role,
-						"locked":                  fmt.Sprintf("%t", userRes.Locked),
-						"two_factor_auth_enabled": fmt.Sprintf("%t", userRes.TwoFactorAuthEnabled),
+					s1.ExtraData = &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							"username":                structpb.NewStringValue(userRes.Login),
+							"name":                    structpb.NewStringValue(userRes.Name),
+							"role":                    structpb.NewStringValue(userRes.Role),
+							"locked":                  structpb.NewBoolValue(userRes.Locked),
+							"two_factor_auth_enabled": structpb.NewBoolValue(userRes.TwoFactorAuthEnabled),
+						},
 					}
 				} else {
 					if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
@@ -95,6 +95,7 @@ func TestFastlyPersonalToken_FromChunk(t *testing.T) {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
+				got[i].ExtraData = nil
 			}
 			if diff := pretty.Compare(got, tt.want); diff != "" {
 				t.Errorf("FastlyPersonalToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
@@ -9,6 +9,7 @@ import (

 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
@@ -85,13 +86,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					res.Body.Close()
 					if err == nil {
 						s1.Verified = true
-						s1.ExtraData = map[string]string{
-							"username":     userResponse.Login,
-							"url":          userResponse.UserURL,
-							"account_type": userResponse.Type,
-							"site_admin":   fmt.Sprintf("%t", userResponse.SiteAdmin),
-							"name":         userResponse.Name,
-							"company":      userResponse.Company,
+						s1.ExtraData = &structpb.Struct{
+							Fields: map[string]*structpb.Value{
+								"username":     structpb.NewStringValue(userResponse.Login),
+								"url":          structpb.NewStringValue(userResponse.UserURL),
+								"account_type": structpb.NewStringValue(userResponse.Type),
+								"site_admin":   structpb.NewBoolValue(userResponse.SiteAdmin),
+								"name":         structpb.NewStringValue(userResponse.Name),
+								"company":      structpb.NewStringValue(userResponse.Company),
+							},
 						}
 					}
 				}
@@ -6,10 +6,10 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
-	"strconv"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 )
@@ -83,15 +83,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					if err == nil {
 						s1.Verified = true
 						org := orgs.Data[0]
-						s1.ExtraData = map[string]string{
-							"id":          org.Id,
-							"title":       org.Title,
-							"user":        org.User,
-							"description": org.Description,
-							"role":        org.Role,
-							"is_personal": strconv.FormatBool(org.Personal),
-							"is_default":  strconv.FormatBool(org.Default),
-							"total_orgs":  fmt.Sprintf("%d", len(orgs.Data)),
+						s1.ExtraData = &structpb.Struct{
+							Fields: map[string]*structpb.Value{
+								"id":          structpb.NewStringValue(org.Id),
+								"title":       structpb.NewStringValue(org.Title),
+								"user":        structpb.NewStringValue(org.User),
+								"description": structpb.NewStringValue(org.Description),
+								"role":        structpb.NewStringValue(org.Role),
+								"is_personal": structpb.NewBoolValue(org.Personal),
+								"is_default":  structpb.NewBoolValue(org.Default),
+								"total_orgs":  structpb.NewNumberValue(float64(len(orgs.Data))),
+							},
 						}
 					}
 				}
@@ -69,16 +69,6 @@ func TestOpenAI_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_OpenAI,
 					Verified:     true,
 					Redacted:     "sk-...xxxx",
-					ExtraData: map[string]string{
-						"description": "",
-						"id":          "",
-						"is_default":  "",
-						"is_personal": "",
-						"role":        "",
-						"title":       "",
-						"total_orgs":  "",
-						"user":        "",
-					},
 				},
 			},
 			wantErr: false,
@@ -10,6 +10,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+	"google.golang.org/protobuf/types/known/structpb"
 )

 type Scanner struct{}
@@ -62,14 +63,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 						shopifyTokenAccessScopes := shopifyTokenAccessScopes{}
 						err := json.NewDecoder(res.Body).Decode(&shopifyTokenAccessScopes)
 						if err == nil {
-							var handleArray []string
+							var handleArray []*structpb.Value
 							for _, handle := range shopifyTokenAccessScopes.AccessScopes {
-								handleArray = append(handleArray, handle.Handle)
-
+								handleArray = append(handleArray, structpb.NewStringValue(handle.Handle))
 							}
 							s1.Verified = true
-							s1.ExtraData = map[string]string{
-								"access_scopes": strings.Join(handleArray, ","),
+							s1.ExtraData = &structpb.Struct{
+								Fields: map[string]*structpb.Value{
+									"access_scopes": structpb.NewListValue(&structpb.ListValue{Values: handleArray}),
+								},
 							}
 						}
 						res.Body.Close()
@@ -6,8 +6,11 @@ import (
 	"testing"
 	"time"

-	"github.com/kylelemons/godebug/pretty"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"

 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -49,8 +52,13 @@ func TestShopify_FromChunk(t *testing.T) {
 					DetectorType: detectorspb.DetectorType_Shopify,
 					Redacted:     domain,
 					Verified:     true,
-					ExtraData: map[string]string{
-						"access_scopes": "read_analytics,unauthenticated_read_selling_plans",
+					ExtraData: &structpb.Struct{
+						Fields: map[string]*structpb.Value{
+							"access_scopes": structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{
+								structpb.NewStringValue("read_analytics"),
+								structpb.NewStringValue("unauthenticated_read_selling_plans"),
+							}}),
+						},
 					},
 				},
 			},
@@ -95,13 +103,19 @@ func TestShopify_FromChunk(t *testing.T) {
 				return
 			}
 			for i := range got {
+				if tt.want[i].ExtraData != nil {
+					if !proto.Equal(got[i].ExtraData, tt.want[i].ExtraData) {
+						t.Errorf("AWS.FromData() %s extra data not equal: got %+v, want %+v", tt.name, got[i].ExtraData, tt.want[i].ExtraData)
+					}
+				}
 				if len(got[i].Raw) == 0 {
 					t.Fatalf("no raw secret present: \n %+v", got[i])
 				}
 				got[i].Raw = nil
 			}
-			if diff := pretty.Compare(got, tt.want); diff != "" {
-				t.Errorf("Shopify.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+			ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "ExtraData")
+			if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
+				t.Errorf("AWS.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
 			}
 		})
 	}
@@ -64,7 +64,34 @@ func (m *Result) validate(all bool) error {

 	// no validation rules for Hash

-	// no validation rules for ExtraData
+	if all {
+		switch v := interface{}(m.GetExtraData()).(type) {
+		case interface{ ValidateAll() error }:
+			if err := v.ValidateAll(); err != nil {
+				errors = append(errors, ResultValidationError{
+					field:  "ExtraData",
+					reason: "embedded message failed validation",
+					cause:  err,
+				})
+			}
+		case interface{ Validate() error }:
+			if err := v.Validate(); err != nil {
+				errors = append(errors, ResultValidationError{
+					field:  "ExtraData",
+					reason: "embedded message failed validation",
+					cause:  err,
+				})
+			}
+		}
+	} else if v, ok := interface{}(m.GetExtraData()).(interface{ Validate() error }); ok {
+		if err := v.Validate(); err != nil {
+			return ResultValidationError{
+				field:  "ExtraData",
+				reason: "embedded message failed validation",
+				cause:  err,
+			}
+		}
+	}

 	if all {
 		switch v := interface{}(m.GetStructuredData()).(type) {
@@ -1,12 +1,12 @@
 package gcs

 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
-	"strings"
+	"sort"
 	"sync"
-	"sync/atomic"

 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
 	"github.com/go-errors/errors"
@@ -14,7 +14,6 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"

-	"github.com/trufflesecurity/trufflehog/v3/pkg/cache"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -22,6 +21,23 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )

+// Source represents a GCS source.
+type Source struct {
+	name        string
+	jobId       int64
+	sourceId    int64
+	concurrency int
+	verify      bool
+
+	gcsManager objectManager
+	stats      *attributes
+	log        logr.Logger
+	chunksCh   chan *sources.Chunk
+
+	mu               sync.Mutex
+	sources.Progress // not safe for concurrent access.
+}
+
 // Ensure the Source satisfies the interface at compile time.
 var _ sources.Source = (*Source)(nil)

@@ -41,25 +57,6 @@ func (s *Source) JobID() int64 {
 	return s.jobId
 }

-// Source represents a GCS source.
-type Source struct {
-	name        string
-	jobId       int64
-	sourceId    int64
-	concurrency int
-	verify      bool
-
-	gcsManager objectManager
-	stats      *attributes
-	log        logr.Logger
-	chunksCh   chan *sources.Chunk
-
-	processedObjects int32
-	cache            cache.Cache
-
-	sources.Progress
-}
-
 // Init returns an initialized GCS source.
 func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.log = aCtx.Logger()
@@ -76,30 +73,6 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}

-	gcsManager, err := configureGCSManager(aCtx, &conn, concurrency)
-	if err != nil {
-		return err
-	}
-	s.gcsManager = gcsManager
-
-	s.log.V(2).Info("enumerating buckets and objects")
-	if err := s.enumerate(aCtx); err != nil {
-		return fmt.Errorf("error enumerating buckets and objects: %w", err)
-	}
-
-	// For runs that aren't associated with a job, we don't need progress or a cache.
-	if id != 0 {
-		s.cache = configureCache(aCtx, &s.Progress)
-	}
-
-	return nil
-}
-
-func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency int) (*gcsManager, error) {
-	if conn == nil {
-		return nil, fmt.Errorf("GCS connection is nil, cannot configure GCS manager")
-	}
-
 	var gcsManagerAuthOption gcsManagerOption

 	switch conn.Credential.(type) {
@@ -108,7 +81,7 @@ func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency
 	case *sourcespb.GCS_ServiceAccountFile:
 		b, err := os.ReadFile(conn.GetServiceAccountFile())
 		if err != nil {
-			return nil, fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
+			return fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
 		}
 		gcsManagerAuthOption = withJSONServiceAccount(aCtx, b)
 	case *sourcespb.GCS_JsonServiceAccount:
@@ -118,28 +91,39 @@ func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency
 	case *sourcespb.GCS_Unauthenticated:
 		gcsManagerAuthOption = withoutAuthentication()
 	default:
-		return nil, fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)
+		return fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)

 	}

+	resume, err := setResumeBucketOffset(s.Progress.EncodedResumeInfo)
+	if err != nil {
+		return fmt.Errorf("error setting resume info: %w", err)
+	}
+
 	gcsManagerOpts := []gcsManagerOption{
 		withConcurrency(concurrency),
+		withBucketOffsets(resume),
 		withMaxObjectSize(conn.MaxObjectSize),
 		gcsManagerAuthOption,
 	}
-	if setGCSManagerBucketOptions(conn) != nil {
-		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(conn))
+	if setGCSManagerBucketOptions(&conn) != nil {
+		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(&conn))
 	}
-	if setGCSManagerObjectOptions(conn) != nil {
-		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(conn))
+	if setGCSManagerObjectOptions(&conn) != nil {
+		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(&conn))
 	}

-	gcsManager, err := newGCSManager(conn.ProjectId, gcsManagerOpts...)
-	if err != nil {
-		return nil, fmt.Errorf("error creating GCS manager: %w", err)
+	if s.gcsManager, err = newGCSManager(conn.ProjectId, gcsManagerOpts...); err != nil {
+		return fmt.Errorf("error creating GCS manager: %w", err)
 	}

-	return gcsManager, nil
+	if len(resume) > 0 {
+		s.log.V(2).Info("resuming from previous run")
+		return nil
+	}
+
+	s.log.V(2).Info("new run, enumerating buckets and objects")
+	return s.enumerate(aCtx)
 }

 func setGCSManagerBucketOptions(conn *sourcespb.GCS) gcsManagerOption {
@@ -163,13 +147,140 @@ func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([
 	return nil
 }

-func configureCache(ctx context.Context, p *sources.Progress) cache.Cache {
-	if p.EncodedResumeInfo != "" {
-		ctx.Logger().V(3).Info("creating storage service with existing progress")
-		return newPersistableCache(p, withMemoryLoadedPersistableCache(strings.Split(p.EncodedResumeInfo, ",")))
+type progressInfo struct {
+	processedBucketCnt,
+	totalBucketsCnt,
+	processedObjectsCnt,
+	totalObjectsCnt int
+
+	// bucketObjects tracks processing and processed objects for each bucket.
+	bucketObjects map[string]*objectsProgress
+}
+
+// objectsProgress keeps track of the progress of processing objects in a bucket.
+// It is marshalled/unmarshalled to/from a string as part of the Source's Progress.
+type objectsProgress struct {
+	// IsBuckedProcessed is true if all objects in the bucket have been processed.
+	IsBucketProcessed bool
+	// ProcessedCnt is the number of objects that have been processed.
+	ProcessedCnt,
+	// TotalCnt is the total number of objects in the bucket.
+	TotalCnt int
+	// Processed is the last object that was fully processed.
+	Processed string
+	// Processing are the objects that were not fully processed.
+	Processing map[string]struct{}
+}
+
+func newObjectsProgress(cnt int) *objectsProgress {
+	return &objectsProgress{TotalCnt: cnt, Processing: map[string]struct{}{}}
+}
+
+// newProgressInfo constructs a progressInfo from the Source's Progress information.
+func newProgressInfo(ctx context.Context, s *sources.Progress) (*progressInfo, error) {
+	totalObjCnt := s.SectionsRemaining
+	processedObjCnt := s.SectionsCompleted
+	info := &progressInfo{
+		totalObjectsCnt:     int(totalObjCnt),
+		processedObjectsCnt: int(processedObjCnt),
 	}

-	return newPersistableCache(p, withMemoryPersistableCache())
+	encodeResume := s.EncodedResumeInfo
+	if encodeResume == "" {
+		info.bucketObjects = make(map[string]*objectsProgress)
+		return info, nil
+	}
+
+	var m map[string]*objectsProgress
+	if err := json.Unmarshal([]byte(encodeResume), &m); err != nil {
+		ctx.Logger().V(1).Info("error unmarshalling progress info", "error", err)
+		return info, nil
+	}
+	info.bucketObjects = m
+
+	return info, nil
+}
+
+type progressStateFn func(string, string, *progressInfo)
+
+func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {
+	fn(obj.bucket, obj.name, p)
+}
+
+func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {
+	progress.processing(bucket, obj)
+}
+
+func (p *progressInfo) processing(bkt, obj string) {
+	p.bucketObjects[bkt].Processing[obj] = struct{}{}
+}
+
+func setProcessedBucketObject(bucket, obj string, progress *progressInfo) {
+	progress.processed(bucket, obj)
+}
+
+func (p *progressInfo) processed(bkt, obj string) {
+	// Remove the object from the processing list and increment the processed count.
+	delete(p.bucketObjects[bkt].Processing, obj)
+
+	p.bucketObjects[bkt].ProcessedCnt++
+	// If all objects in the bucket have been processed, mark the bucket as processed.
+	if p.bucketObjects[bkt].ProcessedCnt == p.bucketObjects[bkt].TotalCnt {
+		p.bucketObjects[bkt].IsBucketProcessed = true
+		p.processedBucketCnt++
+	}
+	p.processedObjectsCnt++
+
+	// If the object is not greater (lexicographically) than the last processed object, we can skip it.
+	// This ensures we keep the greatest object name as the last processed object.
+	if obj < p.bucketObjects[bkt].Processed {
+		return
+	}
+	p.bucketObjects[bkt].Processed = obj
+}
+
+func setResumeBucketOffset(s string) (map[string]offsetInfo, error) {
+	resumeInfo := progressInfo{}
+	if s != "" {
+		if err := json.Unmarshal([]byte(s), &resumeInfo.bucketObjects); err != nil {
+			return nil, fmt.Errorf("error unmarshalling resume info: %w", err)
+		}
+	}
+
+	return calcBktOffset(resumeInfo.bucketObjects)
+}
+
+// In order to calculate the bucket offset, we need to know the last object
+// that was processed for each bucket, as well as any objects that were processing.
+// If the bucket was fully processed, we don't need to set an offset.
+// If there were no objects processing, we can just use the last object that was processed.
+// If there were objects processing, we need to find the first object (lexicographically)
+// that was processing, as that is the next object that needs to be processed.
+func calcBktOffset(resumeInfo map[string]*objectsProgress) (map[string]offsetInfo, error) {
+	bucketOffset := make(map[string]offsetInfo, len(resumeInfo))
+	for objName, progress := range resumeInfo {
+		info := offsetInfo{}
+		if progress.IsBucketProcessed {
+			info.isBucketProcessed = true
+			bucketOffset[objName] = info
+			continue
+		}
+		if len(progress.Processing) == 0 {
+			info.lastProcessedObject = progress.Processed
+			bucketOffset[objName] = info
+			continue
+		}
+
+		processing := make([]string, 0, len(progress.Processing))
+		for k := range progress.Processing {
+			processing = append(processing, k)
+		}
+		sort.Strings(processing)
+		info.lastProcessedObject = processing[0]
+		bucketOffset[objName] = info
+	}
+
+	return bucketOffset, nil
 }

 // enumerate all the objects and buckets in the source and use the results to
@@ -182,7 +293,18 @@ func (s *Source) enumerate(ctx context.Context) error {
 	}
 	s.stats = stats

-	return nil
+	info := progressInfo{
+		totalBucketsCnt: int(s.stats.numBuckets),
+		totalObjectsCnt: int(s.stats.numObjects),
+	}
+
+	progress := make(map[string]*objectsProgress, len(s.stats.bucketObjects))
+	for k, v := range s.stats.bucketObjects {
+		progress[k] = newObjectsProgress(int(v))
+	}
+	info.bucketObjects = progress
+
+	return s.setProgress(ctx, &info, fmt.Sprintf("enumerated %d buckets, %d objects", s.stats.numBuckets, s.stats.numObjects))
 }

 // Chunks emits chunks of bytes over a channel.
@@ -192,7 +314,11 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		return fmt.Errorf("error listing objects: %w", err)
 	}
 	s.chunksCh = chunksChan
-	s.Progress.Message = "starting to process objects..."
+
+	progress, err := newProgressInfo(ctx, &s.Progress)
+	if err != nil {
+		return fmt.Errorf("error constructing progress info: %w", err)
+	}

 	var wg sync.WaitGroup
 	for obj := range objectCh {
@@ -203,46 +329,55 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			continue
 		}

-		if s.cache != nil && s.cache.Exists(o.md5) {
-			ctx.Logger().V(5).Info("skipping object, object already processed", "name", o.name)
+		if err := s.startProcessing(ctx, progress, o); err != nil {
+			ctx.Logger().V(1).Info("GCS source error starting to process objects", "error", err)
 			continue
 		}
-
 		wg.Add(1)
-		go func(obj object) {
+		go func(obj object, resume *progressInfo) {
 			defer wg.Done()

 			if err := s.processObject(ctx, o); err != nil {
 				ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)
 				return
 			}
-			if s.cache != nil {
-				s.setProgress(ctx, o.name, o.md5)
+			if err := s.endProcessing(ctx, resume, o); err != nil {
+				ctx.Logger().V(1).Info("error setting end progress progress", "name", o.name, "error", err)
 			}
-		}(o)
+		}(o, progress)
 	}
 	wg.Wait()

-	s.completeProgress(ctx)
+	ctx.Logger().Info("GCS source finished processing", "name", s.name)
 	return nil
 }

-func (s *Source) setProgress(ctx context.Context, objName, md5 string) {
-	atomic.AddInt32(&s.processedObjects, 1)
-	ctx.Logger().V(5).Info("setting progress for object", "object-name", objName, "md5", md5)
+func (s *Source) startProcessing(ctx context.Context, progress *progressInfo, o object) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()

-	s.cache.Set(md5, md5)
-	processed := atomic.LoadInt32(&s.processedObjects)
-	s.Progress.SectionsCompleted = processed
-	s.Progress.SectionsRemaining = int32(s.stats.numObjects)
-	s.Progress.PercentComplete = int64(float64(processed) / float64(s.stats.numObjects) * 100)
+	progress.setProcessStatus(o, setProcessingBucketObject)
+	return s.setProgress(ctx, progress, fmt.Sprintf("GCS source beginning to process object %s in bucket %s", o.name, o.bucket))
 }

-func (s *Source) completeProgress(ctx context.Context) {
-	msg := fmt.Sprintf("GCS source finished processing %d objects", s.stats.numObjects)
-	ctx.Logger().Info(msg)
-	s.Progress.Message = msg
-	s.cache.Clear()
+func (s *Source) endProcessing(ctx context.Context, progress *progressInfo, o object) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	progress.setProcessStatus(o, setProcessedBucketObject)
+	return s.setProgress(ctx, progress, fmt.Sprintf("GCS source finished processing object %s in bucket %s", o.name, o.bucket))
+}
+
+func (s *Source) setProgress(ctx context.Context, progress *progressInfo, msg string) error {
+	ctx.Logger().V(5).Info(msg)
+
+	b, err := json.Marshal(progress.bucketObjects)
+	if err != nil {
+		return fmt.Errorf("error marshalling resume info: %w", err)
+	}
+	s.SetProgressComplete(progress.processedObjectsCnt, progress.totalObjectsCnt, msg, string(b))
+
+	return nil
 }

 func (s *Source) processObject(ctx context.Context, o object) error {
@@ -2,7 +2,6 @@ package gcs

 import (
 	aCtx "context"
-	"encoding/base64"
 	"fmt"
 	"io"
 	"runtime"
@@ -358,7 +357,6 @@ type object struct {
 	contentType string
 	owner       string
 	link        string
-	md5         string
 	// acl represents an ACLEntities.
 	// https://pkg.go.dev/cloud.google.com/go/storage#ACLEntity
 	acl       []string
@@ -385,10 +383,6 @@ func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes,
 	logger.V(5).Info("enumerating buckets", "numBuckets", len(bkts))
 	stats := newStats(len(bkts))

-	defer func(start time.Time) {
-		logger.V(5).Info("finished enumerating buckets", "duration-seconds", fmt.Sprintf("%.1f", time.Since(start).Seconds()), "num-buckets", len(bkts), "num-objects", stats.numObjects)
-	}(time.Now())
-
 	for _, bkt := range bkts {
 		bkt := bkt
 		g.workerPool.Go(func() error {
@@ -432,6 +426,7 @@ func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes,

 	_ = g.workerPool.Wait()
 	g.attr = stats
+	logger.V(5).Info("finished enumerating buckets", "num-objects", stats.numObjects, "num-buckets", stats.numBuckets)

 	return stats, nil
 }
@@ -636,7 +631,6 @@ func (g *gcsManager) constructObject(ctx context.Context, obj *storage.ObjectHan
 	o.contentType = attrs.ContentType
 	o.owner = attrs.Owner
 	o.link = attrs.MediaLink
-	o.md5 = base64.StdEncoding.EncodeToString(attrs.MD5[:])
 	o.createdAt = attrs.Created
 	o.updatedAt = attrs.Updated
 	o.acl = objectACLs(attrs.ACL)
@@ -473,7 +473,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
 					acl:         []string{},
-					md5:         "ek30X4Oc1ZSsI2sZUkzskg==",
 				},
 				{
 					name:        "moar2.txt",
@@ -482,7 +481,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        12,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
 					acl:         []string{},
-					md5:         "5Z/5eUEET4XfUpfhwwLSYA==",
 				},
 				{
 					name:        "aws3.txt",
@@ -491,7 +489,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
 					acl:         []string{},
-					md5:         "JusDN4oMjpnuDOz5FVrOgQ==",
 				},
 				{
 					name:        "moar.txt",
@@ -500,7 +497,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        6,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
 					acl:         []string{},
-					md5:         "CffgLxKQviEdpweiZvFTsw==",
 				},
 				{
 					name:        "AMAZON_FASHION_5.json",
@@ -509,7 +505,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        1413469,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
 					acl:         []string{},
-					md5:         "Wh2PSDqjRpUlY3kFDUKitw==",
 				},
 			},
 			wantNumBkt: 4,
@@ -527,7 +522,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
 					acl:         []string{},
-					md5:         "ek30X4Oc1ZSsI2sZUkzskg==",
 				},
 				{
 					name:        "moar2.txt",
@@ -536,7 +530,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        12,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
 					acl:         []string{},
-					md5:         "5Z/5eUEET4XfUpfhwwLSYA==",
 				},
 			},
 			wantNumBkt: 1,
@@ -554,7 +547,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        6,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
 					acl:         []string{},
-					md5:         "CffgLxKQviEdpweiZvFTsw==",
 				},
 				{
 					name:        "AMAZON_FASHION_5.json",
@@ -563,7 +555,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        1413469,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
 					acl:         []string{},
-					md5:         "Wh2PSDqjRpUlY3kFDUKitw==",
 				},
 			},
 			wantNumBkt: 2,
@@ -585,7 +576,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
 					acl:         []string{},
-					md5:         "JusDN4oMjpnuDOz5FVrOgQ==",
 				},
 				{
 					name:        "moar.txt",
@@ -594,7 +584,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        6,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
 					acl:         []string{},
-					md5:         "CffgLxKQviEdpweiZvFTsw==",
 				},
 				{
 					name:        "AMAZON_FASHION_5.json",
@@ -603,7 +592,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        1413469,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
 					acl:         []string{},
-					md5:         "Wh2PSDqjRpUlY3kFDUKitw==",
 				},
 			},
 			wantNumBkt: 4, // We still list objects in all buckets.
@@ -625,7 +613,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
 					acl:         []string{},
-					md5:         "ek30X4Oc1ZSsI2sZUkzskg==",
 				},
 			},
 			wantNumBkt: 1,
@@ -647,7 +634,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
 					acl:         []string{},
-					md5:         "ek30X4Oc1ZSsI2sZUkzskg==",
 				},
 			},
 			wantNumBkt: 4,
@@ -670,7 +656,6 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        12,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
 					acl:         []string{},
-					md5:         "5Z/5eUEET4XfUpfhwwLSYA==",
 				},
 			},
 			wantNumBkt: 1,
@@ -758,7 +743,6 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {
 					size:        12,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
 					acl:         []string{},
-					md5:         "5Z/5eUEET4XfUpfhwwLSYA==",
 				},
 			},
 			wantNumBkt: 1,
@@ -1,10 +1,10 @@
 package gcs

 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
-	"strings"
 	"testing"

 	"github.com/google/go-cmp/cmp"
@@ -31,7 +31,7 @@ func TestSourceInit(t *testing.T) {
 	source, conn := createTestSource(&sourcespb.GCS{
 		ProjectId: testProjectID,
 		IncludeBuckets: []string{
-			publicBucket,
+			"bucket1",
 		},
 		ExcludeBuckets: []string{
 			perfTestBucketGlob,
@@ -39,46 +39,22 @@ func TestSourceInit(t *testing.T) {
 		ExcludeObjects: []string{
 			"object1",
 		},
-		Credential: &sourcespb.GCS_Unauthenticated{},
+		Credential: &sourcespb.GCS_ApiKey{
+			ApiKey: testAPIKey,
+		},
 	})

-	testCases := []struct {
-		name      string
-		jobID     int64
-		conn      *anypb.Any
-		source    *Source
-		wantGCS   bool
-		wantCache bool
-	}{
-		{
-			name:      "Non-cacheable GCS source init",
-			jobID:     0,
-			conn:      conn,
-			source:    source,
-			wantGCS:   true,
-			wantCache: false,
-		},
-		{
-			name:      "Cacheable GCS source init",
-			jobID:     1,
-			conn:      conn,
-			source:    source,
-			wantGCS:   true,
-			wantCache: true,
-		},
-	}
+	// Set the EncodeResumeInfo to prevent enumeration.
+	b, err := json.Marshal(map[string]objectsProgress{"other": {}})
+	assert.Nil(t, err)
+	source.Progress.EncodedResumeInfo = string(b)

-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			err := tc.source.Init(context.Background(), "test", tc.jobID, 1, true, tc.conn, 8)
-			assert.Nil(t, err)
-			assert.Equal(t, tc.wantGCS, tc.source.gcsManager != nil)
-			assert.Equal(t, tc.wantCache, tc.source.cache != nil)
-		})
-	}
+	err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+	assert.Nil(t, err)
+	assert.NotNil(t, source.gcsManager)
 }

-func TestConfigureGCSManager(t *testing.T) {
+func TestSourceInit_Conn(t *testing.T) {
 	testCases := []struct {
 		name    string
 		conn    *sourcespb.GCS
@@ -101,6 +77,7 @@ func TestConfigureGCSManager(t *testing.T) {
 			want: &gcsManager{
 				projectID:      testProjectID,
 				excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
+				hasEnumerated:  true,
 			},
 		},
 		{
@@ -118,6 +95,7 @@ func TestConfigureGCSManager(t *testing.T) {
 			want: &gcsManager{
 				projectID:      testProjectID,
 				includeBuckets: map[string]struct{}{"bucket1": {}},
+				hasEnumerated:  true,
 			},
 		},
 		{
@@ -139,20 +117,27 @@ func TestConfigureGCSManager(t *testing.T) {
 				projectID:      testProjectID,
 				includeObjects: map[string]struct{}{"object1": {}},
 				excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
+				hasEnumerated:  true,
 			},
 		},
 	}

 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			ctx := context.Background()
-			got, err := configureGCSManager(ctx, tc.conn, 8)
+			source, conn := createTestSource(tc.conn)
+			// Set the EncodeResumeInfo to prevent enumeration.
+			b, err := json.Marshal(map[string]objectsProgress{"other": {}})
+			assert.Nil(t, err)
+			source.Progress.EncodedResumeInfo = string(b)
+
+			err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
 			if err != nil && !tc.wantErr {
-				t.Errorf("source.configureGCSManager() error = %v", err)
+				t.Errorf("source.Init() got: %v, want: %v", err, nil)
+				return
 			}

 			if !tc.wantErr {
-				if diff := cmp.Diff(tc.want, got,
+				if diff := cmp.Diff(tc.want, source.gcsManager,
 					cmp.AllowUnexported(gcsManager{}),
 					cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize", "attr"),
 				); diff != "" {
@@ -163,6 +148,65 @@ func TestConfigureGCSManager(t *testing.T) {
 	}
 }

+func TestSourceInit_Progress(t *testing.T) {
+	complete := map[string]objectsProgress{testBucket: {
+		Processed:         object2,
+		IsBucketProcessed: true,
+		ProcessedCnt:      1,
+		TotalCnt:          1,
+	}}
+	completeEncodeResumeInfo, err := json.Marshal(complete)
+	assert.Nil(t, err)
+
+	processing := map[string]objectsProgress{
+		testBucket: {
+			IsBucketProcessed: false,
+			ProcessedCnt:      1,
+			Processed:         object2,
+			Processing:        map[string]struct{}{object1: {}},
+			TotalCnt:          2,
+		},
+	}
+	processingEncodeResumeInfo, err := json.Marshal(processing)
+	assert.Nil(t, err)
+
+	testCases := []struct {
+		name              string
+		conn              *sourcespb.GCS
+		encodeResumeInfo  string
+		gcsManagerBuckets map[string]bucket
+	}{
+		{
+			name:              "no progress",
+			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}, ExcludeBuckets: []string{perfTestBucketGlob}},
+			gcsManagerBuckets: map[string]bucket{},
+		},
+		{
+			name:              "progress set, no processing objects in any buckets",
+			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
+			encodeResumeInfo:  string(completeEncodeResumeInfo),
+			gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: "", shouldInclude: false}},
+		},
+		{
+			name:              "progress set, processing objects in some buckets",
+			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
+			encodeResumeInfo:  string(processingEncodeResumeInfo),
+			gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: object1, shouldInclude: true}},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			source, conn := createTestSource(tc.conn)
+			source.Progress.EncodedResumeInfo = tc.encodeResumeInfo
+
+			err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+			assert.Nil(t, err)
+			assert.Equal(t, tc.gcsManagerBuckets, source.gcsManager.(*gcsManager).buckets)
+		})
+	}
+}
+
 type mockObjectManager struct {
 	wantErr bool
 }
@@ -220,7 +264,6 @@ func createTestObject(id int) object {
 		link:        fmt.Sprintf("https://storage.googleapis.com/%s/%s", testBucket, fmt.Sprintf("object%d", id)),
 		acl:         []string{"authenticatedUsers"},
 		size:        42,
-		md5:         fmt.Sprintf("md5hash%d", id),
 		Reader:      &mockReader{data: []byte(fmt.Sprintf("hello world %d", id))},
 	}
 }
@@ -254,9 +297,7 @@ func TestSourceChunks_ListObjects(t *testing.T) {
 	source := &Source{
 		gcsManager: &mockObjectManager{},
 		chunksCh:   chunksCh,
-		Progress:   sources.Progress{},
 	}
-	source.cache = newPersistableCache(&source.Progress, withMemoryPersistableCache())

 	err := source.enumerate(ctx)
 	assert.Nil(t, err)
@@ -308,6 +349,11 @@ func TestSourceInit_Enumerate(t *testing.T) {
 	assert.Equal(t, uint64(5), source.stats.numObjects)
 	assert.Equal(t, uint32(1), source.stats.numBuckets)
 	assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])
+	// Ensure progress is initialized correctly.
+	p := map[string]*objectsProgress{testBucket: {TotalCnt: 5, Processing: map[string]struct{}{}}}
+	b, err := json.Marshal(p)
+	assert.Nil(t, err)
+	assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
 }

 func TestSourceChunks_ListObjects_Error(t *testing.T) {
@@ -324,14 +370,11 @@ func TestSourceChunks_ListObjects_Error(t *testing.T) {
 func TestSourceChunks_ProgressSet(t *testing.T) {
 	ctx := context.Background()
 	chunksCh := make(chan *sources.Chunk, 1)
+
 	source := &Source{
 		gcsManager: &mockObjectManager{},
 		chunksCh:   chunksCh,
-		Progress:   sources.Progress{},
 	}
-
-	source.cache = newPersistableCache(&source.Progress, withMemoryPersistableCache(), withCustomIncrement(5))
-
 	err := source.enumerate(ctx)
 	assert.Nil(t, err)

@@ -354,22 +397,26 @@ func TestSourceChunks_ProgressSet(t *testing.T) {
 	// Ensure we get 5 objects back.
 	assert.Equal(t, len(want), len(got))

-	processing := map[string]struct{}{"md5hash0": {}, "md5hash1": {}, "md5hash2": {}, "md5hash3": {}, "md5hash4": {}}
+	processing := map[string]struct{}{"object0": {}, "object1": {}, "object2": {}, "object3": {}, "object4": {}}
 	for i := 0; i < 5; i++ {
 		delete(processing, fmt.Sprintf("object%d", i))
 	}

 	// Test that the resume progress is set.
 	// The processed values should be the greatest object name lexicographically.
-	progress := "md5hash0,md5hash1,md5hash2,md5hash3,md5hash4"
-	objs := strings.Split(progress, ",")
+	resume := map[string]objectsProgress{
+		testBucket: {
+			Processed:         "object4",
+			Processing:        processing,
+			IsBucketProcessed: true,
+			ProcessedCnt:      5,
+			TotalCnt:          5,
+		},
+	}
+	b, err := json.Marshal(resume)
+	assert.Nil(t, err)

-	encodeResume := strings.Split(source.Progress.EncodedResumeInfo, ",")
-	sort.Strings(encodeResume)
-
-	assert.Equal(t, objs, encodeResume)
+	assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
 	assert.Equal(t, int32(5), source.Progress.SectionsCompleted)
 	assert.Equal(t, int64(100), source.Progress.PercentComplete)
-	assert.Equal(t, 0, source.cache.Count())
-	assert.Equal(t, fmt.Sprintf("GCS source finished processing %d objects", 5), source.Progress.Message)
 }
@@ -1,70 +0,0 @@
-package gcs
-
-import (
-	"github.com/trufflesecurity/trufflehog/v3/pkg/cache"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-)
-
-const defaultCachePersistIncrement = 2500
-
-type persistibleCacheConfiguration func(c *persistableCache)
-
-// persistableCache handles all cache operations with some additional bookkeeping.
-// The threshold value is the percentage of objects that must be processed
-// before the cache is persisted.
-type persistableCache struct {
-	persistIncrement int
-	cache.Cache
-	*sources.Progress
-}
-
-func newPersistableCache(p *sources.Progress, cfgs ...persistibleCacheConfiguration) *persistableCache {
-	pc := &persistableCache{
-		persistIncrement: defaultCachePersistIncrement,
-		Progress:         p,
-	}
-
-	for _, cfg := range cfgs {
-		cfg(pc)
-	}
-	return pc
-}
-
-// withMemoryPersistableCache configures the persistableCache to use an in-memory cache.
-func withMemoryPersistableCache() persistibleCacheConfiguration {
-	return func(pc *persistableCache) {
-		pc.Cache = memory.New()
-	}
-}
-
-// withMemoryLoadedPersistableCache configures the persistableCache to use an in-memory cache with the given contents.
-func withMemoryLoadedPersistableCache(contents []string) persistibleCacheConfiguration {
-	return func(pc *persistableCache) {
-		pc.Cache = memory.NewWithData(contents)
-	}
-}
-
-// withCustomIncrement configures the persistableCache to use the given increment.
-func withCustomIncrement(increment int) persistibleCacheConfiguration {
-	return func(pc *persistableCache) {
-		pc.persistIncrement = increment
-	}
-}
-
-// Set overrides the default Set method in order to persist the cache contents
-// on each persistIncrement.
-func (c *persistableCache) Set(key, value string) {
-	c.Cache.Set(key, value)
-
-	if ok, val := c.shouldPersist(); ok {
-		c.EncodedResumeInfo = val
-	}
-}
-
-func (c *persistableCache) shouldPersist() (bool, string) {
-	if c.Count()%c.persistIncrement != 0 {
-		return false, ""
-	}
-	return true, c.Contents()
-}
@@ -1,74 +0,0 @@
-package gcs
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-)
-
-func TestPersistableCache(t *testing.T) {
-	// Create a new Progress object for testing.
-	progress := sources.Progress{}
-
-	tests := []struct {
-		name string
-		cfgs []persistibleCacheConfiguration
-		fn   func(cache *persistableCache)
-	}{
-		{
-			name: "withMemoryPersistableCache",
-			cfgs: []persistibleCacheConfiguration{withMemoryPersistableCache()},
-			fn: func(cache *persistableCache) {
-				assert.NotNil(t, cache.Cache)
-				assert.IsType(t, &memory.Cache{}, cache.Cache)
-			},
-		},
-		{
-			name: "withMemoryLoadedPersistableCache",
-			cfgs: []persistibleCacheConfiguration{withMemoryLoadedPersistableCache([]string{"foo", "bar"})},
-			fn: func(cache *persistableCache) {
-				assert.NotNil(t, cache.Cache)
-				assert.IsType(t, &memory.Cache{}, cache.Cache)
-				assert.Contains(t, cache.Cache.Contents(), "foo,bar")
-			},
-		},
-		{
-			name: "withCustomIncrement",
-			cfgs: []persistibleCacheConfiguration{withCustomIncrement(100)},
-			fn: func(cache *persistableCache) {
-				assert.Equal(t, 100, cache.persistIncrement)
-			},
-		},
-		{
-			name: "Set",
-			cfgs: []persistibleCacheConfiguration{withMemoryPersistableCache(), withCustomIncrement(3)},
-			fn: func(cache *persistableCache) {
-				cache.Set("foo", "bar")
-				assert.Equal(t, 1, cache.Count())
-
-				cache.Set("baz", "qux")
-				assert.Equal(t, 2, cache.Count())
-
-				// The cache should not persist until the increment (3) is reached.
-				assert.False(t, progress.EncodedResumeInfo != "")
-
-				cache.Set("quux", "corge")
-				assert.Equal(t, 3, cache.Count())
-
-				// The cache should now persist.
-				assert.True(t, progress.EncodedResumeInfo != "")
-			},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			cache := newPersistableCache(&progress, test.cfgs...)
-			assert.NotNil(t, cache)
-			test.fn(cache)
-		})
-	}
-}
@@ -4,6 +4,8 @@ package detectors;

 option go_package = "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb";

+import "google/protobuf/struct.proto";
+
 enum DecoderType {
  UNKNOWN = 0;
  PLAIN = 1;
@@ -926,12 +928,11 @@ message Result {
  string redacted = 3;
  bool verified = 4;
  string hash = 5;
-  map<string, string> extra_data = 6;
+  google.protobuf.Struct extra_data = 6;
  StructuredData structured_data = 7;
  string hash_v2 = 8;
  DecoderType decoder_type = 9;
 }
-
 message StructuredData {
  repeated TlsPrivateKey tls_private_key = 1;
  repeated GitHubSSHKey github_ssh_key = 2;
Author	SHA1	Message	Date
Dustin Decker	f8284323d9	Use structpb for Extra Data to support other types	2023-03-20 10:17:05 -07:00
Dustin Decker	f52778985c	update aws test	2023-03-20 09:57:08 -07:00
Dustin Decker	77f1c28eea	update extra data type	2023-03-20 09:56:58 -07:00