rename file.

Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3 to 4.
- [Release notes](https://github.com/actions/setup-go/releases)
- [Commits](https://github.com/actions/setup-go/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/setup-go
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/lint.yml

@@ -17,7 +17,7 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: '1.18'
       - uses: actions/checkout@v3

.github/workflows/release.yml

@@ -33,7 +33,7 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: '1.18'
       - name: Run GoReleaser

.github/workflows/secrets.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v4
       with:
         go-version: '1.18'
     - name: Checkout code

.github/workflows/smoke.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v4
       with:
         go-version: '1.18'
     - name: Checkout code

.github/workflows/snifftest.yml

@@ -16,7 +16,7 @@ jobs:
     name: Run Snifftest
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v4
         with:
           go-version: '1.18'
       - uses: actions/checkout@v3

.github/workflows/test.yml

@@ -17,7 +17,7 @@ jobs:
       id-token: 'write'
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v4
       with:
         go-version: '1.18'
     - name: Checkout code
@@ -36,7 +36,7 @@ jobs:
       id-token: 'write'
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v4
       with:
         go-version: '1.18'
     - name: Checkout code

go.mod

@@ -10,7 +10,7 @@ replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff
 
 require (
 	cloud.google.com/go/secretmanager v1.10.0
-	cloud.google.com/go/storage v1.29.0
+	cloud.google.com/go/storage v1.30.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
 	github.com/TheZeroSlave/zapsentry v1.14.0
 	github.com/aws/aws-sdk-go v1.44.83
@@ -25,7 +25,7 @@ require (
 	github.com/felixge/fgprof v0.9.3
 	github.com/getsentry/sentry-go v0.19.0
 	github.com/go-errors/errors v1.4.2
-	github.com/go-git/go-git/v5 v5.6.0
+	github.com/go-git/go-git/v5 v5.6.1
 	github.com/go-ldap/ldap/v3 v3.4.4
 	github.com/go-logr/logr v1.2.3
 	github.com/go-logr/zapr v1.2.3
@@ -35,7 +35,7 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/google/go-cmp v0.5.9
 	github.com/google/go-github/v42 v42.0.0
-	github.com/googleapis/gax-go/v2 v2.7.1
+	github.com/googleapis/gax-go/v2 v2.8.0
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-retryablehttp v0.7.2
 	github.com/jlaffaye/ftp v0.1.0
@@ -45,6 +45,7 @@ require (
 	github.com/lib/pq v1.10.7
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/mholt/archiver/v4 v4.0.0-alpha.7
+	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9
 	github.com/pkg/errors v0.9.1
@@ -53,7 +54,7 @@ require (
 	github.com/stretchr/testify v1.8.2
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
 	github.com/xanzy/go-gitlab v0.81.0
-	go.mongodb.org/mongo-driver v1.11.2
+	go.mongodb.org/mongo-driver v1.11.3
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.7.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
@@ -61,7 +62,7 @@ require (
 	golang.org/x/sync v0.1.0
 	golang.org/x/text v0.8.0
 	google.golang.org/api v0.111.0
-	google.golang.org/protobuf v1.29.1
+	google.golang.org/protobuf v1.30.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
 	sigs.k8s.io/yaml v1.3.0
@@ -81,8 +82,8 @@ require (
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
-	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
@@ -94,7 +95,7 @@ require (
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
-	github.com/go-git/go-billy/v5 v5.4.0 // indirect
+	github.com/go-git/go-billy/v5 v5.4.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect

go.sum

@@ -10,8 +10,8 @@ cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB
 cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
 cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
 cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
-cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
-cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
+cloud.google.com/go/storage v1.30.0 h1:g1yrbxAWOrvg/594228pETWkOi00MLTrOWfh56veU5o=
+cloud.google.com/go/storage v1.30.0/go.mod h1:xAVretHSROm1BQX4IIsoVgJqw0LqOyX+I/O2GzRAzdE=
 github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
 github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
@@ -38,12 +38,12 @@ github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e/go.mod h1:chxPXzS
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
-github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
-github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA=
+github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
 github.com/TheZeroSlave/zapsentry v1.14.0 h1:z6H0lIyKVpdXsP5K8J11b4/+V+QCDLDslZmtXsnduMk=
 github.com/TheZeroSlave/zapsentry v1.14.0/go.mod h1:0hUR1Et5tSroEYhWOZcxE2iBqcFkuG7jGGlUmOA7XAs=
-github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
-github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
+github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ=
+github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
@@ -115,12 +115,12 @@ github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
 github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
 github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE=
-github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
+github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4=
+github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg=
 github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ=
 github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo=
-github.com/go-git/go-git/v5 v5.6.0 h1:JvBdYfcttd+0kdpuWO7KTu0FYgCf5W0t5VwkWGobaa4=
-github.com/go-git/go-git/v5 v5.6.0/go.mod h1:6nmJ0tJ3N4noMV1Omv7rC5FG3/o8Cm51TB4CJp7mRmE=
+github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk=
+github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8=
 github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs=
 github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -195,8 +195,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
 github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A=
-github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
+github.com/googleapis/gax-go/v2 v2.8.0 h1:UBtEZqx1bjXtOQ5BVTkuYghXrr3N4V123VKJK67vJZc=
+github.com/googleapis/gax-go/v2 v2.8.0/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@@ -283,6 +283,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y
 github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=
 github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
 github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
@@ -357,8 +359,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
-go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.mongodb.org/mongo-driver v1.11.3 h1:Ql6K6qYHEzB6xvu4+AU0BoRoqf9vFPcc4o7MUIdPW8Y=
+go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
@@ -383,7 +385,7 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -419,7 +421,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -463,15 +466,15 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -479,6 +482,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@@ -532,8 +536,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
-google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

pkg/cache/cache.go

@@ -0,0 +1,20 @@
+// Package cache provides an interface which can be implemented by different cache types.
+package cache
+
+// Cache is used to store key/value pairs.
+type Cache interface {
+	// Set stores the given key/value pair.
+	Set(string, string)
+	// Get returns the value for the given key and a boolean indicating if the key was found.
+	Get(string) (string, bool)
+	// Exists returns true if the given key exists in the cache.
+	Exists(string) bool
+	// Delete the given key from the cache.
+	Delete(string)
+	// Clear all key/value pairs from the cache.
+	Clear()
+	// Count the number of key/value pairs in the cache.
+	Count() int
+	// Contents returns all keys in the cache encoded as a string.
+	Contents() string
+}

pkg/cache/memory/memory.go

@@ -0,0 +1,81 @@
+package memory
+
+import (
+	"strings"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+)
+
+const (
+	expirationInterval = 12 * time.Hour
+	purgeInterval      = 13 * time.Hour
+	defaultExpiration  = cache.DefaultExpiration
+)
+
+// Cache is a wrapper around the go-cache library.
+type Cache struct {
+	c *cache.Cache
+}
+
+// New constructs a new in-memory cache.
+func New() *Cache {
+	c := cache.New(expirationInterval, purgeInterval)
+	return &Cache{c: c}
+}
+
+// NewWithData constructs a new in-memory cache with existing data.
+func NewWithData(data []string) *Cache {
+	items := make(map[string]cache.Item, len(data))
+	for _, d := range data {
+		items[d] = cache.Item{Object: d, Expiration: int64(defaultExpiration)}
+	}
+
+	c := cache.NewFrom(expirationInterval, purgeInterval, items)
+	return &Cache{c: c}
+}
+
+// Set adds a key-value pair to the cache.
+func (c *Cache) Set(key, value string) {
+	c.c.Set(key, value, defaultExpiration)
+}
+
+// Get returns the value for the given key.
+func (c *Cache) Get(key string) (string, bool) {
+	res, ok := c.c.Get(key)
+	if !ok {
+		return "", ok
+	}
+	return res.(string), ok
+}
+
+// Exists returns true if the given key exists in the cache.
+func (c *Cache) Exists(key string) bool {
+	_, ok := c.c.Get(key)
+	return ok
+}
+
+// Delete removes the key-value pair from the cache.
+func (c *Cache) Delete(key string) {
+	c.c.Delete(key)
+}
+
+// Clear removes all key-value pairs from the cache.
+func (c *Cache) Clear() {
+	c.c.Flush()
+}
+
+// Count returns the number of key-value pairs in the cache.
+func (c *Cache) Count() int {
+	return c.c.ItemCount()
+}
+
+// Contents returns all key-value pairs in the cache encodes as a string.
+func (c *Cache) Contents() string {
+	items := c.c.Items()
+	res := make([]string, 0, len(items))
+	for k := range items {
+		res = append(res, k)
+	}
+	return strings.Join(res, ",")
+}

pkg/cache/memory/memory_test.go

@@ -0,0 +1,151 @@
+package memory
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestCache(t *testing.T) {
+	c := New()
+
+	// Test set and get.
+	c.Set("key1", "key1")
+	v, ok := c.Get("key1")
+	if !ok || v != "key1" {
+		t.Fatalf("Unexpected value for key1: %v, %v", v, ok)
+	}
+
+	// Test exists.
+	if !c.Exists("key1") {
+		t.Fatalf("Expected key1 to exist")
+	}
+
+	// Test the count.
+	if c.Count() != 1 {
+		t.Fatalf("Unexpected count: %d", c.Count())
+	}
+
+	// Test delete.
+	c.Delete("key1")
+	v, ok = c.Get("key1")
+	if ok || v != "" {
+		t.Fatalf("Unexpected value for key1 after delete: %v, %v", v, ok)
+	}
+
+	// Test clear.
+	c.Set("key10", "key10")
+	c.Clear()
+	v, ok = c.Get("key10")
+	if ok || v != "" {
+		t.Fatalf("Unexpected value for key10 after clear: %v, %v", v, ok)
+	}
+
+	// Test contents.
+	keys := []string{"key1", "key2", "key3"}
+	for _, k := range keys {
+		c.Set(k, k)
+	}
+
+	items := c.Contents()
+	sort.Strings(keys)
+	res := strings.Split(items, ",")
+	sort.Strings(res)
+
+	if len(keys) != len(res) {
+		t.Fatalf("Unexpected length of items: %d", len(res))
+	}
+	if !cmp.Equal(keys, res) {
+		t.Fatalf("Unexpected items: %v", res)
+	}
+}
+
+func TestCache_NewWithData(t *testing.T) {
+	c := NewWithData([]string{"key1", "key2", "key3"})
+
+	// Test the count.
+	if c.Count() != 3 {
+		t.Fatalf("Unexpected count: %d", c.Count())
+	}
+
+	// Test contents.
+	keys := []string{"key1", "key2", "key3"}
+	items := c.Contents()
+	sort.Strings(keys)
+	res := strings.Split(items, ",")
+	sort.Strings(res)
+
+	if len(keys) != len(res) {
+		t.Fatalf("Unexpected length of items: %d", len(res))
+	}
+	if !cmp.Equal(keys, res) {
+		t.Fatalf("Unexpected items: %v", res)
+	}
+}
+
+func setupBenchmarks(b *testing.B) *Cache {
+	b.Helper()
+
+	c := New()
+
+	for i := 0; i < 500_000; i++ {
+		key := fmt.Sprintf("key%d", i)
+		c.Set(key, key)
+	}
+
+	return c
+}
+
+func BenchmarkSet(b *testing.B) {
+	c := New()
+
+	for i := 0; i < b.N; i++ {
+		key := fmt.Sprintf("key%d", i)
+		c.Set(key, key)
+	}
+}
+
+func BenchmarkGet(b *testing.B) {
+	c := setupBenchmarks(b)
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		key := fmt.Sprintf("key%d", i)
+		c.Get(key)
+	}
+}
+
+func BenchmarkDelete(b *testing.B) {
+	c := setupBenchmarks(b)
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		key := fmt.Sprintf("key%d", i)
+		c.Delete(key)
+	}
+}
+
+func BenchmarkCount(b *testing.B) {
+	c := setupBenchmarks(b)
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		c.Count()
+	}
+}
+
+func BenchmarkContents(b *testing.B) {
+	c := setupBenchmarks(b)
+	b.ResetTimer()
+
+	var s string
+
+	for i := 0; i < b.N; i++ {
+		s = c.Contents()
+	}
+
+	_ = s
+}

pkg/sources/gcs/gcs.go

@@ -1,12 +1,12 @@
 package gcs
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"os"
-	"sort"
+	"strings"
 	"sync"
+	"sync/atomic"
 
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
 	"github.com/go-errors/errors"
@@ -14,6 +14,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -21,23 +22,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 )
 
-// Source represents a GCS source.
-type Source struct {
-	name        string
-	jobId       int64
-	sourceId    int64
-	concurrency int
-	verify      bool
-
-	gcsManager objectManager
-	stats      *attributes
-	log        logr.Logger
-	chunksCh   chan *sources.Chunk
-
-	mu               sync.Mutex
-	sources.Progress // not safe for concurrent access.
-}
-
 // Ensure the Source satisfies the interface at compile time.
 var _ sources.Source = (*Source)(nil)
 
@@ -57,6 +41,25 @@ func (s *Source) JobID() int64 {
 	return s.jobId
 }
 
+// Source represents a GCS source.
+type Source struct {
+	name        string
+	jobId       int64
+	sourceId    int64
+	concurrency int
+	verify      bool
+
+	gcsManager objectManager
+	stats      *attributes
+	log        logr.Logger
+	chunksCh   chan *sources.Chunk
+
+	processedObjects int32
+	cache            cache.Cache
+
+	sources.Progress
+}
+
 // Init returns an initialized GCS source.
 func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int64, verify bool, connection *anypb.Any, concurrency int) error {
 	s.log = aCtx.Logger()
@@ -73,6 +76,30 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
 
+	gcsManager, err := configureGCSManager(aCtx, &conn, concurrency)
+	if err != nil {
+		return err
+	}
+	s.gcsManager = gcsManager
+
+	s.log.V(2).Info("enumerating buckets and objects")
+	if err := s.enumerate(aCtx); err != nil {
+		return fmt.Errorf("error enumerating buckets and objects: %w", err)
+	}
+
+	// For runs that aren't associated with a job, we don't need progress or a cache.
+	if id != 0 {
+		s.cache = configureCache(aCtx, &s.Progress)
+	}
+
+	return nil
+}
+
+func configureGCSManager(aCtx context.Context, conn *sourcespb.GCS, concurrency int) (*gcsManager, error) {
+	if conn == nil {
+		return nil, fmt.Errorf("GCS connection is nil, cannot configure GCS manager")
+	}
+
 	var gcsManagerAuthOption gcsManagerOption
 
 	switch conn.Credential.(type) {
@@ -81,7 +108,7 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
 	case *sourcespb.GCS_ServiceAccountFile:
 		b, err := os.ReadFile(conn.GetServiceAccountFile())
 		if err != nil {
-			return fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
+			return nil, fmt.Errorf("error reading GCS JSON Service Account file: %w", err)
 		}
 		gcsManagerAuthOption = withJSONServiceAccount(aCtx, b)
 	case *sourcespb.GCS_JsonServiceAccount:
@@ -91,39 +118,28 @@ func (s *Source) Init(aCtx context.Context, name string, id int64, sourceID int6
 	case *sourcespb.GCS_Unauthenticated:
 		gcsManagerAuthOption = withoutAuthentication()
 	default:
-		return fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)
+		return nil, fmt.Errorf("unknown GCS authentication type: %T", conn.Credential)
 
 	}
 
-	resume, err := setResumeBucketOffset(s.Progress.EncodedResumeInfo)
-	if err != nil {
-		return fmt.Errorf("error setting resume info: %w", err)
-	}
-
 	gcsManagerOpts := []gcsManagerOption{
 		withConcurrency(concurrency),
-		withBucketOffsets(resume),
 		withMaxObjectSize(conn.MaxObjectSize),
 		gcsManagerAuthOption,
 	}
-	if setGCSManagerBucketOptions(&conn) != nil {
-		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(&conn))
+	if setGCSManagerBucketOptions(conn) != nil {
+		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerBucketOptions(conn))
 	}
-	if setGCSManagerObjectOptions(&conn) != nil {
-		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(&conn))
+	if setGCSManagerObjectOptions(conn) != nil {
+		gcsManagerOpts = append(gcsManagerOpts, setGCSManagerObjectOptions(conn))
 	}
 
-	if s.gcsManager, err = newGCSManager(conn.ProjectId, gcsManagerOpts...); err != nil {
-		return fmt.Errorf("error creating GCS manager: %w", err)
+	gcsManager, err := newGCSManager(conn.ProjectId, gcsManagerOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("error creating GCS manager: %w", err)
 	}
 
-	if len(resume) > 0 {
-		s.log.V(2).Info("resuming from previous run")
-		return nil
-	}
-
-	s.log.V(2).Info("new run, enumerating buckets and objects")
-	return s.enumerate(aCtx)
+	return gcsManager, nil
 }
 
 func setGCSManagerBucketOptions(conn *sourcespb.GCS) gcsManagerOption {
@@ -147,140 +163,13 @@ func setGCSManagerOptions(include, exclude []string, includeFn, excludeFn func([
 	return nil
 }
 
-type progressInfo struct {
-	processedBucketCnt,
-	totalBucketsCnt,
-	processedObjectsCnt,
-	totalObjectsCnt int
-
-	// bucketObjects tracks processing and processed objects for each bucket.
-	bucketObjects map[string]*objectsProgress
-}
-
-// objectsProgress keeps track of the progress of processing objects in a bucket.
-// It is marshalled/unmarshalled to/from a string as part of the Source's Progress.
-type objectsProgress struct {
-	// IsBuckedProcessed is true if all objects in the bucket have been processed.
-	IsBucketProcessed bool
-	// ProcessedCnt is the number of objects that have been processed.
-	ProcessedCnt,
-	// TotalCnt is the total number of objects in the bucket.
-	TotalCnt int
-	// Processed is the last object that was fully processed.
-	Processed string
-	// Processing are the objects that were not fully processed.
-	Processing map[string]struct{}
-}
-
-func newObjectsProgress(cnt int) *objectsProgress {
-	return &objectsProgress{TotalCnt: cnt, Processing: map[string]struct{}{}}
-}
-
-// newProgressInfo constructs a progressInfo from the Source's Progress information.
-func newProgressInfo(ctx context.Context, s *sources.Progress) (*progressInfo, error) {
-	totalObjCnt := s.SectionsRemaining
-	processedObjCnt := s.SectionsCompleted
-	info := &progressInfo{
-		totalObjectsCnt:     int(totalObjCnt),
-		processedObjectsCnt: int(processedObjCnt),
+func configureCache(ctx context.Context, p *sources.Progress) cache.Cache {
+	if p.EncodedResumeInfo != "" {
+		ctx.Logger().V(3).Info("creating storage service with existing progress")
+		return newPersistableCache(p, withMemoryLoadedPersistableCache(strings.Split(p.EncodedResumeInfo, ",")))
 	}
 
-	encodeResume := s.EncodedResumeInfo
-	if encodeResume == "" {
-		info.bucketObjects = make(map[string]*objectsProgress)
-		return info, nil
-	}
-
-	var m map[string]*objectsProgress
-	if err := json.Unmarshal([]byte(encodeResume), &m); err != nil {
-		ctx.Logger().V(1).Info("error unmarshalling progress info", "error", err)
-		return info, nil
-	}
-	info.bucketObjects = m
-
-	return info, nil
-}
-
-type progressStateFn func(string, string, *progressInfo)
-
-func (p *progressInfo) setProcessStatus(obj object, fn progressStateFn) {
-	fn(obj.bucket, obj.name, p)
-}
-
-func setProcessingBucketObject(bucket, obj string, progress *progressInfo) {
-	progress.processing(bucket, obj)
-}
-
-func (p *progressInfo) processing(bkt, obj string) {
-	p.bucketObjects[bkt].Processing[obj] = struct{}{}
-}
-
-func setProcessedBucketObject(bucket, obj string, progress *progressInfo) {
-	progress.processed(bucket, obj)
-}
-
-func (p *progressInfo) processed(bkt, obj string) {
-	// Remove the object from the processing list and increment the processed count.
-	delete(p.bucketObjects[bkt].Processing, obj)
-
-	p.bucketObjects[bkt].ProcessedCnt++
-	// If all objects in the bucket have been processed, mark the bucket as processed.
-	if p.bucketObjects[bkt].ProcessedCnt == p.bucketObjects[bkt].TotalCnt {
-		p.bucketObjects[bkt].IsBucketProcessed = true
-		p.processedBucketCnt++
-	}
-	p.processedObjectsCnt++
-
-	// If the object is not greater (lexicographically) than the last processed object, we can skip it.
-	// This ensures we keep the greatest object name as the last processed object.
-	if obj < p.bucketObjects[bkt].Processed {
-		return
-	}
-	p.bucketObjects[bkt].Processed = obj
-}
-
-func setResumeBucketOffset(s string) (map[string]offsetInfo, error) {
-	resumeInfo := progressInfo{}
-	if s != "" {
-		if err := json.Unmarshal([]byte(s), &resumeInfo.bucketObjects); err != nil {
-			return nil, fmt.Errorf("error unmarshalling resume info: %w", err)
-		}
-	}
-
-	return calcBktOffset(resumeInfo.bucketObjects)
-}
-
-// In order to calculate the bucket offset, we need to know the last object
-// that was processed for each bucket, as well as any objects that were processing.
-// If the bucket was fully processed, we don't need to set an offset.
-// If there were no objects processing, we can just use the last object that was processed.
-// If there were objects processing, we need to find the first object (lexicographically)
-// that was processing, as that is the next object that needs to be processed.
-func calcBktOffset(resumeInfo map[string]*objectsProgress) (map[string]offsetInfo, error) {
-	bucketOffset := make(map[string]offsetInfo, len(resumeInfo))
-	for objName, progress := range resumeInfo {
-		info := offsetInfo{}
-		if progress.IsBucketProcessed {
-			info.isBucketProcessed = true
-			bucketOffset[objName] = info
-			continue
-		}
-		if len(progress.Processing) == 0 {
-			info.lastProcessedObject = progress.Processed
-			bucketOffset[objName] = info
-			continue
-		}
-
-		processing := make([]string, 0, len(progress.Processing))
-		for k := range progress.Processing {
-			processing = append(processing, k)
-		}
-		sort.Strings(processing)
-		info.lastProcessedObject = processing[0]
-		bucketOffset[objName] = info
-	}
-
-	return bucketOffset, nil
+	return newPersistableCache(p, withMemoryPersistableCache())
 }
 
 // enumerate all the objects and buckets in the source and use the results to
@@ -293,18 +182,7 @@ func (s *Source) enumerate(ctx context.Context) error {
 	}
 	s.stats = stats
 
-	info := progressInfo{
-		totalBucketsCnt: int(s.stats.numBuckets),
-		totalObjectsCnt: int(s.stats.numObjects),
-	}
-
-	progress := make(map[string]*objectsProgress, len(s.stats.bucketObjects))
-	for k, v := range s.stats.bucketObjects {
-		progress[k] = newObjectsProgress(int(v))
-	}
-	info.bucketObjects = progress
-
-	return s.setProgress(ctx, &info, fmt.Sprintf("enumerated %d buckets, %d objects", s.stats.numBuckets, s.stats.numObjects))
+	return nil
 }
 
 // Chunks emits chunks of bytes over a channel.
@@ -314,11 +192,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		return fmt.Errorf("error listing objects: %w", err)
 	}
 	s.chunksCh = chunksChan
-
-	progress, err := newProgressInfo(ctx, &s.Progress)
-	if err != nil {
-		return fmt.Errorf("error constructing progress info: %w", err)
-	}
+	s.Progress.Message = "starting to process objects..."
 
 	var wg sync.WaitGroup
 	for obj := range objectCh {
@@ -329,55 +203,46 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			continue
 		}
 
-		if err := s.startProcessing(ctx, progress, o); err != nil {
-			ctx.Logger().V(1).Info("GCS source error starting to process objects", "error", err)
+		if s.cache != nil && s.cache.Exists(o.md5) {
+			ctx.Logger().V(5).Info("skipping object, object already processed", "name", o.name)
 			continue
 		}
+
 		wg.Add(1)
-		go func(obj object, resume *progressInfo) {
+		go func(obj object) {
 			defer wg.Done()
 
 			if err := s.processObject(ctx, o); err != nil {
 				ctx.Logger().V(1).Info("error setting start progress progress", "name", o.name, "error", err)
 				return
 			}
-			if err := s.endProcessing(ctx, resume, o); err != nil {
-				ctx.Logger().V(1).Info("error setting end progress progress", "name", o.name, "error", err)
+			if s.cache != nil {
+				s.setProgress(ctx, o.name, o.md5)
 			}
-		}(o, progress)
+		}(o)
 	}
 	wg.Wait()
 
-	ctx.Logger().Info("GCS source finished processing", "name", s.name)
+	s.completeProgress(ctx)
 	return nil
 }
 
-func (s *Source) startProcessing(ctx context.Context, progress *progressInfo, o object) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
+func (s *Source) setProgress(ctx context.Context, objName, md5 string) {
+	atomic.AddInt32(&s.processedObjects, 1)
+	ctx.Logger().V(5).Info("setting progress for object", "object-name", objName, "md5", md5)
 
-	progress.setProcessStatus(o, setProcessingBucketObject)
-	return s.setProgress(ctx, progress, fmt.Sprintf("GCS source beginning to process object %s in bucket %s", o.name, o.bucket))
+	s.cache.Set(md5, md5)
+	processed := atomic.LoadInt32(&s.processedObjects)
+	s.Progress.SectionsCompleted = processed
+	s.Progress.SectionsRemaining = int32(s.stats.numObjects)
+	s.Progress.PercentComplete = int64(float64(processed) / float64(s.stats.numObjects) * 100)
 }
 
-func (s *Source) endProcessing(ctx context.Context, progress *progressInfo, o object) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	progress.setProcessStatus(o, setProcessedBucketObject)
-	return s.setProgress(ctx, progress, fmt.Sprintf("GCS source finished processing object %s in bucket %s", o.name, o.bucket))
-}
-
-func (s *Source) setProgress(ctx context.Context, progress *progressInfo, msg string) error {
-	ctx.Logger().V(5).Info(msg)
-
-	b, err := json.Marshal(progress.bucketObjects)
-	if err != nil {
-		return fmt.Errorf("error marshalling resume info: %w", err)
-	}
-	s.SetProgressComplete(progress.processedObjectsCnt, progress.totalObjectsCnt, msg, string(b))
-
-	return nil
+func (s *Source) completeProgress(ctx context.Context) {
+	msg := fmt.Sprintf("GCS source finished processing %d objects", s.stats.numObjects)
+	ctx.Logger().Info(msg)
+	s.Progress.Message = msg
+	s.cache.Clear()
 }
 
 func (s *Source) processObject(ctx context.Context, o object) error {

pkg/sources/gcs/gcs_manager.go

@@ -2,6 +2,7 @@ package gcs
 
 import (
 	aCtx "context"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"runtime"
@@ -357,6 +358,7 @@ type object struct {
 	contentType string
 	owner       string
 	link        string
+	md5         string
 	// acl represents an ACLEntities.
 	// https://pkg.go.dev/cloud.google.com/go/storage#ACLEntity
 	acl       []string
@@ -383,6 +385,10 @@ func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes,
 	logger.V(5).Info("enumerating buckets", "numBuckets", len(bkts))
 	stats := newStats(len(bkts))
 
+	defer func(start time.Time) {
+		logger.V(5).Info("finished enumerating buckets", "duration-seconds", fmt.Sprintf("%.1f", time.Since(start).Seconds()), "num-buckets", len(bkts), "num-objects", stats.numObjects)
+	}(time.Now())
+
 	for _, bkt := range bkts {
 		bkt := bkt
 		g.workerPool.Go(func() error {
@@ -426,7 +432,6 @@ func (g *gcsManager) enumerate(ctx context.Context, bkts []bucket) (*attributes,
 
 	_ = g.workerPool.Wait()
 	g.attr = stats
-	logger.V(5).Info("finished enumerating buckets", "num-objects", stats.numObjects, "num-buckets", stats.numBuckets)
 
 	return stats, nil
 }
@@ -631,6 +636,7 @@ func (g *gcsManager) constructObject(ctx context.Context, obj *storage.ObjectHan
 	o.contentType = attrs.ContentType
 	o.owner = attrs.Owner
 	o.link = attrs.MediaLink
+	o.md5 = base64.StdEncoding.EncodeToString(attrs.MD5[:])
 	o.createdAt = attrs.Created
 	o.updatedAt = attrs.Updated
 	o.acl = objectACLs(attrs.ACL)

pkg/sources/gcs/gcs_manager_test.go

@@ -473,6 +473,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
 					acl:         []string{},
+					md5:         "ek30X4Oc1ZSsI2sZUkzskg==",
 				},
 				{
 					name:        "moar2.txt",
@@ -481,6 +482,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        12,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
 					acl:         []string{},
+					md5:         "5Z/5eUEET4XfUpfhwwLSYA==",
 				},
 				{
 					name:        "aws3.txt",
@@ -489,6 +491,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
 					acl:         []string{},
+					md5:         "JusDN4oMjpnuDOz5FVrOgQ==",
 				},
 				{
 					name:        "moar.txt",
@@ -497,6 +500,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        6,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
 					acl:         []string{},
+					md5:         "CffgLxKQviEdpweiZvFTsw==",
 				},
 				{
 					name:        "AMAZON_FASHION_5.json",
@@ -505,6 +509,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        1413469,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
 					acl:         []string{},
+					md5:         "Wh2PSDqjRpUlY3kFDUKitw==",
 				},
 			},
 			wantNumBkt: 4,
@@ -522,6 +527,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
 					acl:         []string{},
+					md5:         "ek30X4Oc1ZSsI2sZUkzskg==",
 				},
 				{
 					name:        "moar2.txt",
@@ -530,6 +536,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        12,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
 					acl:         []string{},
+					md5:         "5Z/5eUEET4XfUpfhwwLSYA==",
 				},
 			},
 			wantNumBkt: 1,
@@ -547,6 +554,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        6,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
 					acl:         []string{},
+					md5:         "CffgLxKQviEdpweiZvFTsw==",
 				},
 				{
 					name:        "AMAZON_FASHION_5.json",
@@ -555,6 +563,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        1413469,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
 					acl:         []string{},
+					md5:         "Wh2PSDqjRpUlY3kFDUKitw==",
 				},
 			},
 			wantNumBkt: 2,
@@ -576,6 +585,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th2/o/aws3.txt?generation=1677871022489611&alt=media",
 					acl:         []string{},
+					md5:         "JusDN4oMjpnuDOz5FVrOgQ==",
 				},
 				{
 					name:        "moar.txt",
@@ -584,6 +594,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        6,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th3/o/moar.txt?generation=1677871042896804&alt=media",
 					acl:         []string{},
+					md5:         "CffgLxKQviEdpweiZvFTsw==",
 				},
 				{
 					name:        "AMAZON_FASHION_5.json",
@@ -592,6 +603,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        1413469,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th4/o/AMAZON_FASHION_5.json?generation=1677871063457469&alt=media",
 					acl:         []string{},
+					md5:         "Wh2PSDqjRpUlY3kFDUKitw==",
 				},
 			},
 			wantNumBkt: 4, // We still list objects in all buckets.
@@ -613,6 +625,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
 					acl:         []string{},
+					md5:         "ek30X4Oc1ZSsI2sZUkzskg==",
 				},
 			},
 			wantNumBkt: 1,
@@ -634,6 +647,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        150,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/aws1.txt?generation=1677870994890594&alt=media",
 					acl:         []string{},
+					md5:         "ek30X4Oc1ZSsI2sZUkzskg==",
 				},
 			},
 			wantNumBkt: 4,
@@ -656,6 +670,7 @@ func TestGCSManagerListObjects(t *testing.T) {
 					size:        12,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
 					acl:         []string{},
+					md5:         "5Z/5eUEET4XfUpfhwwLSYA==",
 				},
 			},
 			wantNumBkt: 1,
@@ -743,6 +758,7 @@ func TestGCSManagerListObjects_Resuming(t *testing.T) {
 					size:        12,
 					link:        "https://storage.googleapis.com/download/storage/v1/b/test-bkt-th/o/moar2.txt?generation=1677871000378542&alt=media",
 					acl:         []string{},
+					md5:         "5Z/5eUEET4XfUpfhwwLSYA==",
 				},
 			},
 			wantNumBkt: 1,

pkg/sources/gcs/gcs_test.go

@@ -1,10 +1,10 @@
 package gcs
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"sort"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -31,7 +31,7 @@ func TestSourceInit(t *testing.T) {
 	source, conn := createTestSource(&sourcespb.GCS{
 		ProjectId: testProjectID,
 		IncludeBuckets: []string{
-			"bucket1",
+			publicBucket,
 		},
 		ExcludeBuckets: []string{
 			perfTestBucketGlob,
@@ -39,22 +39,46 @@ func TestSourceInit(t *testing.T) {
 		ExcludeObjects: []string{
 			"object1",
 		},
-		Credential: &sourcespb.GCS_ApiKey{
-			ApiKey: testAPIKey,
-		},
+		Credential: &sourcespb.GCS_Unauthenticated{},
 	})
 
-	// Set the EncodeResumeInfo to prevent enumeration.
-	b, err := json.Marshal(map[string]objectsProgress{"other": {}})
-	assert.Nil(t, err)
-	source.Progress.EncodedResumeInfo = string(b)
+	testCases := []struct {
+		name      string
+		jobID     int64
+		conn      *anypb.Any
+		source    *Source
+		wantGCS   bool
+		wantCache bool
+	}{
+		{
+			name:      "Non-cacheable GCS source init",
+			jobID:     0,
+			conn:      conn,
+			source:    source,
+			wantGCS:   true,
+			wantCache: false,
+		},
+		{
+			name:      "Cacheable GCS source init",
+			jobID:     1,
+			conn:      conn,
+			source:    source,
+			wantGCS:   true,
+			wantCache: true,
+		},
+	}
 
-	err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
-	assert.Nil(t, err)
-	assert.NotNil(t, source.gcsManager)
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.source.Init(context.Background(), "test", tc.jobID, 1, true, tc.conn, 8)
+			assert.Nil(t, err)
+			assert.Equal(t, tc.wantGCS, tc.source.gcsManager != nil)
+			assert.Equal(t, tc.wantCache, tc.source.cache != nil)
+		})
+	}
 }
 
-func TestSourceInit_Conn(t *testing.T) {
+func TestConfigureGCSManager(t *testing.T) {
 	testCases := []struct {
 		name    string
 		conn    *sourcespb.GCS
@@ -77,7 +101,6 @@ func TestSourceInit_Conn(t *testing.T) {
 			want: &gcsManager{
 				projectID:      testProjectID,
 				excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
-				hasEnumerated:  true,
 			},
 		},
 		{
@@ -95,7 +118,6 @@ func TestSourceInit_Conn(t *testing.T) {
 			want: &gcsManager{
 				projectID:      testProjectID,
 				includeBuckets: map[string]struct{}{"bucket1": {}},
-				hasEnumerated:  true,
 			},
 		},
 		{
@@ -117,27 +139,20 @@ func TestSourceInit_Conn(t *testing.T) {
 				projectID:      testProjectID,
 				includeObjects: map[string]struct{}{"object1": {}},
 				excludeBuckets: map[string]struct{}{perfTestBucketGlob: {}},
-				hasEnumerated:  true,
 			},
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			source, conn := createTestSource(tc.conn)
-			// Set the EncodeResumeInfo to prevent enumeration.
-			b, err := json.Marshal(map[string]objectsProgress{"other": {}})
-			assert.Nil(t, err)
-			source.Progress.EncodedResumeInfo = string(b)
-
-			err = source.Init(context.Background(), "test", 1, 1, true, conn, 8)
+			ctx := context.Background()
+			got, err := configureGCSManager(ctx, tc.conn, 8)
 			if err != nil && !tc.wantErr {
-				t.Errorf("source.Init() got: %v, want: %v", err, nil)
-				return
+				t.Errorf("source.configureGCSManager() error = %v", err)
 			}
 
 			if !tc.wantErr {
-				if diff := cmp.Diff(tc.want, source.gcsManager,
+				if diff := cmp.Diff(tc.want, got,
 					cmp.AllowUnexported(gcsManager{}),
 					cmpopts.IgnoreFields(gcsManager{}, "client", "workerPool", "concurrency", "buckets", "maxObjectSize", "attr"),
 				); diff != "" {
@@ -148,65 +163,6 @@ func TestSourceInit_Conn(t *testing.T) {
 	}
 }
 
-func TestSourceInit_Progress(t *testing.T) {
-	complete := map[string]objectsProgress{testBucket: {
-		Processed:         object2,
-		IsBucketProcessed: true,
-		ProcessedCnt:      1,
-		TotalCnt:          1,
-	}}
-	completeEncodeResumeInfo, err := json.Marshal(complete)
-	assert.Nil(t, err)
-
-	processing := map[string]objectsProgress{
-		testBucket: {
-			IsBucketProcessed: false,
-			ProcessedCnt:      1,
-			Processed:         object2,
-			Processing:        map[string]struct{}{object1: {}},
-			TotalCnt:          2,
-		},
-	}
-	processingEncodeResumeInfo, err := json.Marshal(processing)
-	assert.Nil(t, err)
-
-	testCases := []struct {
-		name              string
-		conn              *sourcespb.GCS
-		encodeResumeInfo  string
-		gcsManagerBuckets map[string]bucket
-	}{
-		{
-			name:              "no progress",
-			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}, ExcludeBuckets: []string{perfTestBucketGlob}},
-			gcsManagerBuckets: map[string]bucket{},
-		},
-		{
-			name:              "progress set, no processing objects in any buckets",
-			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
-			encodeResumeInfo:  string(completeEncodeResumeInfo),
-			gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: "", shouldInclude: false}},
-		},
-		{
-			name:              "progress set, processing objects in some buckets",
-			conn:              &sourcespb.GCS{ProjectId: testProjectID, Credential: &sourcespb.GCS_Adc{}},
-			encodeResumeInfo:  string(processingEncodeResumeInfo),
-			gcsManagerBuckets: map[string]bucket{testBucket: {name: testBucket, startOffset: object1, shouldInclude: true}},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			source, conn := createTestSource(tc.conn)
-			source.Progress.EncodedResumeInfo = tc.encodeResumeInfo
-
-			err := source.Init(context.Background(), "test", 1, 1, true, conn, 8)
-			assert.Nil(t, err)
-			assert.Equal(t, tc.gcsManagerBuckets, source.gcsManager.(*gcsManager).buckets)
-		})
-	}
-}
-
 type mockObjectManager struct {
 	wantErr bool
 }
@@ -264,6 +220,7 @@ func createTestObject(id int) object {
 		link:        fmt.Sprintf("https://storage.googleapis.com/%s/%s", testBucket, fmt.Sprintf("object%d", id)),
 		acl:         []string{"authenticatedUsers"},
 		size:        42,
+		md5:         fmt.Sprintf("md5hash%d", id),
 		Reader:      &mockReader{data: []byte(fmt.Sprintf("hello world %d", id))},
 	}
 }
@@ -297,7 +254,9 @@ func TestSourceChunks_ListObjects(t *testing.T) {
 	source := &Source{
 		gcsManager: &mockObjectManager{},
 		chunksCh:   chunksCh,
+		Progress:   sources.Progress{},
 	}
+	source.cache = newPersistableCache(&source.Progress, withMemoryPersistableCache())
 
 	err := source.enumerate(ctx)
 	assert.Nil(t, err)
@@ -349,11 +308,6 @@ func TestSourceInit_Enumerate(t *testing.T) {
 	assert.Equal(t, uint64(5), source.stats.numObjects)
 	assert.Equal(t, uint32(1), source.stats.numBuckets)
 	assert.Equal(t, uint64(5), source.stats.bucketObjects[testBucket])
-	// Ensure progress is initialized correctly.
-	p := map[string]*objectsProgress{testBucket: {TotalCnt: 5, Processing: map[string]struct{}{}}}
-	b, err := json.Marshal(p)
-	assert.Nil(t, err)
-	assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
 }
 
 func TestSourceChunks_ListObjects_Error(t *testing.T) {
@@ -370,11 +324,14 @@ func TestSourceChunks_ListObjects_Error(t *testing.T) {
 func TestSourceChunks_ProgressSet(t *testing.T) {
 	ctx := context.Background()
 	chunksCh := make(chan *sources.Chunk, 1)
-
 	source := &Source{
 		gcsManager: &mockObjectManager{},
 		chunksCh:   chunksCh,
+		Progress:   sources.Progress{},
 	}
+
+	source.cache = newPersistableCache(&source.Progress, withMemoryPersistableCache(), withCustomIncrement(5))
+
 	err := source.enumerate(ctx)
 	assert.Nil(t, err)
 
@@ -397,26 +354,22 @@ func TestSourceChunks_ProgressSet(t *testing.T) {
 	// Ensure we get 5 objects back.
 	assert.Equal(t, len(want), len(got))
 
-	processing := map[string]struct{}{"object0": {}, "object1": {}, "object2": {}, "object3": {}, "object4": {}}
+	processing := map[string]struct{}{"md5hash0": {}, "md5hash1": {}, "md5hash2": {}, "md5hash3": {}, "md5hash4": {}}
 	for i := 0; i < 5; i++ {
 		delete(processing, fmt.Sprintf("object%d", i))
 	}
 
 	// Test that the resume progress is set.
 	// The processed values should be the greatest object name lexicographically.
-	resume := map[string]objectsProgress{
-		testBucket: {
-			Processed:         "object4",
-			Processing:        processing,
-			IsBucketProcessed: true,
-			ProcessedCnt:      5,
-			TotalCnt:          5,
-		},
-	}
-	b, err := json.Marshal(resume)
-	assert.Nil(t, err)
+	progress := "md5hash0,md5hash1,md5hash2,md5hash3,md5hash4"
+	objs := strings.Split(progress, ",")
 
-	assert.Equal(t, string(b), source.Progress.EncodedResumeInfo)
+	encodeResume := strings.Split(source.Progress.EncodedResumeInfo, ",")
+	sort.Strings(encodeResume)
+
+	assert.Equal(t, objs, encodeResume)
 	assert.Equal(t, int32(5), source.Progress.SectionsCompleted)
 	assert.Equal(t, int64(100), source.Progress.PercentComplete)
+	assert.Equal(t, 0, source.cache.Count())
+	assert.Equal(t, fmt.Sprintf("GCS source finished processing %d objects", 5), source.Progress.Message)
 }

pkg/sources/gcs/persistable_cache.go

@@ -0,0 +1,70 @@
+package gcs
+
+import (
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+const defaultCachePersistIncrement = 2500
+
+type persistibleCacheConfiguration func(c *persistableCache)
+
+// persistableCache handles all cache operations with some additional bookkeeping.
+// The threshold value is the percentage of objects that must be processed
+// before the cache is persisted.
+type persistableCache struct {
+	persistIncrement int
+	cache.Cache
+	*sources.Progress
+}
+
+func newPersistableCache(p *sources.Progress, cfgs ...persistibleCacheConfiguration) *persistableCache {
+	pc := &persistableCache{
+		persistIncrement: defaultCachePersistIncrement,
+		Progress:         p,
+	}
+
+	for _, cfg := range cfgs {
+		cfg(pc)
+	}
+	return pc
+}
+
+// withMemoryPersistableCache configures the persistableCache to use an in-memory cache.
+func withMemoryPersistableCache() persistibleCacheConfiguration {
+	return func(pc *persistableCache) {
+		pc.Cache = memory.New()
+	}
+}
+
+// withMemoryLoadedPersistableCache configures the persistableCache to use an in-memory cache with the given contents.
+func withMemoryLoadedPersistableCache(contents []string) persistibleCacheConfiguration {
+	return func(pc *persistableCache) {
+		pc.Cache = memory.NewWithData(contents)
+	}
+}
+
+// withCustomIncrement configures the persistableCache to use the given increment.
+func withCustomIncrement(increment int) persistibleCacheConfiguration {
+	return func(pc *persistableCache) {
+		pc.persistIncrement = increment
+	}
+}
+
+// Set overrides the default Set method in order to persist the cache contents
+// on each persistIncrement.
+func (c *persistableCache) Set(key, value string) {
+	c.Cache.Set(key, value)
+
+	if ok, val := c.shouldPersist(); ok {
+		c.EncodedResumeInfo = val
+	}
+}
+
+func (c *persistableCache) shouldPersist() (bool, string) {
+	if c.Count()%c.persistIncrement != 0 {
+		return false, ""
+	}
+	return true, c.Contents()
+}

pkg/sources/gcs/persistable_cache_test.go

@@ -0,0 +1,74 @@
+package gcs
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/cache/memory"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestPersistableCache(t *testing.T) {
+	// Create a new Progress object for testing.
+	progress := sources.Progress{}
+
+	tests := []struct {
+		name string
+		cfgs []persistibleCacheConfiguration
+		fn   func(cache *persistableCache)
+	}{
+		{
+			name: "withMemoryPersistableCache",
+			cfgs: []persistibleCacheConfiguration{withMemoryPersistableCache()},
+			fn: func(cache *persistableCache) {
+				assert.NotNil(t, cache.Cache)
+				assert.IsType(t, &memory.Cache{}, cache.Cache)
+			},
+		},
+		{
+			name: "withMemoryLoadedPersistableCache",
+			cfgs: []persistibleCacheConfiguration{withMemoryLoadedPersistableCache([]string{"foo", "bar"})},
+			fn: func(cache *persistableCache) {
+				assert.NotNil(t, cache.Cache)
+				assert.IsType(t, &memory.Cache{}, cache.Cache)
+				assert.Contains(t, cache.Cache.Contents(), "foo,bar")
+			},
+		},
+		{
+			name: "withCustomIncrement",
+			cfgs: []persistibleCacheConfiguration{withCustomIncrement(100)},
+			fn: func(cache *persistableCache) {
+				assert.Equal(t, 100, cache.persistIncrement)
+			},
+		},
+		{
+			name: "Set",
+			cfgs: []persistibleCacheConfiguration{withMemoryPersistableCache(), withCustomIncrement(3)},
+			fn: func(cache *persistableCache) {
+				cache.Set("foo", "bar")
+				assert.Equal(t, 1, cache.Count())
+
+				cache.Set("baz", "qux")
+				assert.Equal(t, 2, cache.Count())
+
+				// The cache should not persist until the increment (3) is reached.
+				assert.False(t, progress.EncodedResumeInfo != "")
+
+				cache.Set("quux", "corge")
+				assert.Equal(t, 3, cache.Count())
+
+				// The cache should now persist.
+				assert.True(t, progress.EncodedResumeInfo != "")
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			cache := newPersistableCache(&progress, test.cfgs...)
+			assert.NotNil(t, cache)
+			test.fn(cache)
+		})
+	}
+}