Create devcontainer.json for codespaces

.devcontainer/devcontainer.json

@@ -0,0 +1,5 @@
+{
+  "image": "trufflesecurity/protos:1.18-0",
+  "features": {
+  }
+}

.github/ISSUE_TEMPLATE/bug_report.md

@@ -6,7 +6,11 @@ labels: bug, needs triage
 assignees: trufflesecurity/product-eng
 ---
 
-Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
+### Community Note
+
+* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
+* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
+* If you are interested in working on this issue or have submitted a pull request, please leave a comment
 
 ### TruffleHog Version
 <!--- Please run `trufflehog --version` to show the version. If you are not running the latest version, please upgrade because your issue may have already been fixed. --->

.github/ISSUE_TEMPLATE/feature_request.md

@@ -6,27 +6,33 @@ labels: enhancement, needs triage
 assignees: trufflesecurity/product-eng
 ---
 
-Please review the [Community Note](https://github.com/trufflesecurity/trufflehog/blob/main/.github/community_note.md) before submitting
+### Community Note
+
+* Please vote on this issue by adding a 👍 [reaction](https://blog.github.com/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) to the original issue to help the community and maintainers prioritize this request
+* Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request
+* If you are interested in working on this issue or have submitted a pull request, please leave a comment
+
+### Description
 
-## Description
 <!--- Please leave a helpful description of the feature request here. --->
 
-### Preferred Solution
-<!--- A clear and concise description of what you want to happen. What
-information may be required and what would be the preferred way to provide it?
+## Problem to be Addressed
+<!--- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] --->
+
+## Description of the Preferred Solution
+<!--- A clear and concise description of what you want to happen. What 
+information may be required and what would be the preferred way to provide it? 
 What should the output include? --->
 
-### Additional Context
+## Additional Context
 <!--- Add any other context or screenshots about the feature request here. --->
 
-#### References
+### References
 
 <!---
 Information about referencing Github Issues: https://help.github.com/articles/basic-writing-and-formatting-syntax/#referencing-issues-and-pull-requests
 
 Are there any other GitHub issues (open or closed) or pull requests that should be linked here? Vendor blog posts or documentation? For example:
-
-* #0000
 --->
 
-
+* #0000

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,11 +2,3 @@
 Please create an issue to collect feedback prior to feature additions. Please also reference that issue in any PRs.
 If possible try to keep PRs scoped to one feature, and add tests for new features.
 -->
-
-### Description:
-Explain the purpose of the PR.
-
-### Checklist:
-* [ ] Tests passing (`make test-community`)?
-* [ ] Lint passing (`make lint` this requires [golangci-lint](https://golangci-lint.run/usage/install/#local-installation))?
-

.github/community_note.md

@@ -1,7 +0,0 @@
-## Community Note
-
-Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request.
-
-Please do not leave "+1" or other comments that do not add relevant new information or questions, they generate extra noise for issue followers and do not help prioritize the request.
-
-If you are interested in working on this issue or have submitted a pull request, please leave a comment.

.github/workflows/test.yml

@@ -52,8 +52,8 @@ jobs:
         service_account: 'github-ci-external@trufflehog-testing.iam.gserviceaccount.com'
     - name: Test
       run: make test-detectors
-      continue-on-error: true
-  test-community:
+      continue-on-error: true  
+  test-fork:
     if: ${{ github.event.pull_request.head.repo.fork }}
     runs-on: ubuntu-latest
     permissions:
@@ -67,4 +67,4 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v3
     - name: Test
-      run: make test-community
+      run: make test-forks

Makefile

@@ -39,7 +39,7 @@ test-race:
 test-detectors:
 	CGO_ENABLED=0 go test -tags=detectors -timeout=5m $(shell go list ./... | grep pkg/detectors)
 
-test-community:
+test-forks:
 	CGO_ENABLED=0 go test -timeout=5m $(shell go list ./... | grep -v /vendor/ | grep -v pkg/detectors | grep -v pkg/sources)
 
 bench:

README.md

@@ -8,6 +8,7 @@
 
 <div align="center">
 
+[![CI Status](https://github.com/trufflesecurity/trufflehog/actions/workflows/release.yml/badge.svg)](https://github.com/trufflesecurity/trufflehog/actions)
 [![Go Report Card](https://goreportcard.com/badge/github.com/trufflesecurity/trufflehog/v3)](https://goreportcard.com/report/github.com/trufflesecurity/trufflehog/v3)
 [![License](https://img.shields.io/badge/license-AGPL--3.0-brightgreen)](/LICENSE)
 [![Total Detectors](https://img.shields.io/github/directory-file-count/trufflesecurity/truffleHog/pkg/detectors?label=Total%20Detectors&type=dir)](/pkg/detectors)

pkg/context/context.go

@@ -2,7 +2,10 @@ package context
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"runtime/debug"
+	"sync"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -34,7 +37,9 @@ type CancelFunc = context.CancelFunc
 type logCtx struct {
 	// Embed context.Context to get all methods for free.
 	context.Context
-	log logr.Logger
+	log     logr.Logger
+	err     *error
+	errLock *sync.Mutex
 }
 
 // Logger returns a structured logger.
@@ -42,6 +47,13 @@ func (l logCtx) Logger() logr.Logger {
 	return l.log
 }
 
+func (l logCtx) Err() error {
+	if l.err != nil && *l.err != nil {
+		return *l.err
+	}
+	return l.Context.Err()
+}
+
 // Background returns context.Background with a default logger.
 func Background() Context {
 	return logCtx{
@@ -65,7 +77,7 @@ func WithCancel(parent Context) (Context, context.CancelFunc) {
 		log:     parent.Logger(),
 		Context: ctx,
 	}
-	return lCtx, cancel
+	return captureCancelCallstack(lCtx, cancel)
 }
 
 // WithDeadline returns context.WithDeadline with the log object propagated and
@@ -76,7 +88,7 @@ func WithDeadline(parent Context, d time.Time) (Context, context.CancelFunc) {
 		log:     parent.Logger().WithValues("deadline", d),
 		Context: ctx,
 	}
-	return lCtx, cancel
+	return captureCancelCallstack(lCtx, cancel)
 }
 
 // WithTimeout returns context.WithTimeout with the log object propagated and
@@ -87,7 +99,7 @@ func WithTimeout(parent Context, timeout time.Duration) (Context, context.Cancel
 		log:     parent.Logger().WithValues("timeout", timeout),
 		Context: ctx,
 	}
-	return lCtx, cancel
+	return captureCancelCallstack(lCtx, cancel)
 }
 
 // WithValue returns context.WithValue with the log object propagated and
@@ -138,3 +150,29 @@ func AddLogger(parent context.Context) Context {
 func SetDefaultLogger(l logr.Logger) {
 	defaultLogger = l
 }
+
+// captureCancelCallstack is a helper function to capture the callstack where
+// the cancel function was first called.
+func captureCancelCallstack(ctx logCtx, f context.CancelFunc) (Context, context.CancelFunc) {
+	if ctx.err == nil {
+		var err error
+		ctx.err = &err
+		ctx.errLock = &sync.Mutex{}
+	}
+	return ctx, func() {
+		ctx.errLock.Lock()
+		defer ctx.errLock.Unlock()
+		// We must check Err() before calling f() since f() sets the error.
+		// If there's already an error, do nothing special.
+		if ctx.Err() != nil {
+			f()
+			return
+		}
+		f()
+		// Set the error with the stacktrace if the err pointer is non-nil.
+		*ctx.err = fmt.Errorf(
+			"%w (canceled at %v\n%s)",
+			ctx.Err(), time.Now(), string(debug.Stack()),
+		)
+	}
+}

pkg/context/context_test.go

@@ -170,10 +170,51 @@ func TestDefaultLogger(t *testing.T) {
 	ctx.Logger().Info("this shouldn't panic")
 }
 
-func TestRace(t *testing.T) {
-	ctx, cancel := WithCancel(Background())
-	go cancel()
-	go func() { _ = ctx.Err() }()
+func TestErrCallstack(t *testing.T) {
+	c, cancel := WithCancel(Background())
+	ctx := c.(logCtx)
+	cancel()
+	select {
+	case <-ctx.Done():
+		assert.Contains(t, ctx.Err().Error(), "TestErrCallstack")
+	case <-time.After(1 * time.Second):
+		assert.Fail(t, "context should be done")
+	}
+}
+
+func TestErrCallstackTimeout(t *testing.T) {
+	ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
+	defer cancel()
+
+	select {
+	case <-ctx.Done():
+		// Deadline exceeded errors will not have a callstack from the cancel
+		// function.
+		assert.NotContains(t, ctx.Err().Error(), "TestErrCallstackTimeout")
+	case <-time.After(1 * time.Second):
+		assert.Fail(t, "context should be done")
+	}
+}
+
+func TestErrCallstackTimeoutCancel(t *testing.T) {
+	ctx, cancel := WithTimeout(Background(), 10*time.Millisecond)
+
+	var err error
+	select {
+	case <-ctx.Done():
+		err = ctx.Err()
+	case <-time.After(1 * time.Second):
+		assert.Fail(t, "context should be done")
+	}
+
+	// Calling cancel after deadline exceeded should not overwrite the original
+	// error.
+	cancel()
+	assert.Equal(t, err, ctx.Err())
+}
+
+func TestRace(t *testing.T) {
+	_, cancel := WithCancel(Background())
+	go cancel()
 	cancel()
-	_ = ctx.Err()
 }

pkg/detectors/timezoneapi/timezoneapi.go

@@ -20,7 +20,7 @@ var (
 	client = common.SaneHttpClient()
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"timezoneapi"}) + `\b([a-zA-Z]{20})\b`)
+	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"timezoneapi"}) + `\b([a-zA-Z0-9]{20})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -54,12 +54,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 			res, err := client.Do(req)
 			if err == nil {
-				verifiedBodyResponse, err := common.ResponseContainsSubstring(res.Body, "date")
-				if err != nil {
-					return nil, err
-				}
 				defer res.Body.Close()
-				if res.StatusCode >= 200 && res.StatusCode < 300 && verifiedBodyResponse {
+				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
 					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.

pkg/sources/s3/s3.go

@@ -229,19 +229,19 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 		// ignore large files
 		if *obj.Size > s.maxObjectSize {
 			s.log.V(3).Info("Skipping %d byte file (over maxObjectSize limit)", "object", *obj.Key)
-			continue
+			return
 		}
 
 		// file empty file
 		if *obj.Size == 0 {
 			s.log.V(5).Info("Skipping 0 byte file", "object", *obj.Key)
-			continue
+			return
 		}
 
 		// skip incompatible extensions
 		if common.SkipFile(*obj.Key) {
 			s.log.V(5).Info("Skipping file with incompatible extension", "object", *obj.Key)
-			continue
+			return
 		}
 
 		s.jobPool.Go(func() error {