fixup - remove change in behavior, cannot yet confirm that results have unique ids

go.mod

@@ -4,9 +4,7 @@ go 1.18
 
 replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
 
-replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7
-
-replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff v0.7.6-zombies
+replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3
 
 require (
 	cloud.google.com/go/secretmanager v1.4.0
@@ -19,7 +17,7 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.6.7
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
-	github.com/gitleaks/go-gitdiff v0.7.4
+	github.com/gitleaks/go-gitdiff v0.7.6
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible

go.sum

@@ -142,6 +142,8 @@ github.com/felixge/fgprof v0.9.2 h1:tAMHtWMyl6E0BimjVbFt7fieU6FpjttsZN7j0wT5blc=
 github.com/felixge/fgprof v0.9.2/go.mod h1:+VNi+ZXtHIQ6wIw6bUT8nXQRefQflWECoFyRealT5sg=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gitleaks/go-gitdiff v0.7.6 h1:atcfoNPD9erzPs9C89a+i2Y+EUmR2QKB5QHJTfB4n60=
+github.com/gitleaks/go-gitdiff v0.7.6/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
 github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -359,10 +361,8 @@ github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7 h1:K4Ah1RMmlKjvIwLEdn9kADugbTd9AM7RYxVOsrkmcTk=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
-github.com/trufflesecurity/go-gitdiff v0.7.6-zombies h1:Htd8qFVUCRT6mqoHDMXWvPyRB+WYU+H7AiN4Y1r5D+Q=
-github.com/trufflesecurity/go-gitdiff v0.7.6-zombies/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3 h1:Xc61NkfI7aDHd8eHa0gglK0ZVF5UF54M4u4C5tuAKcw=
+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3/go.mod h1:Em2rda83ePrhmaX4ZdvNjnUADRiOJirEAqln0ZtN8og=
 github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
 github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
 github.com/xanzy/go-gitlab v0.65.0 h1:9xSA9cRVhz3Z54JacIHdvWnNmNAoSz/BDnyMGOf3yIg=

pkg/detectors/aws/aws.go

@@ -28,10 +28,6 @@ var (
 	// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
 	idPat     = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
 	secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)
-	// Hashes, like those for git, do technically match the secret pattern.
-	// But they are extremely unlikely to be generated as an actual AWS secret.
-	// So when we find them, if they're not verified, we should ignore the result.
-	falsePositiveSecretCheck = regexp.MustCompile(`[a-f0-9]{40}`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -152,44 +148,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 			}
 
-			// If the result is unverified and matches something like a git hash, don't include it in the results.
-			if !s1.Verified && falsePositiveSecretCheck.MatchString(resSecretMatch) {
-				continue
-			}
-
 			results = append(results, s1)
-			// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
-			if s1.Verified {
-				break
-			}
 		}
 	}
-	return awsCustomCleanResults(results), nil
-}
-
-func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
-	if len(results) == 0 {
-		return results
-	}
-
-	// For every ID, we want at most one result, preferrably verified.
-	idResults := map[string]detectors.Result{}
-	for _, result := range results {
-		// Always accept the verified result as the result for the given ID.
-		if result.Verified {
-			idResults[result.Redacted] = result
-			continue
-		}
-
-		// Only include an unverified result if we don't already have a result for a given ID.
-		if _, exist := idResults[result.Redacted]; !exist {
-			idResults[result.Redacted] = result
-		}
-	}
-
-	out := []detectors.Result{}
-	for _, r := range idResults {
-		out = append(out, r)
-	}
-	return out
+	return detectors.CleanResults(results), nil
 }

pkg/detectors/aws/aws_test.go

@@ -2,16 +2,15 @@ package aws
 
 import (
 	"context"
-	"crypto/sha256"
 	"fmt"
 	"testing"
 	"time"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 
 func TestAWS_FromChunk(t *testing.T) {
@@ -24,10 +23,6 @@ func TestAWS_FromChunk(t *testing.T) {
 	secret := testSecrets.MustGetField("AWS")
 	id := testSecrets.MustGetField("AWS_ID")
 	inactiveSecret := testSecrets.MustGetField("AWS_INACTIVE")
-	inactiveID := id[:len(id)-3] + "XYZ"
-	hasher := sha256.New()
-	hasher.Write([]byte(inactiveSecret))
-	hash := string(hasher.Sum(nil))
 
 	type args struct {
 		ctx    context.Context
@@ -86,61 +81,6 @@ func TestAWS_FromChunk(t *testing.T) {
 			want:    nil,
 			wantErr: false,
 		},
-		{
-			name: "found two, one included for every ID found",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     true,
-					Redacted:     id,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     false,
-					Redacted:     inactiveID,
-				},
-			},
-			wantErr: false,
-		},
-		{
-			name: "not found, because unverified secret was a hash",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", hash, id)), // The secret would satisfy the regex but be filtered out after not passing validation.
-				verify: true,
-			},
-			want:    nil,
-			wantErr: false,
-		},
-		{
-			name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",
-			s:    Scanner{},
-			args: args{
-				ctx:    context.Background(),
-				data:   []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),
-				verify: true,
-			},
-			want: []detectors.Result{
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     true,
-					Redacted:     id,
-				},
-				{
-					DetectorType: detectorspb.DetectorType_AWS,
-					Verified:     false,
-					Redacted:     inactiveID,
-				},
-			},
-			wantErr: false,
-		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

pkg/output/legacy_json.go

@@ -19,27 +19,13 @@ import (
 )
 
 func PrintLegacyJSON(r *detectors.ResultWithMetadata) {
-	var repo string
-	switch r.SourceType {
-	case sourcespb.SourceType_SOURCE_TYPE_GIT:
-		repo = r.SourceMetadata.GetGit().Repository
-	case sourcespb.SourceType_SOURCE_TYPE_GITHUB:
-		repo = r.SourceMetadata.GetGithub().Repository
-	case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
-		repo = r.SourceMetadata.GetGitlab().Repository
-	default:
-		logrus.Errorf("unsupported source type for legacy json output: %s", r.SourceType)
-	}
-
-	// cloning the repo again here is not great and only works with unauthed repos
-	repoPath, remote, err := git.PrepareRepo(repo)
+	repoPath, remote, err := git.PrepareRepo(r.SourceMetadata.GetGithub().Repository)
 	if err != nil || repoPath == "" {
 		logrus.WithError(err).Fatal("error preparing git repo for scanning")
 	}
 	if remote {
 		defer os.RemoveAll(repoPath)
 	}
-
 	legacy := ConvertToLegacyJSON(r, repoPath)
 	out, err := json.Marshal(legacy)
 	if err != nil {

pkg/sources/git/git.go

@@ -136,14 +136,12 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if len(repoURI) == 0 {
 				continue
 			}
-			err := func(repoURI string) error {
-				path, repo, err := CloneRepoUsingToken(token, repoURI, user)
-				defer os.RemoveAll(path)
-				if err != nil {
-					return err
-				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
-			}(repoURI)
+			path, repo, err := CloneRepoUsingToken(token, repoURI, user)
+			defer os.RemoveAll(path)
+			if err != nil {
+				return err
+			}
+			err = s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
 			}
@@ -154,14 +152,12 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if len(repoURI) == 0 {
 				continue
 			}
-			err := func(repoURI string) error {
-				path, repo, err := CloneRepoUsingUnauthenticated(repoURI)
-				defer os.RemoveAll(path)
-				if err != nil {
-					return err
-				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
-			}(repoURI)
+			path, repo, err := CloneRepoUsingUnauthenticated(repoURI)
+			defer os.RemoveAll(path)
+			if err != nil {
+				return err
+			}
+			err = s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
 			}
@@ -182,16 +178,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if err != nil {
 				return err
 			}
+			if strings.HasPrefix(u, filepath.Join(os.TempDir(), "trufflehog")) {
+				defer os.RemoveAll(u)
+			}
 
-			err = func(repoPath string) error {
-				if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
-					defer os.RemoveAll(repoPath)
-				}
-
-				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
-			}(u)
+			err = s.git.ScanRepo(ctx, repo, u, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
+
 			}
 		}
 

pkg/sources/github/github.go

@@ -24,11 +24,12 @@ import (
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -354,8 +355,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 	}
 
 	// If there is resume information available, limit this scan to only the repos that still need scanning.
-	reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
-	s.repos = reposToScan
+	progressIndexOffset := s.filterReposToResume(s.GetProgress().EncodedResumeInfo)
 
 	for i, repoURL := range s.repos {
 		if err := s.jobSem.Acquire(ctx, 1); err != nil {
@@ -370,13 +370,9 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch
 			defer s.jobSem.Release(1)
 			defer wg.Done()
 
-			s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)
+			s.setProgressCompleteWithRepo(i+progressIndexOffset, repoURL)
 			// Ensure the repo is removed from the resume info after being scanned.
-			defer func(s *Source) {
-				s.resumeInfoMutex.Lock()
-				defer s.resumeInfoMutex.Unlock()
-				s.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)
-			}(s)
+			defer s.removeRepoFromResumeInfo(repoURL)
 
 			if !strings.HasSuffix(repoURL, ".git") {
 				return
@@ -682,11 +678,7 @@ func (s *Source) addReposByApp(ctx context.Context, apiClient *github.Client) er
 
 func (s *Source) addAllVisibleOrgs(ctx context.Context, apiClient *github.Client) {
 	s.log.Debug("enumerating all visibile organizations on GHE")
-	// Enumeration on this endpoint does not use pages. it uses a since ID.
-	// The endpoint will return organizations with an ID greater than the given since ID.
-	// Empty org response is our cue to break the enumeration loop.
 	orgOpts := &github.OrganizationsListOptions{
-		Since: 0,
 		ListOptions: github.ListOptions{
 			PerPage: 100,
 		},
@@ -703,13 +695,7 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context, apiClient *github.Client
 			log.WithError(err).Errorf("Could not list all organizations")
 			return
 		}
-		if len(orgs) == 0 {
-			break
-		}
-		lastOrgID := *orgs[len(orgs)-1].ID
-		s.log.Debugf("listed organization IDs %d through %d", orgOpts.Since, lastOrgID)
-		orgOpts.Since = lastOrgID
-
+		s.log.Debugf("listed organization page %d/%d", orgOpts.Page, resp.LastPage)
 		for _, org := range orgs {
 			var name string
 			if org.Name != nil {
@@ -719,9 +705,13 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context, apiClient *github.Client
 			} else {
 				continue
 			}
-			s.log.Debugf("adding organization %d for repository enumeration: %s", org.ID, name)
+			s.log.Debug("adding organization for repository enumeration: ", name)
 			common.AddStringSliceItem(name, &s.orgs)
 		}
+		if resp.NextPage == 0 {
+			break
+		}
+		orgOpts.Page = resp.NextPage
 	}
 }
 
@@ -794,7 +784,7 @@ func (s *Source) normalizeRepos(ctx context.Context, apiClient *github.Client) {
 }
 
 // setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
-func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
+func (s *Source) setProgressCompleteWithRepo(index int, repoURL string) {
 	s.resumeInfoMutex.Lock()
 	defer s.resumeInfoMutex.Unlock()
 
@@ -803,7 +793,91 @@ func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL stri
 	sort.Strings(s.resumeInfoSlice)
 
 	// Make the resume info string from the slice.
-	encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)
+	encodedResumeInfo := s.encodeResumeInfo()
 
-	s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
+	s.SetProgressComplete(index, len(s.repos), fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
+}
+
+// removeRepoFromResumeInfo removes the repoURL from the resume info.
+func (s *Source) removeRepoFromResumeInfo(repoURL string) {
+	s.resumeInfoMutex.Lock()
+	defer s.resumeInfoMutex.Unlock()
+
+	index := -1
+	for i, repo := range s.resumeInfoSlice {
+		if repoURL == repo {
+			index = i
+		}
+	}
+
+	if index == -1 {
+		// We should never be able to be here. But if we are, it means the resume info never had the repo added.
+		// So log the error and do nothing.
+		s.log.Errorf("repoURL (%q) not found in list of encode resume info: %q", repoURL, s.EncodedResumeInfo)
+		return
+	}
+
+	// This removes the element at the given index.
+	s.resumeInfoSlice = append(s.resumeInfoSlice[:index], s.resumeInfoSlice[index+1:]...)
+}
+
+func (s *Source) encodeResumeInfo() string {
+	return strings.Join(s.resumeInfoSlice, "\t")
+}
+
+func (s *Source) decodeResumeInfo(resumeInfo string) {
+	// strings.Split will, for an empty string, return []string{""},
+	// which is an element, where as when there is no resume info we want an empty slice.
+	if resumeInfo == "" {
+		return
+	}
+	s.resumeInfoSlice = strings.Split(resumeInfo, "\t")
+}
+
+// filterReposToResume filters the existing repos down to those that are included in the encoded resume info.
+// It also returns the difference between the original length of the repos and the new length to use for progress reporting.
+// It is required that both the resumeInfo repos and the existing repos in s.repos are sorted.
+func (s *Source) filterReposToResume(resumeInfo string) int {
+	if resumeInfo == "" {
+		return 0
+	}
+
+	s.resumeInfoMutex.Lock()
+	defer s.resumeInfoMutex.Unlock()
+
+	s.decodeResumeInfo(resumeInfo)
+
+	// Because this scanner is multithreaded, it is possible that we have scanned a range of repositories
+	// with some gaps of unlisted but completed repositories in between the ones in resumeInfo.
+	// So we know repositories that have not finished scanning are the ones included in the resumeInfo,
+	// and those that come after the last repository in the resumeInfo.
+	// However, it is possible that a resumed scan does not include all or even any of the repos within the resumeInfo.
+	// In this case, we must ensure we still scan all repos that come after the last found repo in the list.
+	reposToScan := []string{}
+	lastFoundRepoIndex := -1
+	resumeRepoIndex := 0
+	for i, repoURL := range s.repos {
+		// If the repoURL is bigger than what we're looking for, move to the next one.
+		if repoURL > s.resumeInfoSlice[resumeRepoIndex] {
+			resumeRepoIndex++
+		}
+
+		// If we've found all of our repositories end the filter.
+		if resumeRepoIndex == len(s.resumeInfoSlice) {
+			break
+		}
+
+		// If the repoURL is the one we're looking for, add it and update the lastFoundRepoIndex.
+		if repoURL == s.resumeInfoSlice[resumeRepoIndex] {
+			lastFoundRepoIndex = i
+			reposToScan = append(reposToScan, repoURL)
+		}
+	}
+
+	// Append all repos after the last one we've found.
+	reposToScan = append(reposToScan, s.repos[lastFoundRepoIndex+1:]...)
+
+	progressOffsetCount := len(s.repos) - len(reposToScan)
+	s.repos = reposToScan
+	return progressOffsetCount
 }

pkg/sources/github/github_test.go

@@ -340,7 +340,7 @@ func TestEnumerateWithApp(t *testing.T) {
 }
 
 // This only tests the resume info slice portion of setProgressCompleteWithRepo.
-func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
+func Test_setProgressCompleteWithRepo(t *testing.T) {
 	tests := []struct {
 		startingResumeInfoSlice []string
 		repoURL                 string
@@ -367,68 +367,158 @@ func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
 
 	for _, tt := range tests {
 		s.resumeInfoSlice = tt.startingResumeInfoSlice
-		s.setProgressCompleteWithRepo(0, 0, tt.repoURL)
+		s.setProgressCompleteWithRepo(0, tt.repoURL)
 		if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
 			t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
 		}
 	}
 }
 
-func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
-	repos := []string{"a", "b", "c", "d", "e"}
-	tests := map[string]struct {
-		repos                 []string
-		index                 int
-		offset                int
-		wantPercentComplete   int64
-		wantSectionsCompleted int32
-		wantSectionsRemaining int32
+func Test_removeRepoFromResumeInfo(t *testing.T) {
+	tests := []struct {
+		startingResumeInfoSlice []string
+		repoURL                 string
+		wantResumeInfoSlice     []string
 	}{
-		"starting from the beginning, no offset": {
-			repos:                 repos,
-			index:                 0,
-			offset:                0,
-			wantPercentComplete:   0,
-			wantSectionsCompleted: 0,
-			wantSectionsRemaining: 5,
+		{
+			startingResumeInfoSlice: []string{"a", "b", "c"},
+			repoURL:                 "a",
+			wantResumeInfoSlice:     []string{"b", "c"},
 		},
-		"resume from the third, offset 2": {
-			repos:                 repos[2:],
-			index:                 0,
-			offset:                2,
-			wantPercentComplete:   40,
-			wantSectionsCompleted: 2,
-			wantSectionsRemaining: 5,
+		{
+			startingResumeInfoSlice: []string{"a", "b", "c"},
+			repoURL:                 "b",
+			wantResumeInfoSlice:     []string{"a", "c"},
 		},
-		"resume from the third, on last repo, offset 2": {
-			repos:                 repos[2:],
-			index:                 2,
-			offset:                2,
-			wantPercentComplete:   80,
-			wantSectionsCompleted: 4,
-			wantSectionsRemaining: 5,
+		{ // This is the probably can't happen case of a repo not in the list.
+			startingResumeInfoSlice: []string{"a", "b", "c"},
+			repoURL:                 "not in the list",
+			wantResumeInfoSlice:     []string{"a", "b", "c"},
 		},
 	}
 
 	logger := logrus.New()
 	logger.Out = io.Discard
+	s := &Source{
+		repos: []string{},
+		log:   logger.WithField("no", "output"),
+	}
 
 	for _, tt := range tests {
-		s := &Source{
-			repos: tt.repos,
-			log:   logger.WithField("no", "output"),
-		}
-
-		s.setProgressCompleteWithRepo(tt.index, tt.offset, "")
-		gotProgress := s.GetProgress()
-		if gotProgress.PercentComplete != tt.wantPercentComplete {
-			t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)
-		}
-		if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {
-			t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)
-		}
-		if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {
-			t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)
+		s.resumeInfoSlice = tt.startingResumeInfoSlice
+		s.removeRepoFromResumeInfo(tt.repoURL)
+		if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
+			t.Errorf("s.removeRepoFromResumeInfo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
+		}
+	}
+}
+
+func Test_encodeResumeInfo(t *testing.T) {
+	tests := []struct {
+		startingResumeInfoSlice []string
+		wantEncodedResumeInfo   string
+	}{
+		{
+			startingResumeInfoSlice: []string{"a", "b", "c"},
+			wantEncodedResumeInfo:   "a\tb\tc",
+		},
+		{
+			startingResumeInfoSlice: []string{},
+			wantEncodedResumeInfo:   "",
+		},
+	}
+
+	logger := logrus.New()
+	logger.Out = io.Discard
+	s := &Source{
+		repos: []string{},
+		log:   logger.WithField("no", "output"),
+	}
+
+	for _, tt := range tests {
+		s.resumeInfoSlice = tt.startingResumeInfoSlice
+		gotEncodedResumeInfo := s.encodeResumeInfo()
+		if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {
+			t.Errorf("s.encodeResumeInfo() got: %q, want: %q", gotEncodedResumeInfo, tt.wantEncodedResumeInfo)
+		}
+	}
+}
+
+func Test_decodeResumeInfo(t *testing.T) {
+	tests := []struct {
+		resumeInfo          string
+		wantResumeInfoSlice []string
+	}{
+		{
+			resumeInfo:          "a\tb\tc",
+			wantResumeInfoSlice: []string{"a", "b", "c"},
+		},
+		{
+			resumeInfo:          "",
+			wantResumeInfoSlice: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		s := &Source{}
+		s.decodeResumeInfo(tt.resumeInfo)
+		if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
+			t.Errorf("s.decodeResumeInfo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
+		}
+	}
+}
+
+func Test_filterReposToResume(t *testing.T) {
+	startingRepos := []string{"a", "b", "c", "d", "e", "f", "g"}
+
+	tests := map[string]struct {
+		resumeInfo              string
+		wantProgressOffsetCount int
+		wantReposToScan         []string
+	}{
+		"blank resume info": {
+			resumeInfo:              "",
+			wantProgressOffsetCount: 0,
+			wantReposToScan:         startingRepos,
+		},
+		"starting repos": {
+			resumeInfo:              "a\tb",
+			wantProgressOffsetCount: 0,
+			wantReposToScan:         startingRepos,
+		},
+		"early contiguous repos": {
+			resumeInfo:              "b\tc",
+			wantProgressOffsetCount: 1,
+			wantReposToScan:         []string{"b", "c", "d", "e", "f", "g"},
+		},
+		"non-contiguous repos": {
+			resumeInfo:              "b\te",
+			wantProgressOffsetCount: 3,
+			wantReposToScan:         []string{"b", "e", "f", "g"},
+		},
+		"no repos found in the repo list": {
+			resumeInfo:              "not\tthere",
+			wantProgressOffsetCount: 0,
+			wantReposToScan:         startingRepos,
+		},
+		"only some repos in the list": {
+			resumeInfo:              "c\tnot\tthere",
+			wantProgressOffsetCount: 2,
+			wantReposToScan:         []string{"c", "d", "e", "f", "g"},
+		},
+	}
+
+	for name, tt := range tests {
+		s := &Source{
+			repos: startingRepos,
+		}
+
+		gotProgressOffsetCount := s.filterReposToResume(tt.resumeInfo)
+		if gotProgressOffsetCount != tt.wantProgressOffsetCount {
+			t.Errorf("s.filterReposToResume() name: %q got: %d, want: %d", name, gotProgressOffsetCount, tt.wantProgressOffsetCount)
+		}
+		if !reflect.DeepEqual(s.repos, tt.wantReposToScan) {
+			t.Errorf("s.filterReposToResume() name: %q got: %v, want: %v", name, s.repos, tt.wantReposToScan)
 		}
 	}
 }

pkg/sources/gitlab/gitlab.go

@@ -6,10 +6,12 @@ import (
 	"net/url"
 	"os"
 	"runtime"
-	"sort"
 	"strings"
 	"sync"
 
+	"github.com/go-errors/errors"
+	gogit "github.com/go-git/go-git/v5"
+	log "github.com/sirupsen/logrus"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
@@ -17,10 +19,6 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
-
-	"github.com/go-errors/errors"
-	gogit "github.com/go-git/go-git/v5"
-	log "github.com/sirupsen/logrus"
 	"github.com/xanzy/go-gitlab"
 	"golang.org/x/sync/semaphore"
 	"google.golang.org/protobuf/proto"
@@ -28,22 +26,20 @@ import (
 )
 
 type Source struct {
-	name            string
-	sourceId        int64
-	jobId           int64
-	verify          bool
-	authMethod      string
-	user            string
-	password        string
-	token           string
-	url             string
-	repos           []string
-	git             *git.Git
-	aCtx            context.Context
-	resumeInfoSlice []string
-	resumeInfoMutex sync.Mutex
-	jobSem          *semaphore.Weighted
+	name       string
+	sourceId   int64
+	jobId      int64
+	verify     bool
+	authMethod string
+	user       string
+	password   string
+	token      string
+	url        string
+	repos      []string
+	git        *git.Git
+	aCtx       context.Context
 	sources.Progress
+	jobSem *semaphore.Weighted
 }
 
 // Ensure the Source satisfies the interface at compile time.
@@ -245,35 +241,33 @@ func (s *Source) getAllProjects(apiClient *gitlab.Client) ([]*gitlab.Project, er
 	return projectList, nil
 }
 
-func (s *Source) getRepos() ([]string, []error) {
-	if len(s.repos) == 0 {
-		return nil, nil
-	}
-
-	var validRepos []string
+func (s *Source) getRepos() ([]*url.URL, []error) {
+	var validRepos []*url.URL
 	var errs []error
-	for _, prj := range s.repos {
-		repo, err := giturl.NormalizeGitlabRepo(prj)
-		if err != nil {
-			errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
-			continue
-		}
+	if len(s.repos) > 0 {
+		for _, prj := range s.repos {
+			repo, err := giturl.NormalizeGitlabRepo(prj)
+			if err != nil {
+				errs = append(errs, errors.WrapPrefix(err, fmt.Sprintf("unable to normalize gitlab repo url %s", prj), 0))
+				continue
+			}
 
-		validRepos = append(validRepos, repo)
+			// The repo normalization has already successfully parsed the URL at this point, so we can ignore the error.
+			u, _ := url.ParseRequestURI(repo)
+			validRepos = append(validRepos, u)
+		}
+		return validRepos, errs
 	}
-	return validRepos, errs
+	return nil, nil
+
 }
 
-func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk) []error {
+func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk, repos []*url.URL) []error {
 	wg := sync.WaitGroup{}
 	var errs []error
 	var errsMut sync.Mutex
 
-	// If there is resume information available, limit this scan to only the repos that still need scanning.
-	reposToScan, progressIndexOffset := sources.FilterReposToResume(s.repos, s.GetProgress().EncodedResumeInfo)
-	s.repos = reposToScan
-
-	for i, repo := range s.repos {
+	for i, u := range repos {
 		if common.IsDone(ctx) {
 			// We are returning nil instead of the scanErrors slice here because
 			// we don't want to mark this scan as errored if we cancelled it.
@@ -284,26 +278,19 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 			continue
 		}
 		wg.Add(1)
-		go func(ctx context.Context, repoURL string, i int) {
+		go func(ctx context.Context, repoURL *url.URL, i int) {
 			defer s.jobSem.Release(1)
 			defer wg.Done()
-			if len(repoURL) == 0 {
+			if len(repoURL.String()) == 0 {
 				return
 			}
-
-			s.setProgressCompleteWithRepo(i, progressIndexOffset, repoURL)
-			// Ensure the repo is removed from the resume info after being scanned.
-			defer func(s *Source) {
-				s.resumeInfoMutex.Lock()
-				defer s.resumeInfoMutex.Unlock()
-				s.resumeInfoSlice = sources.RemoveRepoFromResumeInfo(s.resumeInfoSlice, repoURL)
-			}(s)
+			s.SetProgressComplete(i, len(repos), fmt.Sprintf("Repo: %s", repoURL), "")
 
 			var path string
 			var repo *gogit.Repository
 			var err error
 			if s.authMethod == "UNAUTHENTICATED" {
-				path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL)
+				path, repo, err = git.CloneRepoUsingUnauthenticated(repoURL.String())
 			} else {
 				// If a username is not provided we need to use a default one in order to clone a private repo.
 				// Not setting "placeholder" as s.user on purpose in case any downstream services rely on a "" value for s.user.
@@ -311,7 +298,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				if user == "" {
 					user = "placeholder"
 				}
-				path, repo, err = git.CloneRepoUsingToken(s.token, repoURL, user)
+				path, repo, err = git.CloneRepoUsingToken(s.token, repoURL.String(), user)
 			}
 			defer os.RemoveAll(path)
 			if err != nil {
@@ -320,7 +307,7 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				errsMut.Unlock()
 				return
 			}
-			log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(s.repos), repoURL)
+			log.Debugf("Starting to scan repo %d/%d: %s", i+1, len(repos), repoURL.String())
 			err = s.git.ScanRepo(ctx, repo, path, git.NewScanOptions(), chunksChan)
 			if err != nil {
 				errsMut.Lock()
@@ -328,8 +315,8 @@ func (s *Source) scanRepos(ctx context.Context, chunksChan chan *sources.Chunk)
 				errsMut.Unlock()
 				return
 			}
-			log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(s.repos), repoURL)
-		}(ctx, repo, i)
+			log.Debugf("Completed scanning repo %d/%d: %s", i+1, len(repos), repoURL.String())
+		}(ctx, u, i)
 	}
 	wg.Wait()
 
@@ -362,23 +349,17 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		}
 		// Turn projects into URLs for Git cloner.
 		for _, prj := range projects {
-			// Ensure the urls are valid before adding them to the repo list.
-			_, err := url.Parse(prj.HTTPURLToRepo)
+			u, err := url.Parse(prj.HTTPURLToRepo)
 			if err != nil {
 				fmt.Printf("could not parse url given by project: %s", prj.HTTPURLToRepo)
 			}
-			repos = append(repos, prj.HTTPURLToRepo)
+			repos = append(repos, u)
 		}
 		if repos == nil {
 			return errors.Errorf("unable to discover any repos")
 		}
 	}
-
-	s.repos = repos
-	// We must sort the repos so we can resume later if necessary.
-	sort.Strings(s.repos)
-
-	errs = s.scanRepos(ctx, chunksChan)
+	errs = s.scanRepos(ctx, chunksChan, repos)
 	for _, err := range errs {
 		log.WithError(err).WithFields(
 			log.Fields{
@@ -405,19 +386,3 @@ func (s *Source) basicAuthSuccessful(apiClient *gitlab.Client) bool {
 	}
 	return false
 }
-
-// setProgressCompleteWithRepo calls the s.SetProgressComplete after safely setting up the encoded resume info string.
-func (s *Source) setProgressCompleteWithRepo(index int, offset int, repoURL string) {
-	s.resumeInfoMutex.Lock()
-	defer s.resumeInfoMutex.Unlock()
-
-	// Add the repoURL to the resume info slice.
-	s.resumeInfoSlice = append(s.resumeInfoSlice, repoURL)
-	sort.Strings(s.resumeInfoSlice)
-
-	// Make the resume info string from the slice.
-	encodedResumeInfo := sources.EncodeResumeInfo(s.resumeInfoSlice)
-
-	// Add the offset to both the index and the repos to give the proper place and proper repo count.
-	s.SetProgressComplete(index+offset, len(s.repos)+offset, fmt.Sprintf("Repo: %s", repoURL), encodedResumeInfo)
-}

pkg/sources/gitlab/gitlab_test.go

@@ -3,8 +3,6 @@ package gitlab
 import (
 	"context"
 	"fmt"
-	"io"
-	"reflect"
 	"testing"
 
 	"github.com/kylelemons/godebug/pretty"
@@ -162,90 +160,3 @@ func TestSource_Scan(t *testing.T) {
 		})
 	}
 }
-
-func Test_setProgressCompleteWithRepo_resumeInfo(t *testing.T) {
-	tests := []struct {
-		startingResumeInfoSlice []string
-		repoURL                 string
-		wantResumeInfoSlice     []string
-	}{
-		{
-			startingResumeInfoSlice: []string{},
-			repoURL:                 "a",
-			wantResumeInfoSlice:     []string{"a"},
-		},
-		{
-			startingResumeInfoSlice: []string{"b"},
-			repoURL:                 "a",
-			wantResumeInfoSlice:     []string{"a", "b"},
-		},
-	}
-
-	log.SetOutput(io.Discard)
-	s := &Source{repos: []string{}}
-
-	for _, tt := range tests {
-		s.resumeInfoSlice = tt.startingResumeInfoSlice
-		s.setProgressCompleteWithRepo(0, 0, tt.repoURL)
-		if !reflect.DeepEqual(s.resumeInfoSlice, tt.wantResumeInfoSlice) {
-			t.Errorf("s.setProgressCompleteWithRepo() got: %v, want: %v", s.resumeInfoSlice, tt.wantResumeInfoSlice)
-		}
-	}
-}
-
-func Test_setProgressCompleteWithRepo_Progress(t *testing.T) {
-	repos := []string{"a", "b", "c", "d", "e"}
-	tests := map[string]struct {
-		repos                 []string
-		index                 int
-		offset                int
-		wantPercentComplete   int64
-		wantSectionsCompleted int32
-		wantSectionsRemaining int32
-	}{
-		"starting from the beginning, no offset": {
-			repos:                 repos,
-			index:                 0,
-			offset:                0,
-			wantPercentComplete:   0,
-			wantSectionsCompleted: 0,
-			wantSectionsRemaining: 5,
-		},
-		"resume from the third, offset 2": {
-			repos:                 repos[2:],
-			index:                 0,
-			offset:                2,
-			wantPercentComplete:   40,
-			wantSectionsCompleted: 2,
-			wantSectionsRemaining: 5,
-		},
-		"resume from the third, on last repo, offset 2": {
-			repos:                 repos[2:],
-			index:                 2,
-			offset:                2,
-			wantPercentComplete:   80,
-			wantSectionsCompleted: 4,
-			wantSectionsRemaining: 5,
-		},
-	}
-
-	log.SetOutput(io.Discard)
-
-	for _, tt := range tests {
-		s := &Source{
-			repos: tt.repos,
-		}
-
-		s.setProgressCompleteWithRepo(tt.index, tt.offset, "")
-		gotProgress := s.GetProgress()
-		if gotProgress.PercentComplete != tt.wantPercentComplete {
-			t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.PercentComplete, tt.wantPercentComplete)
-		}
-		if gotProgress.SectionsCompleted != tt.wantSectionsCompleted {
-			t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsCompleted, tt.wantSectionsCompleted)
-		}
-		if gotProgress.SectionsRemaining != tt.wantSectionsRemaining {
-			t.Errorf("s.setProgressCompleteWithRepo() PercentComplete got: %v want: %v", gotProgress.SectionsRemaining, tt.wantSectionsRemaining)
-		}
-	}
-}

pkg/sources/resume.go

@@ -1,84 +0,0 @@
-package sources
-
-import (
-	"strings"
-
-	"github.com/sirupsen/logrus"
-)
-
-// RemoveRepoFromResumeInfo removes the repoURL from the resume info.
-func RemoveRepoFromResumeInfo(resumeRepos []string, repoURL string) []string {
-	index := -1
-	for i, repo := range resumeRepos {
-		if repoURL == repo {
-			index = i
-			break
-		}
-	}
-
-	if index == -1 {
-		// We should never be able to be here. But if we are, it means the resume info never had the repo added.
-		// So log the error and do nothing.
-		logrus.Errorf("repoURL (%q) not found in list of encode resume info: %v", repoURL, resumeRepos)
-		return resumeRepos
-	}
-
-	// This removes the element at the given index.
-	return append(resumeRepos[:index], resumeRepos[index+1:]...)
-}
-
-// FilterReposToResume filters the existing repos down to those that are included in the encoded resume info.
-// It returns the new slice of repos to be scanned.
-// It also returns the difference between the original length of the repos and the new length to use for progress reporting.
-// It is required that both the resumeInfo repos and the existing repos are sorted.
-func FilterReposToResume(repos []string, resumeInfo string) (reposToScan []string, progressOffsetCount int) {
-	if resumeInfo == "" {
-		return repos, 0
-	}
-
-	resumeInfoSlice := DecodeResumeInfo(resumeInfo)
-
-	// Because this scanner is multithreaded, it is possible that we have scanned a range of repositories
-	// with some gaps of unlisted but completed repositories in between the ones in resumeInfo.
-	// So we know repositories that have not finished scanning are the ones included in the resumeInfo,
-	// and those that come after the last repository in the resumeInfo.
-	// However, it is possible that a resumed scan does not include all or even any of the repos within the resumeInfo.
-	// In this case, we must ensure we still scan all repos that come after the last found repo in the list.
-	lastFoundRepoIndex := -1
-	resumeRepoIndex := 0
-	for i, repoURL := range repos {
-		// If the repoURL is bigger than what we're looking for, move to the next one.
-		if repoURL > resumeInfoSlice[resumeRepoIndex] {
-			resumeRepoIndex++
-		}
-
-		// If we've found all of our repositories end the filter.
-		if resumeRepoIndex == len(resumeInfoSlice) {
-			break
-		}
-
-		// If the repoURL is the one we're looking for, add it and update the lastFoundRepoIndex.
-		if repoURL == resumeInfoSlice[resumeRepoIndex] {
-			lastFoundRepoIndex = i
-			reposToScan = append(reposToScan, repoURL)
-		}
-	}
-
-	// Append all repos after the last one we've found.
-	reposToScan = append(reposToScan, repos[lastFoundRepoIndex+1:]...)
-	progressOffsetCount = len(repos) - len(reposToScan)
-	return
-}
-
-func EncodeResumeInfo(resumeInfoSlice []string) string {
-	return strings.Join(resumeInfoSlice, "\t")
-}
-
-func DecodeResumeInfo(resumeInfo string) []string {
-	// strings.Split will, for an empty string, return []string{""},
-	// which is an element, where as when there is no resume info we want an empty slice.
-	if resumeInfo == "" {
-		return nil
-	}
-	return strings.Split(resumeInfo, "\t")
-}

pkg/sources/resume_test.go

@@ -1,143 +0,0 @@
-package sources
-
-import (
-	"io"
-	"reflect"
-	"testing"
-
-	"github.com/sirupsen/logrus"
-)
-
-func TestRemoveRepoFromResumeInfo(t *testing.T) {
-	tests := []struct {
-		startingResumeInfoSlice []string
-		repoURL                 string
-		wantResumeInfoSlice     []string
-	}{
-		{
-			startingResumeInfoSlice: []string{"a", "b", "c"},
-			repoURL:                 "a",
-			wantResumeInfoSlice:     []string{"b", "c"},
-		},
-		{
-			startingResumeInfoSlice: []string{"a", "b", "c"},
-			repoURL:                 "b",
-			wantResumeInfoSlice:     []string{"a", "c"},
-		},
-		{ // This is the probably can't happen case of a repo not in the list.
-			startingResumeInfoSlice: []string{"a", "b", "c"},
-			repoURL:                 "not in the list",
-			wantResumeInfoSlice:     []string{"a", "b", "c"},
-		},
-	}
-
-	// Prevent the error message from showing up.
-	logrus.SetOutput(io.Discard)
-
-	for _, tt := range tests {
-		gotResumeInfoSlice := RemoveRepoFromResumeInfo(tt.startingResumeInfoSlice, tt.repoURL)
-		if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {
-			t.Errorf("RemoveRepoFromResumeInfo() got: %v, want: %v", gotResumeInfoSlice, tt.wantResumeInfoSlice)
-		}
-	}
-}
-
-func TestEncodeResumeInfo(t *testing.T) {
-	tests := []struct {
-		startingResumeInfoSlice []string
-		wantEncodedResumeInfo   string
-	}{
-		{
-			startingResumeInfoSlice: []string{"a", "b", "c"},
-			wantEncodedResumeInfo:   "a\tb\tc",
-		},
-		{
-			startingResumeInfoSlice: []string{},
-			wantEncodedResumeInfo:   "",
-		},
-	}
-
-	logger := logrus.New()
-	logger.Out = io.Discard
-
-	for _, tt := range tests {
-		gotEncodedResumeInfo := EncodeResumeInfo(tt.startingResumeInfoSlice)
-		if gotEncodedResumeInfo != tt.wantEncodedResumeInfo {
-			t.Errorf("EncodeResumeInfo() got: %q, want: %q", gotEncodedResumeInfo, tt.wantEncodedResumeInfo)
-		}
-	}
-}
-
-func Test_decodeResumeInfo(t *testing.T) {
-	tests := []struct {
-		resumeInfo          string
-		wantResumeInfoSlice []string
-	}{
-		{
-			resumeInfo:          "a\tb\tc",
-			wantResumeInfoSlice: []string{"a", "b", "c"},
-		},
-		{
-			resumeInfo:          "",
-			wantResumeInfoSlice: nil,
-		},
-	}
-
-	for _, tt := range tests {
-		gotResumeInfoSlice := DecodeResumeInfo(tt.resumeInfo)
-		if !reflect.DeepEqual(gotResumeInfoSlice, tt.wantResumeInfoSlice) {
-			t.Errorf("DecodeResumeInfo() got: %v, want: %v", gotResumeInfoSlice, tt.wantResumeInfoSlice)
-		}
-	}
-}
-
-func Test_filterReposToResume(t *testing.T) {
-	startingRepos := []string{"a", "b", "c", "d", "e", "f", "g"}
-
-	tests := map[string]struct {
-		resumeInfo              string
-		wantProgressOffsetCount int
-		wantReposToScan         []string
-	}{
-		"blank resume info": {
-			resumeInfo:              "",
-			wantProgressOffsetCount: 0,
-			wantReposToScan:         startingRepos,
-		},
-		"starting repos": {
-			resumeInfo:              "a\tb",
-			wantProgressOffsetCount: 0,
-			wantReposToScan:         startingRepos,
-		},
-		"early contiguous repos": {
-			resumeInfo:              "b\tc",
-			wantProgressOffsetCount: 1,
-			wantReposToScan:         []string{"b", "c", "d", "e", "f", "g"},
-		},
-		"non-contiguous repos": {
-			resumeInfo:              "b\te",
-			wantProgressOffsetCount: 3,
-			wantReposToScan:         []string{"b", "e", "f", "g"},
-		},
-		"no repos found in the repo list": {
-			resumeInfo:              "not\tthere",
-			wantProgressOffsetCount: 0,
-			wantReposToScan:         startingRepos,
-		},
-		"only some repos in the list": {
-			resumeInfo:              "c\tnot\tthere",
-			wantProgressOffsetCount: 2,
-			wantReposToScan:         []string{"c", "d", "e", "f", "g"},
-		},
-	}
-
-	for name, tt := range tests {
-		gotReposToScan, gotProgressOffsetCount := FilterReposToResume(startingRepos, tt.resumeInfo)
-		if !reflect.DeepEqual(gotReposToScan, tt.wantReposToScan) {
-			t.Errorf("FilterReposToResume() name: %q got: %v, want: %v", name, gotReposToScan, tt.wantReposToScan)
-		}
-		if gotProgressOffsetCount != tt.wantProgressOffsetCount {
-			t.Errorf("FilterReposToResume() name: %q got: %d, want: %d", name, gotProgressOffsetCount, tt.wantProgressOffsetCount)
-		}
-	}
-}

pkg/sources/s3/s3.go

@@ -248,9 +248,9 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				errorCount.Store(prefix, nErr)
 				//too many consective errors on this page
 				if nErr.(int) > 3 {
-					s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)
+					s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
 				}
-				log.Debugf("Error Counts: %s:%d", prefix, nErr)
+				log.Debugf("Error Counts: %s:%s", prefix, nErr)
 				return
 			}
 			body, err := ioutil.ReadAll(res.Body)
@@ -269,7 +269,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				errorCount.Store(prefix, nErr)
 
 				if nErr.(int) > 3 {
-					s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)
+					s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
 				}
 				return
 			}