fixup - remove change in behavior, cannot yet confirm that results have unique ids

go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
 
-replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7
+replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3
 
 require (
 	cloud.google.com/go/secretmanager v1.4.0
@@ -17,7 +17,7 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.6.7
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
-	github.com/gitleaks/go-gitdiff v0.7.4
+	github.com/gitleaks/go-gitdiff v0.7.6
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible

go.sum

@@ -142,8 +142,8 @@ github.com/felixge/fgprof v0.9.2 h1:tAMHtWMyl6E0BimjVbFt7fieU6FpjttsZN7j0wT5blc=
 github.com/felixge/fgprof v0.9.2/go.mod h1:+VNi+ZXtHIQ6wIw6bUT8nXQRefQflWECoFyRealT5sg=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gitleaks/go-gitdiff v0.7.4 h1:8vICc4moyRR2poklblThdQ0ckMet22mEvFJSxPsiDlk=
-github.com/gitleaks/go-gitdiff v0.7.4/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
+github.com/gitleaks/go-gitdiff v0.7.6 h1:atcfoNPD9erzPs9C89a+i2Y+EUmR2QKB5QHJTfB4n60=
+github.com/gitleaks/go-gitdiff v0.7.6/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
 github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -361,8 +361,8 @@ github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7 h1:K4Ah1RMmlKjvIwLEdn9kADugbTd9AM7RYxVOsrkmcTk=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3 h1:Xc61NkfI7aDHd8eHa0gglK0ZVF5UF54M4u4C5tuAKcw=
+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3/go.mod h1:Em2rda83ePrhmaX4ZdvNjnUADRiOJirEAqln0ZtN8og=
 github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
 github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
 github.com/xanzy/go-gitlab v0.65.0 h1:9xSA9cRVhz3Z54JacIHdvWnNmNAoSz/BDnyMGOf3yIg=

pkg/detectors/aws/aws.go

@@ -149,10 +149,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			results = append(results, s1)
-			// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
-			if s1.Verified {
-				break
-			}
 		}
 	}
 	return detectors.CleanResults(results), nil

pkg/sources/git/git.go

@@ -136,14 +136,12 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if len(repoURI) == 0 {
 				continue
 			}
-			err := func(repoURI string) error {
-				path, repo, err := CloneRepoUsingToken(token, repoURI, user)
-				defer os.RemoveAll(path)
-				if err != nil {
-					return err
-				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
-			}(repoURI)
+			path, repo, err := CloneRepoUsingToken(token, repoURI, user)
+			defer os.RemoveAll(path)
+			if err != nil {
+				return err
+			}
+			err = s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
 			}
@@ -154,14 +152,12 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if len(repoURI) == 0 {
 				continue
 			}
-			err := func(repoURI string) error {
-				path, repo, err := CloneRepoUsingUnauthenticated(repoURI)
-				defer os.RemoveAll(path)
-				if err != nil {
-					return err
-				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
-			}(repoURI)
+			path, repo, err := CloneRepoUsingUnauthenticated(repoURI)
+			defer os.RemoveAll(path)
+			if err != nil {
+				return err
+			}
+			err = s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
 			}
@@ -182,16 +178,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if err != nil {
 				return err
 			}
+			if strings.HasPrefix(u, filepath.Join(os.TempDir(), "trufflehog")) {
+				defer os.RemoveAll(u)
+			}
 
-			err = func(repoPath string) error {
-				if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
-					defer os.RemoveAll(repoPath)
-				}
-
-				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
-			}(u)
+			err = s.git.ScanRepo(ctx, repo, u, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
+
 			}
 		}
 

pkg/sources/github/github.go

@@ -678,11 +678,7 @@ func (s *Source) addReposByApp(ctx context.Context, apiClient *github.Client) er
 
 func (s *Source) addAllVisibleOrgs(ctx context.Context, apiClient *github.Client) {
 	s.log.Debug("enumerating all visibile organizations on GHE")
-	// Enumeration on this endpoint does not use pages. it uses a since ID.
-	// The endpoint will return organizations with an ID greater than the given since ID.
-	// Empty org response is our cue to break the enumeration loop.
 	orgOpts := &github.OrganizationsListOptions{
-		Since: 0,
 		ListOptions: github.ListOptions{
 			PerPage: 100,
 		},
@@ -699,13 +695,7 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context, apiClient *github.Client
 			log.WithError(err).Errorf("Could not list all organizations")
 			return
 		}
-		if len(orgs) == 0 {
-			break
-		}
-		lastOrgID := *orgs[len(orgs)-1].ID
-		s.log.Debugf("listed organization IDs %d through %d", orgOpts.Since, lastOrgID)
-		orgOpts.Since = lastOrgID
-
+		s.log.Debugf("listed organization page %d/%d", orgOpts.Page, resp.LastPage)
 		for _, org := range orgs {
 			var name string
 			if org.Name != nil {
@@ -715,9 +705,13 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context, apiClient *github.Client
 			} else {
 				continue
 			}
-			s.log.Debugf("adding organization %d for repository enumeration: %s", org.ID, name)
+			s.log.Debug("adding organization for repository enumeration: ", name)
 			common.AddStringSliceItem(name, &s.orgs)
 		}
+		if resp.NextPage == 0 {
+			break
+		}
+		orgOpts.Page = resp.NextPage
 	}
 }
 

pkg/sources/s3/s3.go

@@ -248,9 +248,9 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				errorCount.Store(prefix, nErr)
 				//too many consective errors on this page
 				if nErr.(int) > 3 {
-					s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)
+					s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
 				}
-				log.Debugf("Error Counts: %s:%d", prefix, nErr)
+				log.Debugf("Error Counts: %s:%s", prefix, nErr)
 				return
 			}
 			body, err := ioutil.ReadAll(res.Body)
@@ -269,7 +269,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				errorCount.Store(prefix, nErr)
 
 				if nErr.(int) > 3 {
-					s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)
+					s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
 				}
 				return
 			}