Initialize resumeInfoMutex in github

go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 replace github.com/jpillora/overseer => github.com/trufflesecurity/overseer v1.1.7-custom5
 
-replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7
+replace github.com/zricethezav/gitleaks/v8 => github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3
 
 require (
 	cloud.google.com/go/secretmanager v1.4.0
@@ -17,7 +17,7 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.6.7
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
-	github.com/gitleaks/go-gitdiff v0.7.4
+	github.com/gitleaks/go-gitdiff v0.7.6
 	github.com/go-errors/errors v1.4.2
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/golang-jwt/jwt v3.2.2+incompatible

go.sum

@@ -142,8 +142,8 @@ github.com/felixge/fgprof v0.9.2 h1:tAMHtWMyl6E0BimjVbFt7fieU6FpjttsZN7j0wT5blc=
 github.com/felixge/fgprof v0.9.2/go.mod h1:+VNi+ZXtHIQ6wIw6bUT8nXQRefQflWECoFyRealT5sg=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gitleaks/go-gitdiff v0.7.4 h1:8vICc4moyRR2poklblThdQ0ckMet22mEvFJSxPsiDlk=
-github.com/gitleaks/go-gitdiff v0.7.4/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
+github.com/gitleaks/go-gitdiff v0.7.6 h1:atcfoNPD9erzPs9C89a+i2Y+EUmR2QKB5QHJTfB4n60=
+github.com/gitleaks/go-gitdiff v0.7.6/go.mod h1:pKz0X4YzCKZs30BL+weqBIG7mx0jl4tF1uXV9ZyNvrA=
 github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@@ -361,8 +361,8 @@ github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7 h1:K4Ah1RMmlKjvIwLEdn9kADugbTd9AM7RYxVOsrkmcTk=
-github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom7/go.mod h1:2iZpX4Epnmx7VK2atbIMEjHW9rivie5RRe0ZhPWUFvM=
+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3 h1:Xc61NkfI7aDHd8eHa0gglK0ZVF5UF54M4u4C5tuAKcw=
+github.com/trufflesecurity/gitleaks/v8 v8.6.1-custom3/go.mod h1:Em2rda83ePrhmaX4ZdvNjnUADRiOJirEAqln0ZtN8og=
 github.com/trufflesecurity/overseer v1.1.7-custom5 h1:xu+Fg6fkSRifUPzUCl7N8HmobJ6WGOkIApGnM7mJS6w=
 github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhNfzAFaPjthvxgpDV3XKsxYkcI=
 github.com/xanzy/go-gitlab v0.65.0 h1:9xSA9cRVhz3Z54JacIHdvWnNmNAoSz/BDnyMGOf3yIg=

pkg/detectors/aws/aws.go

@@ -26,8 +26,8 @@ var (
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
-	idPat     = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
-	secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)
+	keyPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)
+	idPat  = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -58,25 +58,25 @@ func GetHMAC(key []byte, data []byte) []byte {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
-	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
 
-	for _, idMatch := range idMatches {
-		if len(idMatch) != 2 {
+	for _, match := range matches {
+		if len(match) != 2 {
 			continue
 		}
-		resIDMatch := strings.TrimSpace(idMatch[1])
+		resMatch := strings.TrimSpace(match[1])
 
-		for _, secretMatch := range secretMatches {
-			if len(secretMatch) != 2 {
+		for _, idMatch := range idMatches {
+			if len(idMatch) != 2 {
 				continue
 			}
-			resSecretMatch := strings.TrimSpace(secretMatch[1])
+
+			resIdMatch := strings.TrimSpace(idMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_AWS,
-				Raw:          []byte(resIDMatch),
-				Redacted:     resIDMatch,
+				Raw:          []byte(resMatch),
 			}
 
 			if verify {
@@ -106,7 +106,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				params.Add("Action", "GetCallerIdentity")
 				params.Add("Version", "2011-06-15")
 				params.Add("X-Amz-Algorithm", algorithm)
-				params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
+				params.Add("X-Amz-Credential", resIdMatch+"/"+credentialScope)
 				params.Add("X-Amz-Date", amzDate)
 				params.Add("X-Amz-Expires", "30")
 				params.Add("X-Amz-SignedHeaders", signedHeaders)
@@ -120,7 +120,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				// TASK 3: CALCULATE THE SIGNATURE.
 				// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
-				hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
+				hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resMatch)), []byte(datestamp))
 				hash = GetHMAC(hash, []byte(region))
 				hash = GetHMAC(hash, []byte(service))
 				hash = GetHMAC(hash, []byte("aws4_request"))
@@ -135,13 +135,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				res, err := client.Do(req)
 				if err == nil {
-					// Response body is unused so close it immediately.
-					res.Body.Close()
+					defer res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
 						// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
-						if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
+						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}
@@ -149,10 +148,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 
 			results = append(results, s1)
-			// If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
-			if s1.Verified {
-				break
-			}
 		}
 	}
 	return detectors.CleanResults(results), nil

pkg/detectors/aws/aws_test.go

@@ -48,7 +48,6 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
-					Redacted:     "AKIAWARWQKZNHMZBLY4I",
 				},
 			},
 			wantErr: false,
@@ -65,7 +64,6 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
-					Redacted:     "AKIAWARWQKZNHMZBLY4I",
 				},
 			},
 			wantErr: false,

pkg/sources/git/git.go

@@ -136,14 +136,12 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if len(repoURI) == 0 {
 				continue
 			}
-			err := func(repoURI string) error {
-				path, repo, err := CloneRepoUsingToken(token, repoURI, user)
-				defer os.RemoveAll(path)
-				if err != nil {
-					return err
-				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
-			}(repoURI)
+			path, repo, err := CloneRepoUsingToken(token, repoURI, user)
+			defer os.RemoveAll(path)
+			if err != nil {
+				return err
+			}
+			err = s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
 			}
@@ -154,14 +152,12 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if len(repoURI) == 0 {
 				continue
 			}
-			err := func(repoURI string) error {
-				path, repo, err := CloneRepoUsingUnauthenticated(repoURI)
-				defer os.RemoveAll(path)
-				if err != nil {
-					return err
-				}
-				return s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
-			}(repoURI)
+			path, repo, err := CloneRepoUsingUnauthenticated(repoURI)
+			defer os.RemoveAll(path)
+			if err != nil {
+				return err
+			}
+			err = s.git.ScanRepo(ctx, repo, path, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
 			}
@@ -182,16 +178,14 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			if err != nil {
 				return err
 			}
+			if strings.HasPrefix(u, filepath.Join(os.TempDir(), "trufflehog")) {
+				defer os.RemoveAll(u)
+			}
 
-			err = func(repoPath string) error {
-				if strings.HasPrefix(repoPath, filepath.Join(os.TempDir(), "trufflehog")) {
-					defer os.RemoveAll(repoPath)
-				}
-
-				return s.git.ScanRepo(ctx, repo, repoPath, NewScanOptions(), chunksChan)
-			}(u)
+			err = s.git.ScanRepo(ctx, repo, u, NewScanOptions(), chunksChan)
 			if err != nil {
 				return err
+
 			}
 		}
 

pkg/sources/github/github.go

@@ -105,6 +105,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
 	s.jobID = jobID
 	s.verify = verify
 	s.jobSem = semaphore.NewWeighted(int64(concurrency))
+	s.resumeInfoMutex = &sync.Mutex{}
 
 	s.httpClient = common.SaneHttpClient()
 
@@ -172,12 +173,10 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
 	var err error
 	// If we're using public github, make a regular client.
 	// Otherwise make an enterprise client
-	var isGHE bool
 	var apiClient *github.Client
 	if apiEndpoint == "https://api.github.com" {
 		apiClient = github.NewClient(tc)
 	} else {
-		isGHE = true
 		apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, tc)
 		if err != nil {
 			return nil, errors.New(err)
@@ -213,15 +212,9 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
 		if err := s.addReposByUser(ctx, apiClient, user.GetLogin()); err != nil {
 			log.WithError(err).Error("error fetching repos by user")
 		}
-
-		if isGHE {
-			s.addAllVisibleOrgs(ctx, apiClient)
-		} else {
-			// Scan for orgs is default with a token. GitHub App enumerates the repositories
-			// that were assigned to it in GitHub App settings.
-			s.addOrgsByUser(ctx, apiClient, user.GetLogin())
-		}
-
+		// Scan for orgs is default with a token. GitHub App enumerates the repositories
+		// that were assigned to it in GitHub App settings.
+		s.addOrgsByUser(ctx, apiClient, user.GetLogin())
 		for _, org := range s.orgs {
 			if err := s.addReposByOrg(ctx, apiClient, org); err != nil {
 				log.WithError(err).Error("error fetching repos by org")
@@ -464,8 +457,6 @@ func handleRateLimit(errIn error, res *github.Response) bool {
 }
 
 func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, org string) ([]string, error) {
-	log := s.log.WithField("org", org)
-
 	repos := []string{}
 	opts := &github.RepositoryListByOrgOptions{
 		ListOptions: github.ListOptions{
@@ -484,7 +475,6 @@ func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, or
 		if err != nil {
 			return nil, fmt.Errorf("could not list repos for org %s: %w", org, err)
 		}
-		log.Debugf("listed repos page %d/%d", opts.Page, res.LastPage)
 		if len(someRepos) == 0 {
 			break
 		}
@@ -503,7 +493,7 @@ func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, or
 		}
 		opts.Page = res.NextPage
 	}
-	log.Debugf("found %d repos (%d forks)", numRepos, numForks)
+	log.WithField("org", org).Debugf("Found %d repos (%d forks)", numRepos, numForks)
 	return repos, nil
 }
 
@@ -676,51 +666,6 @@ func (s *Source) addReposByApp(ctx context.Context, apiClient *github.Client) er
 	return nil
 }
 
-func (s *Source) addAllVisibleOrgs(ctx context.Context, apiClient *github.Client) {
-	s.log.Debug("enumerating all visibile organizations on GHE")
-	// Enumeration on this endpoint does not use pages. it uses a since ID.
-	// The endpoint will return organizations with an ID greater than the given since ID.
-	// Empty org response is our cue to break the enumeration loop.
-	orgOpts := &github.OrganizationsListOptions{
-		Since: 0,
-		ListOptions: github.ListOptions{
-			PerPage: 100,
-		},
-	}
-	for {
-		orgs, resp, err := apiClient.Organizations.ListAll(ctx, orgOpts)
-		if err == nil {
-			defer resp.Body.Close()
-		}
-		if handled := handleRateLimit(err, resp); handled {
-			continue
-		}
-		if err != nil {
-			log.WithError(err).Errorf("Could not list all organizations")
-			return
-		}
-		if len(orgs) == 0 {
-			break
-		}
-		lastOrgID := *orgs[len(orgs)-1].ID
-		s.log.Debugf("listed organization IDs %d through %d", orgOpts.Since, lastOrgID)
-		orgOpts.Since = lastOrgID
-
-		for _, org := range orgs {
-			var name string
-			if org.Name != nil {
-				name = *org.Name
-			} else if org.Login != nil {
-				name = *org.Login
-			} else {
-				continue
-			}
-			s.log.Debugf("adding organization %d for repository enumeration: %s", org.ID, name)
-			common.AddStringSliceItem(name, &s.orgs)
-		}
-	}
-}
-
 func (s *Source) addOrgsByUser(ctx context.Context, apiClient *github.Client, user string) {
 	orgOpts := &github.ListOptions{
 		PerPage: 100,

pkg/sources/s3/s3.go

@@ -248,9 +248,9 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				errorCount.Store(prefix, nErr)
 				//too many consective errors on this page
 				if nErr.(int) > 3 {
-					s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)
+					s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
 				}
-				log.Debugf("Error Counts: %s:%d", prefix, nErr)
+				log.Debugf("Error Counts: %s:%s", prefix, nErr)
 				return
 			}
 			body, err := ioutil.ReadAll(res.Body)
@@ -269,7 +269,7 @@ func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan
 				errorCount.Store(prefix, nErr)
 
 				if nErr.(int) > 3 {
-					s.log.Warnf("Too many consecutive errors. Excluding %s", prefix)
+					s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
 				}
 				return
 			}