fixup - remove change in behavior, cannot yet confirm that results have unique ids

pkg/detectors/aws/aws.go

@@ -26,8 +26,8 @@ var (
 
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
-	keyPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)
-	idPat  = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
+	idPat     = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
+	secretPat = regexp.MustCompile(`\b([A-Za-z0-9+/]{40})\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
@@ -58,25 +58,25 @@ func GetHMAC(key []byte, data []byte) []byte {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
+	secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
 
-	for _, match := range matches {
-		if len(match) != 2 {
+	for _, idMatch := range idMatches {
+		if len(idMatch) != 2 {
 			continue
 		}
-		resMatch := strings.TrimSpace(match[1])
+		resIDMatch := strings.TrimSpace(idMatch[1])
 
-		for _, idMatch := range idMatches {
-			if len(idMatch) != 2 {
+		for _, secretMatch := range secretMatches {
+			if len(secretMatch) != 2 {
 				continue
 			}
-
-			resIdMatch := strings.TrimSpace(idMatch[1])
+			resSecretMatch := strings.TrimSpace(secretMatch[1])
 
 			s1 := detectors.Result{
 				DetectorType: detectorspb.DetectorType_AWS,
-				Raw:          []byte(resMatch),
+				Raw:          []byte(resIDMatch),
+				Redacted:     resIDMatch,
 			}
 
 			if verify {
@@ -106,7 +106,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				params.Add("Action", "GetCallerIdentity")
 				params.Add("Version", "2011-06-15")
 				params.Add("X-Amz-Algorithm", algorithm)
-				params.Add("X-Amz-Credential", resIdMatch+"/"+credentialScope)
+				params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
 				params.Add("X-Amz-Date", amzDate)
 				params.Add("X-Amz-Expires", "30")
 				params.Add("X-Amz-SignedHeaders", signedHeaders)
@@ -120,7 +120,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				// TASK 3: CALCULATE THE SIGNATURE.
 				// https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
-				hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resMatch)), []byte(datestamp))
+				hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
 				hash = GetHMAC(hash, []byte(region))
 				hash = GetHMAC(hash, []byte(service))
 				hash = GetHMAC(hash, []byte("aws4_request"))
@@ -135,12 +135,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 
 				res, err := client.Do(req)
 				if err == nil {
-					defer res.Body.Close()
+					// Response body is unused so close it immediately.
+					res.Body.Close()
 					if res.StatusCode >= 200 && res.StatusCode < 300 {
 						s1.Verified = true
 					} else {
 						// This function will check false positives for common test words, but also it will make sure the key appears "random" enough to be a real key.
-						if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+						if detectors.IsKnownFalsePositive(resSecretMatch, detectors.DefaultFalsePositives, true) {
 							continue
 						}
 					}

pkg/detectors/aws/aws_test.go

@@ -48,6 +48,7 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     true,
+					Redacted:     "AKIAWARWQKZNHMZBLY4I",
 				},
 			},
 			wantErr: false,
@@ -64,6 +65,7 @@ func TestAWS_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_AWS,
 					Verified:     false,
+					Redacted:     "AKIAWARWQKZNHMZBLY4I",
 				},
 			},
 			wantErr: false,

pkg/sources/github/github.go

@@ -105,7 +105,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID, sourceID int64,
 	s.jobID = jobID
 	s.verify = verify
 	s.jobSem = semaphore.NewWeighted(int64(concurrency))
-	s.resumeInfoMutex = &sync.Mutex{}
 
 	s.httpClient = common.SaneHttpClient()
 
@@ -173,10 +172,12 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
 	var err error
 	// If we're using public github, make a regular client.
 	// Otherwise make an enterprise client
+	var isGHE bool
 	var apiClient *github.Client
 	if apiEndpoint == "https://api.github.com" {
 		apiClient = github.NewClient(tc)
 	} else {
+		isGHE = true
 		apiClient, err = github.NewEnterpriseClient(apiEndpoint, apiEndpoint, tc)
 		if err != nil {
 			return nil, errors.New(err)
@@ -212,9 +213,15 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri
 		if err := s.addReposByUser(ctx, apiClient, user.GetLogin()); err != nil {
 			log.WithError(err).Error("error fetching repos by user")
 		}
-		// Scan for orgs is default with a token. GitHub App enumerates the repositories
-		// that were assigned to it in GitHub App settings.
-		s.addOrgsByUser(ctx, apiClient, user.GetLogin())
+
+		if isGHE {
+			s.addAllVisibleOrgs(ctx, apiClient)
+		} else {
+			// Scan for orgs is default with a token. GitHub App enumerates the repositories
+			// that were assigned to it in GitHub App settings.
+			s.addOrgsByUser(ctx, apiClient, user.GetLogin())
+		}
+
 		for _, org := range s.orgs {
 			if err := s.addReposByOrg(ctx, apiClient, org); err != nil {
 				log.WithError(err).Error("error fetching repos by org")
@@ -457,6 +464,8 @@ func handleRateLimit(errIn error, res *github.Response) bool {
 }
 
 func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, org string) ([]string, error) {
+	log := s.log.WithField("org", org)
+
 	repos := []string{}
 	opts := &github.RepositoryListByOrgOptions{
 		ListOptions: github.ListOptions{
@@ -475,6 +484,7 @@ func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, or
 		if err != nil {
 			return nil, fmt.Errorf("could not list repos for org %s: %w", org, err)
 		}
+		log.Debugf("listed repos page %d/%d", opts.Page, res.LastPage)
 		if len(someRepos) == 0 {
 			break
 		}
@@ -493,7 +503,7 @@ func (s *Source) getReposByOrg(ctx context.Context, apiClient *github.Client, or
 		}
 		opts.Page = res.NextPage
 	}
-	log.WithField("org", org).Debugf("Found %d repos (%d forks)", numRepos, numForks)
+	log.Debugf("found %d repos (%d forks)", numRepos, numForks)
 	return repos, nil
 }
 
@@ -666,6 +676,45 @@ func (s *Source) addReposByApp(ctx context.Context, apiClient *github.Client) er
 	return nil
 }
 
+func (s *Source) addAllVisibleOrgs(ctx context.Context, apiClient *github.Client) {
+	s.log.Debug("enumerating all visibile organizations on GHE")
+	orgOpts := &github.OrganizationsListOptions{
+		ListOptions: github.ListOptions{
+			PerPage: 100,
+		},
+	}
+	for {
+		orgs, resp, err := apiClient.Organizations.ListAll(ctx, orgOpts)
+		if err == nil {
+			defer resp.Body.Close()
+		}
+		if handled := handleRateLimit(err, resp); handled {
+			continue
+		}
+		if err != nil {
+			log.WithError(err).Errorf("Could not list all organizations")
+			return
+		}
+		s.log.Debugf("listed organization page %d/%d", orgOpts.Page, resp.LastPage)
+		for _, org := range orgs {
+			var name string
+			if org.Name != nil {
+				name = *org.Name
+			} else if org.Login != nil {
+				name = *org.Login
+			} else {
+				continue
+			}
+			s.log.Debug("adding organization for repository enumeration: ", name)
+			common.AddStringSliceItem(name, &s.orgs)
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		orgOpts.Page = resp.NextPage
+	}
+}
+
 func (s *Source) addOrgsByUser(ctx context.Context, apiClient *github.Client, user string) {
 	orgOpts := &github.ListOptions{
 		PerPage: 100,