[chore] - Add regex and keyword for api_org tokens (#2240)

* Add regex and keyword for api_org tokens.

* handle org token auth struct

* update keywords

pkg/detectors/huggingface/huggingface.go

@@ -23,13 +23,13 @@ var _ detectors.Detector = (*Scanner)(nil)
 var (
 	defaultClient = common.SaneHttpClient()
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
-	keyPat = regexp.MustCompile(`\bhf_[a-zA-Z0-9]{34}\b`)
+	keyPat = regexp.MustCompile(`\b(?:hf_|api_org_)[a-zA-Z0-9]{34}\b`)
 )
 
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface"
+	return []string{"hf_", "api_org_"} // Huggingface docs occasionally use "hf" instead of "huggingface"
 }
 
 // FromData will find and optionally verify Huggingface secrets in a given set of bytes.
@@ -92,15 +92,29 @@ func (s Scanner) verifyResult(ctx context.Context, apiKey string) (bool, map[str
 			return true, nil, err
 		}
 
-		t := whoamiRes.Auth.AccessToken
+		var tokenInfo string
+		switch {
+		case whoamiRes.Auth.AccessToken.DisplayName != "" || whoamiRes.Auth.AccessToken.Role != "":
+			// hf_xxxx token
+			t := whoamiRes.Auth.AccessToken
+			tokenInfo = fmt.Sprintf("%s (%s)", t.DisplayName, t.Role)
+
+		case whoamiRes.Auth.Type != "":
+			// api_org_xxxx token
+			tokenInfo = whoamiRes.Auth.Type
+
+		default:
+			tokenInfo = "Unknown Token Type"
+		}
+
 		extraData := map[string]string{
 			"Username": whoamiRes.Name,
 			"Email":    whoamiRes.Email,
-			"Token":    fmt.Sprintf("%s (%s)", t.DisplayName, t.Role),
+			"Token":    tokenInfo,
 		}
 
 		// Condense a list of organizations + roles.
-		var orgs []string
+		orgs := make([]string, 0, len(whoamiRes.Organizations))
 		for _, org := range whoamiRes.Organizations {
 			orgs = append(orgs, fmt.Sprintf("%s:%s", org.Name, org.Role))
 		}
@@ -136,7 +150,8 @@ type organization struct {
 
 type auth struct {
 	AccessToken struct {
-		DisplayName string `json:"displayName"`
-		Role        string `json:"role"`
-	} `json:"accessToken"`
+		DisplayName string `json:"displayName,omitempty"`
+		Role        string `json:"role,omitempty"`
+	} `json:"accessToken,omitempty"`
+	Type string `json:"type,omitempty"`
 }

pkg/sources/git/git.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"sync/atomic"
@@ -451,11 +452,18 @@ func CloneRepoUsingUnauthenticated(ctx context.Context, url string, args ...stri
 }
 
 // CloneRepoUsingSSH clones a repo using SSH.
-func CloneRepoUsingSSH(ctx context.Context, gitUrl string, args ...string) (string, *git.Repository, error) {
+func CloneRepoUsingSSH(ctx context.Context, gitURL string, args ...string) (string, *git.Repository, error) {
+	if isCodeCommitURL(gitURL) {
+		return CloneRepo(ctx, nil, gitURL, args...)
+	}
 	userInfo := url.User("git")
-	return CloneRepo(ctx, userInfo, gitUrl, args...)
+	return CloneRepo(ctx, userInfo, gitURL, args...)
 }
 
+var codeCommitRE = regexp.MustCompile(`ssh://git-codecommit\.[\w-]+\.amazonaws\.com`)
+
+func isCodeCommitURL(gitURL string) bool { return codeCommitRE.MatchString(gitURL) }
+
 func (s *Git) CommitsScanned() uint64 {
 	return atomic.LoadUint64(&s.metrics.commitsScanned)
 }

pkg/sources/github/github.go

@@ -357,6 +357,10 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
 		s.mu.Unlock()
 	}()
 	logger := s.log.WithValues("repo", repoURL)
+	if _, unauthenticated := s.conn.GetCredential().(*sourcespb.GitHub_Unauthenticated); unauthenticated {
+		logger.V(3).Info("assuming unauthenticated scan has public visibility")
+		return source_metadatapb.Visibility_public
+	}
 	logger.V(2).Info("Checking public status")
 	u, err := url.Parse(repoURL)
 	if err != nil {
@@ -379,10 +383,6 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
 			}
 		}
 		if err != nil || gist == nil {
-			if _, unauthenticated := s.conn.GetCredential().(*sourcespb.GitHub_Unauthenticated); unauthenticated {
-				logger.Info("Unauthenticated scans cannot determine if a repository is private.")
-				visibility = source_metadatapb.Visibility_private
-			}
 			logger.Error(err, "Could not get Github repository")
 			return
 		}
@@ -402,10 +402,6 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s
 		}
 		if err != nil || repo == nil {
 			logger.Error(err, "Could not get Github repository")
-			if _, unauthenticated := s.conn.GetCredential().(*sourcespb.GitHub_Unauthenticated); unauthenticated {
-				logger.Info("Unauthenticated scans cannot determine if a repository is private.")
-				visibility = source_metadatapb.Visibility_private
-			}
 			return
 		}
 		if *repo.Private {

proto/detectors.proto

@@ -977,6 +977,17 @@ enum DetectorType {
   Ngrok = 966;
   Replicate = 967;
   Postgres = 968;
+  AzureActiveDirectoryApplicationSecret = 969;
+  AzureCacheForRedisAccessKey = 970;
+  AzureCosmosDBKeyIdentifiable = 971;
+  AzureDevopsPersonalAccessToken = 972;
+  AzureFunctionKey = 973;
+  AzureMLWebServiceClassicIdentifiableKey = 974;
+  AzureSasToken = 975;
+  AzureSearchAdminKey = 976;
+  AzureSearchQueryKey = 977;
+  AzureManagementCertificate = 978;
+  AzureSQL = 979;
 }
 
 message Result {