actions/trufflehog
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add s3 support to CLI (#76)

Browse Source 
* Add s3 support to CLI

* Clean up comments

Co-authored-by: Dustin Decker <dustin@trufflesec.com>
This commit is contained in:
Bill Rich
2022-03-14 16:07:07 -08:00
committed by GitHub
parent 2cb5b27d3b
commit 6486c18565
6 changed files with 493 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1
View File
@@ -6,6 +6,7 @@ require (
cloud.google.com/go/secretmanager v1.3.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1465
github.com/aws/aws-sdk-go v1.43.18
github.com/aws/aws-sdk-go-v2/credentials v1.10.0
github.com/aws/aws-sdk-go-v2/service/sts v1.16.0
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
go.sum
+2
View File
@@ -93,6 +93,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
github.com/aws/aws-sdk-go v1.43.18 h1:nwLaIz2m1f7YBEMNyEc6bBB276AIEaGaIQrc2G9h4zY=
github.com/aws/aws-sdk-go v1.43.18/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v1.15.0 h1:f9kWLNfyCzCB43eupDAk3/XgJ2EpgktiySD6leqs0js=
github.com/aws/aws-sdk-go-v2 v1.15.0/go.mod h1:lJYcuZZEHWNIb6ugJjbQY1fykdoobWbOS7kJYb4APoI=
github.com/aws/aws-sdk-go-v2/credentials v1.10.0 h1:M/FFpf2w31F7xqJqJLgiM0mFpLOtBvwZggORr6QCpo8=
main.go
+9 -3
View File
@@ -67,8 +67,11 @@ func main() {
// filesystemScanIncludePaths := filesystemScan.Flag("include_paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
// filesystemScanExcludePaths := filesystemScan.Flag("exclude_paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
s3Scan := cli.Command("s3", "Coming soon. Find credentials in an S3 bucket.")
s3Scan := cli.Command("s3", "Find credentials in S3 buckets.")
s3ScanKey := s3Scan.Flag("key", "S3 key used to authenticate.").String()
s3ScanSecret := s3Scan.Flag("secret", "S3 secret used to authenticate.").String()
s3ScanCloudEnv := s3Scan.Flag("cloud-environment", "Use IAM credentials in cloud environment.").Bool()
s3ScanBuckets := s3Scan.Flag("bucket", "Name of S3 bucket to scan. You can repeat this flag.").Strings()
cmd := kingpin.MustParse(cli.Parse(os.Args[1:]))
// When setting a base commit, chunks must be scanned in order.
@@ -143,7 +146,10 @@ func main() {
logrus.WithError(err).Fatal("Failed to scan filesystem")
}
case s3Scan.FullCommand():
log.Fatal("S3 not implemented. Coming soon.")
err := e.ScanS3(ctx, *s3ScanKey, *s3ScanSecret, *s3ScanCloudEnv, *s3ScanBuckets)
if err != nil {
logrus.WithError(err).Fatal("Failed to scan S3.")
}
}
if !*jsonLegacy && !*jsonOut {
pkg/engine/s3.go
+57
View File
@@ -0,0 +1,57 @@
package engine
import (
"context"
"fmt"
"github.com/go-errors/errors"
"github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/s3"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
"runtime"
)
func (e *Engine) ScanS3(ctx context.Context, key, secret string, cloudCred bool, buckets []string) error {
connection := &sourcespb.S3{
Credential: &sourcespb.S3_Unauthenticated{},
}
if cloudCred {
if len(key) > 0 || len(secret) > 0 {
return fmt.Errorf("cannot use cloud credentials and basic auth together")
}
connection.Credential = &sourcespb.S3_CloudEnvironment{}
}
if len(key) > 0 && len(secret) > 0 {
connection.Credential = &sourcespb.S3_AccessKey{
AccessKey: &credentialspb.KeySecret{
Key: key,
Secret: secret,
},
}
}
if len(buckets) > 0 {
connection.Buckets = buckets
}
var conn anypb.Any
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
if err != nil {
logrus.WithError(err).Error("failed to marshal github connection")
return err
}
s3Source := s3.Source{}
err = s3Source.Init(ctx, "trufflehog - s3", 0, int64(sourcespb.SourceType_SOURCE_TYPE_S3), true, &conn, runtime.NumCPU())
if err != nil {
return errors.WrapPrefix(err, "failed to init S3 source", 0)
}
go func() {
err := s3Source.Chunks(ctx, e.ChunksChan())
if err != nil {
logrus.WithError(err).Error("error scanning s3")
}
close(e.ChunksChan())
}()
return nil
}
pkg/sources/s3/s3.go
+327
View File
@@ -0,0 +1,327 @@
package s3
import (
"context"
"fmt"
"io/ioutil"
"strings"
"sync"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/go-errors/errors"
log "github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"golang.org/x/sync/semaphore"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
)
type Source struct {
name string
sourceId int64
jobId int64
verify bool
concurrency int
aCtx context.Context
log *log.Entry
sources.Progress
errorCount *sync.Map
conn *sourcespb.S3
}
// Ensure the Source satisfies the interface at compile time
var _ sources.Source = (*Source)(nil)
// Type returns the type of source
func (s *Source) Type() sourcespb.SourceType {
return sourcespb.SourceType_SOURCE_TYPE_S3
}
func (s *Source) SourceID() int64 {
return s.sourceId
}
func (s *Source) JobID() int64 {
return s.jobId
}
// Init returns an initialized AWS source
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error {
s.log = log.WithField("source", s.Type()).WithField("name", name)
s.aCtx = aCtx
s.name = name
s.sourceId = sourceId
s.jobId = jobId
s.verify = verify
s.concurrency = concurrency
s.errorCount = &sync.Map{}
var conn sourcespb.S3
err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
if err != nil {
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
}
s.conn = &conn
return nil
}
func (s *Source) newClient(region string) (*s3.S3, error) {
cfg := aws.NewConfig()
cfg.CredentialsChainVerboseErrors = aws.Bool(true)
cfg.Region = aws.String(region)
switch cred := s.conn.GetCredential().(type) {
case *sourcespb.S3_AccessKey:
cfg.Credentials = credentials.NewStaticCredentials(cred.AccessKey.Key, cred.AccessKey.Secret, "")
case *sourcespb.S3_Unauthenticated:
cfg.Credentials = credentials.AnonymousCredentials
case *sourcespb.S3_CloudEnvironment:
// Nothing needs to be done!
default:
return nil, errors.Errorf("invalid configuration given for %s source", s.name)
}
sess, err := session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
Config: *cfg,
})
if err != nil {
return nil, err
}
return s3.New(sess), nil
}
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
client, err := s.newClient("us-east-1")
if err != nil {
return errors.WrapPrefix(err, "could not create s3 client", 0)
}
bucketsToScan := []string{}
switch s.conn.GetCredential().(type) {
case *sourcespb.S3_AccessKey, *sourcespb.S3_CloudEnvironment:
if len(s.conn.Buckets) == 0 {
res, err := client.ListBuckets(&s3.ListBucketsInput{})
if err != nil {
s.log.Errorf("could not list s3 buckets: %s", err)
return errors.WrapPrefix(err, "could not list s3 buckets", 0)
}
buckets := res.Buckets
for _, bucket := range buckets {
bucketsToScan = append(bucketsToScan, *bucket.Name)
}
} else {
bucketsToScan = s.conn.Buckets
}
case *sourcespb.S3_Unauthenticated:
bucketsToScan = s.conn.Buckets
default:
return errors.Errorf("invalid configuration given for %s source", s.name)
}
for i, bucket := range bucketsToScan {
if common.IsDone(ctx) {
return nil
}
s.SetProgressComplete(i, len(bucketsToScan), fmt.Sprintf("Bucket: %s", bucket))
s.log.Debugf("Scanning bucket: %s", bucket)
region, err := s3manager.GetBucketRegionWithClient(context.Background(), client, bucket)
if err != nil {
s.log.WithError(err).Errorf("could not get s3 region for bucket: %s", bucket)
continue
}
var regionalClient *s3.S3
if region != "us-east-1" {
regionalClient, err = s.newClient(region)
if err != nil {
s.log.WithError(err).Error("could not make regional s3 client")
}
} else {
regionalClient = client
}
//Forced prefix for testing
//pf := "public"
errorCount := sync.Map{}
err = regionalClient.ListObjectsV2PagesWithContext(
ctx, &s3.ListObjectsV2Input{Bucket: &bucket},
func(page *s3.ListObjectsV2Output, last bool) bool {
s.pageChunker(ctx, regionalClient, chunksChan, bucket, page, &errorCount)
return true
})
if err != nil {
s.log.WithError(err).Errorf("could not list objects in s3 bucket: %s", bucket)
return errors.WrapPrefix(err, fmt.Sprintf("could not list objects in s3 bucket: %s", bucket), 0)
}
}
return nil
}
// pageChunker emits chunks onto the given channel from a page
func (s *Source) pageChunker(ctx context.Context, client *s3.S3, chunksChan chan *sources.Chunk, bucket string, page *s3.ListObjectsV2Output, errorCount *sync.Map) {
sem := semaphore.NewWeighted(int64(s.concurrency))
var wg sync.WaitGroup
for _, obj := range page.Contents {
if common.IsDone(ctx) {
return
}
err := sem.Acquire(ctx, 1)
if err != nil {
log.WithError(err).Error("could not acquire semaphore")
continue
}
wg.Add(1)
go func(ctx context.Context, wg *sync.WaitGroup, sem *semaphore.Weighted, obj *s3.Object) {
defer sem.Release(1)
defer wg.Done()
//defer log.Debugf("DONE - %s", *obj.Key)
if (*obj.Key)[len(*obj.Key)-1:] == "/" {
return
}
//log.Debugf("Object: %s", *obj.Key)
path := strings.Split(*obj.Key, "/")
prefix := strings.Join(path[:len(path)-1], "/")
nErr, ok := errorCount.Load(prefix)
if !ok {
nErr = 0
}
if nErr.(int) > 3 {
log.Debugf("Skipped: %s", *obj.Key)
return
}
// ignore large files
if *obj.Size > int64(10*common.MB) {
return
}
//file is 0 bytes - likely no permissions - skipping
if *obj.Size == 0 {
return
}
//files break with spaces, must replace with +
//objKey := strings.ReplaceAll(*obj.Key, " ", "+")
ctx, cancel := context.WithTimeout(ctx, time.Second*5)
defer cancel()
res, err := client.GetObjectWithContext(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: obj.Key,
})
if err != nil {
if !strings.Contains(err.Error(), "AccessDenied") {
s.log.WithError(err).Errorf("could not get S3 object: %s", *obj.Key)
}
nErr, ok := errorCount.Load(prefix)
if !ok {
nErr = 0
}
if nErr.(int) > 3 {
log.Debugf("Skipped: %s", *obj.Key)
return
}
nErr = nErr.(int) + 1
errorCount.Store(prefix, nErr)
//too many consective errors on this page
if nErr.(int) > 3 {
s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
}
log.Debugf("Error Counts: %s:%s", prefix, nErr)
return
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
s.log.WithError(err).Error("could not read S3 object body")
nErr, ok := errorCount.Load(prefix)
if !ok {
nErr = 0
}
//too many consective errors on this page
if nErr.(int) > 3 {
log.Debugf("Skipped: %s", *obj.Key)
return
}
nErr = nErr.(int) + 1
errorCount.Store(prefix, nErr)
if nErr.(int) > 3 {
s.log.Warnf("Too many consecutive errors. Blacklisting %s", prefix)
}
return
}
// ignore files that don't have secrets
if common.SkipFile(*obj.Key, body) {
return
}
email := "Unknown"
if obj.Owner != nil {
email = *obj.Owner.DisplayName
}
modified := obj.LastModified.String()
chunk := sources.Chunk{
SourceType: s.Type(),
SourceName: s.name,
SourceID: s.SourceID(),
Data: body,
SourceMetadata: &source_metadatapb.MetaData{
Data: &source_metadatapb.MetaData_S3{
S3: &source_metadatapb.S3{
Bucket: bucket,
File: sanitizer.UTF8(*obj.Key),
Link: sanitizer.UTF8(makeS3Link(bucket, *client.Config.Region, *obj.Key)),
Email: sanitizer.UTF8(email),
Timestamp: sanitizer.UTF8(modified),
},
},
},
Verify: s.verify,
}
nErr, ok = errorCount.Load(prefix)
if !ok {
nErr = 0
}
if nErr.(int) > 0 {
errorCount.Store(prefix, 0)
}
chunksChan <- &chunk
}(ctx, &wg, sem, obj)
}
wg.Wait()
}
// S3 links currently have the general format of:
// https://[bucket].s3[.region unless us-east-1].amazonaws.com/[key]
func makeS3Link(bucket, region, key string) string {
if region == "us-east-1" {
region = ""
} else {
region = "." + region
}
return fmt.Sprintf("https://%s.s3%s.amazonaws.com/%s", bucket, region, key)
}
pkg/sources/s3/s3_test.go
+97
View File
@@ -0,0 +1,97 @@
package s3
import (
"context"
"encoding/base64"
"fmt"
"os"
"sync"
"testing"
"time"
"github.com/kylelemons/godebug/pretty"
log "github.com/sirupsen/logrus"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"google.golang.org/protobuf/types/known/anypb"
)
func TestSource_Chunks(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
secret, err := common.GetTestSecret(ctx)
if err != nil {
t.Fatal(fmt.Errorf("failed to access secret: %v", err))
}
s3key := secret.MustGetField("AWS_S3_KEY")
s3secret := secret.MustGetField("AWS_S3_SECRET")
type init struct {
name string
verify bool
connection *sourcespb.S3
}
tests := []struct {
name string
init init
wantErr bool
wantChunkData string
}{
{
name: "gets chunks",
init: init{
connection: &sourcespb.S3{
Credential: &sourcespb.S3_AccessKey{
AccessKey: &credentialspb.KeySecret{
Key: s3key,
Secret: s3secret,
},
},
Buckets: []string{"thog-tmp-test"},
},
},
wantErr: false,
wantChunkData: `W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkID0gQUtJQTM1T0hYMkRTT1pHNjQ3TkgKYXdzX3NlY3JldF9hY2Nlc3Nfa2V5ID0gUXk5OVMrWkIvQ1dsRk50eFBBaWQ3Z0d6dnNyWGhCQjd1ckFDQUxwWgpvdXRwdXQgPSBqc29uCnJlZ2lvbiA9IHVzLWVhc3QtMg==`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
log.SetFormatter(&log.TextFormatter{ForceColors: true})
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
var cancelOnce sync.Once
defer cancelOnce.Do(cancel)
s := Source{}
log.SetLevel(log.DebugLevel)
conn, err := anypb.New(tt.init.connection)
if err != nil {
t.Fatal(err)
}
err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 10)
if (err != nil) != tt.wantErr {
t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr)
return
}
chunksCh := make(chan *sources.Chunk, 100)
go func() {
err = s.Chunks(ctx, chunksCh)
if (err != nil) != tt.wantErr {
t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr)
os.Exit(1)
}
}()
gotChunk := <-chunksCh
wantData, _ := base64.StdEncoding.DecodeString(tt.wantChunkData)
if diff := pretty.Compare(gotChunk.Data, wantData); diff != "" {
t.Errorf("%s: Source.Chunks() diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}