Merge branch 'main' into docker-tar-identification-from-filesystem-2

pkg/handlers/archive.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	"github.com/h2non/filetype"
 	"github.com/mholt/archiver/v4"
 
@@ -36,12 +38,33 @@ var (
 // Ensure the Archive satisfies the interfaces at compile time.
 var _ SpecializedHandler = (*Archive)(nil)
 
+// tempEnv contains the temporary file and directory information for the archive.
+type tempEnv struct {
+	tempFile     *os.File
+	tempFileName string
+	extractPath  string
+}
+
+// dockerImageDetails contains the details of the Docker image and the temporary environment.
+type dockerImageDetails struct {
+	image v1.Image
+	temp  tempEnv
+}
+
 // Archive is a handler for extracting and decompressing archives.
 type Archive struct {
 	size         int
 	currentDepth int
 }
 
+// DockerTarReader is a wrapper for io.Reader that also stores the temp filepath and image of the tar file.
+// Temporary file path information needed for inner-workings of go-containerregistry Image methods.
+type DockerTarReader struct {
+	io.Reader
+	img    v1.Image
+	tmpEnv tempEnv
+}
+
 // New sets a default maximum size and current size counter.
 func (a *Archive) New() {
 	a.size = 0
@@ -213,6 +236,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
 const (
 	arMimeType  = "application/x-unix-archive"
 	rpmMimeType = "application/x-rpm"
+	tarMimeType = "application/x-tar"
 )
 
 // Define a map of mime types to corresponding command-line tools
@@ -266,6 +290,21 @@ func (a *Archive) HandleSpecialized(ctx logContext.Context, reader io.Reader) (i
 			return nil, false, err
 		}
 		reader, err = a.extractRpmContent(ctx, reader)
+	case tarMimeType:
+		//check if tar is a docker image
+		isImg, details, err := a.isDockerImage(ctx, reader)
+		if err != nil {
+			return nil, false, err
+		}
+		if !isImg {
+			return nil, false, nil
+		}
+		// Build dockerReader containing docker image
+		reader = DockerTarReader{
+			Reader: reader,
+			img:    details.image,
+			tmpEnv: details.temp,
+		}
 	default:
 		return reader, false, nil
 	}
@@ -290,8 +329,8 @@ func (a *Archive) extractDebContent(ctx logContext.Context, file io.Reader) (io.
 	if err != nil {
 		return nil, err
 	}
-	defer os.Remove(tmpEnv.tempFileName)
 	defer os.RemoveAll(tmpEnv.extractPath)
+	defer os.Remove(tmpEnv.tempFileName)
 
 	cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
 	cmd.Dir = tmpEnv.extractPath
@@ -328,8 +367,8 @@ func (a *Archive) extractRpmContent(ctx logContext.Context, file io.Reader) (io.
 	if err != nil {
 		return nil, err
 	}
-	defer os.Remove(tmpEnv.tempFileName)
 	defer os.RemoveAll(tmpEnv.extractPath)
+	defer os.Remove(tmpEnv.tempFileName)
 
 	// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
 	cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
@@ -428,12 +467,6 @@ func (a *Archive) handleExtractedFiles(ctx logContext.Context, env tempEnv, hand
 	return dataArchiveName, nil
 }
 
-type tempEnv struct {
-	tempFile     *os.File
-	tempFileName string
-	extractPath  string
-}
-
 // createTempEnv creates a temporary file and a temporary directory for extracting archives.
 // The caller is responsible for removing these temporary resources
 // (both the file and directory) when they are no longer needed.
@@ -477,3 +510,22 @@ func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser,
 	}
 	return dataFile, nil
 }
+
+// isDockerImage checks if a reader object is a docker image.
+// Returns true if the tar file is a docker image, dockerImageDetails, and error.
+// Caller is responsible for removing temporary files and directories IF the file is a docker image.
+// The docker scanner requires a file path to the image, so we must leave the temporary file on disk.
+func (a *Archive) isDockerImage(ctx context.Context, file io.Reader) (isImg bool, details dockerImageDetails, err error) {
+	tmpEnv, err := a.createTempEnv(ctx, file)
+	if err != nil {
+		return false, dockerImageDetails{}, err
+	}
+
+	img, err := tarball.ImageFromPath(tmpEnv.tempFileName, nil)
+	if err != nil {
+		os.Remove(tmpEnv.tempFileName)
+		os.RemoveAll(tmpEnv.extractPath)
+		return false, dockerImageDetails{}, err
+	}
+	return true, dockerImageDetails{image: img, temp: tmpEnv}, nil
+}

pkg/handlers/archive_integration_test.go

@@ -0,0 +1,102 @@
+//go:build integration
+// +build integration
+
+package handlers
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/stretchr/testify/assert"
+
+	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+)
+
+func TestFoundKeyInDockerTar(t *testing.T) {
+	// URI of the Docker image to clone
+	// imageURI := "ghcr.io/joeleonjr/getting-started-app-with-canary-token:main"
+	imageURI := "trufflesecurity/secrets"
+
+	var imageName name.Reference
+	imageName, err := name.NewTag(imageURI)
+	if err != nil {
+		t.Errorf("%s: error creating image name: %s", imageURI, err)
+	}
+
+	// Pull the image
+	img, err := remote.Image(imageName)
+	if err != nil {
+		t.Errorf("%s: error pulling image: %s", imageURI, err)
+	}
+
+	tempFile, err := os.CreateTemp("", "archive_test_docker_img.tar")
+	if err != nil {
+		t.Errorf("%s: error creating temporary file: %s", imageURI, err)
+		return
+	}
+
+	defer os.Remove(tempFile.Name()) // Clean up the temporary file
+
+	// Save the image as a tar file
+	err = tarball.WriteToFile(tempFile.Name(), imageName, img)
+	if err != nil {
+		t.Errorf("%s: error saving image as tar file: %s", imageURI, err)
+		return
+	}
+
+	ctx := logContext.AddLogger(context.Background())
+
+	inputFile, err := os.Open(tempFile.Name())
+	if err != nil {
+		t.Errorf("%s: error opening tar file: %s", imageURI, err)
+		return
+	}
+	defer inputFile.Close()
+
+	reReader, err := diskbufferreader.New(inputFile)
+	if err != nil {
+		t.Errorf("%s: error creating re-readable reader: %s", imageURI, err)
+		return
+	}
+	defer reReader.Close()
+
+	chunkSkel := &sources.Chunk{
+		SourceType: 1,
+		SourceName: "filesystem",
+		SourceID:   1,
+		SourceMetadata: &source_metadatapb.MetaData{
+			Data: &source_metadatapb.MetaData_Filesystem{
+				Filesystem: &source_metadatapb.Filesystem{
+					File: sanitizer.UTF8(tempFile.Name()),
+				},
+			},
+		},
+		Verify: true,
+	}
+
+	chunksChan := make(chan *sources.Chunk, 1)
+
+	go func() {
+		defer close(chunksChan)
+		HandleFile(ctx, reReader, chunkSkel, chunksChan)
+	}()
+
+	secret := "AKIAXYZDQCEN4B6JSJQI"
+
+	// Read from the channel and validate the secrets.
+	foundSecret := ""
+	for chunkCh := range chunksChan {
+		foundSecret += string(chunkCh.Data)
+	}
+
+	assert.Contains(t, foundSecret, secret)
+
+}

pkg/handlers/archive_test.go

@@ -108,7 +108,8 @@ func TestHandleFile(t *testing.T) {
 	ch := make(chan *sources.Chunk, 2)
 
 	// Context cancels the operation.
-	canceledCtx, cancel := context.WithCancel(context.Background())
+	ctx := logContext.AddLogger(context.Background())
+	canceledCtx, cancel := logContext.WithCancel(ctx)
 	cancel()
 	assert.False(t, HandleFile(canceledCtx, strings.NewReader("file"), &sources.Chunk{}, ch))
 
@@ -123,7 +124,7 @@ func TestHandleFile(t *testing.T) {
 	assert.NoError(t, err)
 
 	assert.Equal(t, 0, len(ch))
-	assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
+	assert.True(t, HandleFile(ctx, reader, &sources.Chunk{}, ch))
 	assert.Equal(t, 1, len(ch))
 }
 
@@ -202,7 +203,7 @@ func TestExtractTarContent(t *testing.T) {
 	assert.Nil(t, err)
 	defer file.Close()
 
-	ctx := context.Background()
+	ctx := logContext.AddLogger(context.Background())
 
 	chunkCh := make(chan *sources.Chunk)
 	go func() {

pkg/handlers/handlers.go

@@ -3,11 +3,13 @@ package handlers
 import (
 	"context"
 	"io"
+	"os"
 
 	diskbufferreader "github.com/bill-rich/disk-buffer-reader"
 
 	logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
 )
 
 func DefaultHandlers() []Handler {
@@ -37,7 +39,7 @@ type Handler interface {
 // packages them in the provided chunk skeleton, and sends them to chunksChan.
 // The function returns true if processing was successful and false otherwise.
 // Context is used for cancellation, and the caller is responsible for canceling it if needed.
-func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
+func HandleFile(ctx logContext.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
 	aCtx := logContext.AddLogger(ctx)
 	for _, h := range DefaultHandlers() {
 		h.New()
@@ -56,9 +58,15 @@ func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, c
 		if specialHandler, ok := h.(SpecializedHandler); ok {
 			file, isSpecial, err := specialHandler.HandleSpecialized(aCtx, reReader)
 			if isSpecial {
+				if dockerTarReader, ok := file.(DockerTarReader); ok {
+					// Clean up temporary files when done.
+					defer os.RemoveAll(dockerTarReader.tmpEnv.extractPath)
+					defer os.Remove(dockerTarReader.tmpEnv.tempFileName)
+					err = docker.ScanDockerImg(ctx, dockerTarReader.img, chunksChan, chunkSkel)
+					return err == nil
+				}
 				return handleChunks(aCtx, h.FromFile(ctx, file), chunkSkel, chunksChan)
 			}
-
 			if err != nil {
 				aCtx.Logger().Error(err, "error handling file")
 			}

pkg/sources/docker/docker.go

@@ -78,7 +78,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 		return err
 	}
 
-	logger := ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
+	ctx.Logger().WithValues("source_type", s.Type(), "source_name", s.name)
 
 	for _, image := range s.conn.GetImages() {
 		var img v1.Image
@@ -115,96 +115,120 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) err
 			}
 		}
 
-		logger = logger.WithValues("image", base, "tag", tag)
-		logger.V(2).Info("scanning image")
-
-		layers, err := img.Layers()
+		chunkSkel := &sources.Chunk{
+			SourceType: s.Type(),
+			SourceName: s.name,
+			SourceID:   s.SourceID(),
+      JobID:      s.JobID(),
+			SourceMetadata: &source_metadatapb.MetaData{
+				Data: &source_metadatapb.MetaData_Docker{
+					Docker: &source_metadatapb.Docker{
+						Image: base,
+						Tag:   tag,
+					},
+				},
+			},
+			Verify: s.verify,
+		}
+		err = ScanDockerImg(ctx, img, chunksChan, chunkSkel)
 		if err != nil {
 			return err
 		}
 
-		for _, layer := range layers {
-
-			digest, err := layer.Digest()
-			if err != nil {
-				return err
-			}
-
-			logger = logger.WithValues("layer", digest.String())
-			logger.V(2).Info("scanning layer")
-
-			rc, err := layer.Compressed()
-			if err != nil {
-				return err
-			}
-
-			defer rc.Close()
-
-			gzipReader, err := gzip.NewReader(rc)
-			if err != nil {
-				return err
-			}
-
-			defer gzipReader.Close()
-
-			tarReader := tar.NewReader(gzipReader)
-
-			for {
-				header, err := tarReader.Next()
-				if errors.Is(err, io.EOF) {
-					break // End of archive
-				}
-				if err != nil {
-					return err
-				}
-
-				// Skip files larger than FilesizeLimitBytes
-				if header.Size > FilesizeLimitBytes {
-					continue
-				}
-
-				file := bytes.NewBuffer(make([]byte, 0, header.Size))
-
-				_, err = io.Copy(file, tarReader)
-				if err != nil {
-					return err
-				}
-
-				chunk := &sources.Chunk{
-					SourceType: s.Type(),
-					SourceName: s.name,
-					SourceID:   s.SourceID(),
-					JobID:      s.JobID(),
-					Data:       file.Bytes(),
-					SourceMetadata: &source_metadatapb.MetaData{
-						Data: &source_metadatapb.MetaData_Docker{
-							Docker: &source_metadatapb.Docker{
-								File:  "/" + header.Name, // Prepend /, because the file is relative to the root of the image
-								Image: base,
-								Tag:   tag,
-								Layer: digest.String(),
-							},
-						},
-					},
-					Verify: s.verify,
-				}
-
-				select {
-				case chunksChan <- chunk:
-				case <-ctx.Done():
-					return ctx.Err()
-				}
-			}
-
-			dockerLayersScanned.WithLabelValues(s.name).Inc()
-		}
-
-		dockerImagesScanned.WithLabelValues(s.name).Inc()
 	}
 
 	return nil
 }
 
+func ScanDockerImg(ctx context.Context, img v1.Image, chunksChan chan *sources.Chunk, chunkSkel *sources.Chunk) error {
+	base, tag := chunkSkel.BaseAndTagForDockerImg()
+
+	logger := ctx.Logger().WithValues("image", base, "tag", tag)
+	logger.V(2).Info("scanning image")
+
+	layers, err := img.Layers()
+	if err != nil {
+		return err
+	}
+	for _, layer := range layers {
+		digest, err := layer.Digest()
+		if err != nil {
+			return err
+		}
+
+		logger = logger.WithValues("layer", digest.String())
+		logger.V(2).Info("scanning layer")
+
+		rc, err := layer.Compressed()
+		if err != nil {
+			return err
+		}
+
+		defer rc.Close()
+
+		gzipReader, err := gzip.NewReader(rc)
+		if err != nil {
+			return err
+		}
+
+		defer gzipReader.Close()
+
+		tarReader := tar.NewReader(gzipReader)
+
+		for {
+			header, err := tarReader.Next()
+			if errors.Is(err, io.EOF) {
+				break // End of archive
+			}
+			if err != nil {
+				return err
+			}
+
+			// Skip files larger than FilesizeLimitBytes
+			if header.Size > FilesizeLimitBytes {
+				continue
+			}
+
+			file := bytes.NewBuffer(make([]byte, 0, header.Size))
+
+			_, err = io.Copy(file, tarReader)
+			if err != nil {
+				return err
+			}
+
+			chunk := &sources.Chunk{
+				// This must be TYPE_DOCKER or SetResultLineNumber could panic
+				SourceType: sourcespb.SourceType_SOURCE_TYPE_DOCKER,
+				SourceName: chunkSkel.SourceName,
+				SourceID:   chunkSkel.SourceID,
+				Data:       file.Bytes(),
+				SourceMetadata: &source_metadatapb.MetaData{
+					Data: &source_metadatapb.MetaData_Docker{
+						Docker: &source_metadatapb.Docker{
+							File:  "/" + header.Name, // Prepend /, because the file is relative to the root of the image
+							Image: base,
+							Tag:   tag,
+							Layer: digest.String(),
+						},
+					},
+				},
+				Verify: chunkSkel.Verify,
+			}
+
+			select {
+			case chunksChan <- chunk:
+			case <-ctx.Done():
+				return ctx.Err()
+			}
+		}
+
+		dockerLayersScanned.WithLabelValues(chunkSkel.SourceName).Inc()
+	}
+
+	dockerImagesScanned.WithLabelValues(chunkSkel.SourceName).Inc()
+	return nil
+}
+
 func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {
 	regRepoDelimiter := "/"
 	tagDelim := ":"

pkg/sources/sources.go

@@ -30,6 +30,48 @@ type Chunk struct {
 	Verify bool
 }
 
+// BaseAndTagForDockerImg returns a base and tag value for a
+// docker image regardless of the source type. If the source type
+// is not a docker image, the base will be a filename/path from the source metadata.
+// The default is the entire source metadata string. Update for new source types.
+func (c *Chunk) BaseAndTagForDockerImg() (base string, tag string) {
+	switch c.SourceType {
+	case sourcespb.SourceType_SOURCE_TYPE_DOCKER:
+		base = c.SourceMetadata.GetDocker().GetImage()
+		tag = c.SourceMetadata.GetDocker().GetTag()
+	case sourcespb.SourceType_SOURCE_TYPE_FILESYSTEM:
+		base = c.SourceMetadata.GetFilesystem().GetFile()
+	case sourcespb.SourceType_SOURCE_TYPE_S3:
+		bucket := c.SourceMetadata.GetS3().GetBucket()
+		file := c.SourceMetadata.GetS3().GetFile()
+		base = bucket + "/" + file
+	case sourcespb.SourceType_SOURCE_TYPE_GCS:
+		bucket := c.SourceMetadata.GetGcs().GetBucket()
+		file := c.SourceMetadata.GetGcs().GetFilename()
+		base = bucket + "/" + file
+	case sourcespb.SourceType_SOURCE_TYPE_SYSLOG:
+		host := c.SourceMetadata.GetSyslog().GetHostname()
+		app := c.SourceMetadata.GetSyslog().GetAppname()
+		base = host + "/" + app
+	case sourcespb.SourceType_SOURCE_TYPE_GIT:
+		repo := c.SourceMetadata.GetGit().GetRepository()
+		file := c.SourceMetadata.GetGit().GetFile()
+		base = file + " (repo: " + repo + ")"
+	case sourcespb.SourceType_SOURCE_TYPE_GITLAB:
+		repo := c.SourceMetadata.GetGitlab().GetRepository()
+		file := c.SourceMetadata.GetGitlab().GetFile()
+		base = file + " (repo: " + repo + ")"
+	case sourcespb.SourceType_SOURCE_TYPE_GITHUB:
+		repo := c.SourceMetadata.GetGithub().GetRepository()
+		file := c.SourceMetadata.GetGithub().GetFile()
+		base = file + " (repo: " + repo + ")"
+	default:
+		base = c.SourceMetadata.String()
+		tag = ""
+	}
+	return base, tag
+}
+
 // Source defines the interface required to implement a source chunker.
 type Source interface {
 	// Type returns the source type, used for matching against configuration and jobs.