Compare commits

..

2 Commits

Author SHA1 Message Date
Joe Leon 92f1978a85 reworked to apply docker identification in handlers.go 2023-08-16 16:29:08 -04:00
Joe Leon c8f8af8771 poc working with docker scanner invoked when filesystem finds tarball 2023-08-14 16:28:04 -04:00
23 changed files with 134 additions and 730 deletions
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
steps:
- uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- uses: actions/checkout@v3
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
+3 -3
View File
@@ -1,4 +1,4 @@
name: Performance Test
name: Performance Test
on: [pull_request]
@@ -7,13 +7,13 @@ jobs:
# skip if PR is from a fork.
# TODO: this could probabaly be refactored a bit so that it runs on forks
if: ${{ ! github.event.pull_request.head.repo.fork }}
runs-on: ubuntu-latest
steps:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- name: Checkout code
uses: actions/checkout@v3
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v4
with:
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- name: Checkout code
uses: actions/checkout@v3
with:
+1 -1
View File
@@ -11,7 +11,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- name: Checkout code
uses: actions/checkout@v3
- name: Smoke
+2 -2
View File
@@ -18,7 +18,7 @@ jobs:
steps:
- uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- uses: actions/checkout@v3
- name: Run Snifftest
run: make snifftest
run: make snifftest
+3 -3
View File
@@ -21,7 +21,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- name: Checkout code
uses: actions/checkout@v3
- id: 'auth'
@@ -42,7 +42,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- name: Checkout code
uses: actions/checkout@v3
- id: 'auth'
@@ -63,7 +63,7 @@ jobs:
- name: Install Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: '1.20'
- name: Checkout code
uses: actions/checkout@v3
- name: Test
-1
View File
@@ -43,7 +43,6 @@ require (
github.com/google/go-containerregistry v0.15.2
github.com/google/go-github/v42 v42.0.0
github.com/googleapis/gax-go/v2 v2.12.0
github.com/h2non/filetype v1.1.3
github.com/hashicorp/go-retryablehttp v0.7.4
github.com/hashicorp/golang-lru v0.5.1
github.com/jlaffaye/ftp v0.2.0
-2
View File
@@ -315,8 +315,6 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+1 -1
View File
@@ -133,7 +133,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
-91
View File
@@ -1,91 +0,0 @@
package huggingface
import (
"context"
"fmt"
"net/http"
"regexp"
"strings"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
type Scanner struct {
client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\bhf_[a-zA-Z]{34}\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
return []string{"huggingface", "hugging_face", "hf"} // Huggingface docs occasionally use "hf" instead of "huggingface"
}
// FromData will find and optionally verify Huggingface secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
if len(match) != 1 {
continue
}
resMatch := strings.TrimSpace(match[0])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_HuggingFace,
Raw: []byte(resMatch),
}
if verify {
client := s.client
if client == nil {
client = defaultClient
}
req, err := http.NewRequestWithContext(ctx, "GET", "https://huggingface.co/api/whoami-v2", nil)
if err != nil {
continue
}
req.Header.Set("Authorization", "Bearer "+resMatch)
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
} else if res.StatusCode == 401 {
// The secret is determinately not verified (nothing to do)
} else {
s1.VerificationError = fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
} else {
s1.VerificationError = err
}
}
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
continue
}
results = append(results, s1)
}
return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_HuggingFace
}
@@ -1,160 +0,0 @@
//go:build detectors
// +build detectors
package huggingface
import (
"context"
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"testing"
"time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
func TestHuggingface_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
secret := testSecrets.MustGetField("HUGGINGFACE")
inactiveSecret := testSecrets.MustGetField("HUGGINGFACE_INACTIVE")
type args struct {
ctx context.Context
data []byte
verify bool
}
tests := []struct {
name string
s Scanner
args args
want []detectors.Result
wantErr bool
wantVerificationErr bool
}{
{
name: "found, verified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_HuggingFace,
Verified: true,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, unverified",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within but not valid", inactiveSecret)), // the secret would satisfy the regex but not pass validation
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_HuggingFace,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: false,
},
{
name: "not found",
s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
verify: true,
},
want: nil,
wantErr: false,
wantVerificationErr: false,
},
{
name: "found, would be verified if not for timeout",
s: Scanner{client: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_HuggingFace,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
{
name: "found, verified but unexpected api surface",
s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a huggingface secret %s within", secret)),
verify: true,
},
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_HuggingFace,
Verified: false,
},
},
wantErr: false,
wantVerificationErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("Huggingface.FromData() error = %v, wantErr %v", err, tt.wantErr)
return
}
for i := range got {
if len(got[i].Raw) == 0 {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationErr {
t.Fatalf(" wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "VerificationError")
if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {
t.Errorf("Huggingface.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
}
})
}
}
func BenchmarkFromData(benchmark *testing.B) {
ctx := context.Background()
s := Scanner{}
for name, data := range detectors.MustGetBenchmarkData() {
benchmark.Run(name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
_, err := s.FromData(ctx, false, data)
if err != nil {
b.Fatal(err)
}
}
})
}
}
+1 -1
View File
@@ -52,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/alerts", nil)
req, err := http.NewRequestWithContext(ctx, "GET", "https://api.opsgenie.com/v2/users", nil)
if err != nil {
continue
}
+10 -13
View File
@@ -595,20 +595,17 @@ func SupportsLineNumbers(sourceType sourcespb.SourceType) bool {
// FragmentLineOffset sets the line number for a provided source chunk with a given detector result.
func FragmentLineOffset(chunk *sources.Chunk, result *detectors.Result) (int64, bool) {
before, after, found := bytes.Cut(chunk.Data, result.Raw)
if !found {
return 0, false
lines := bytes.Split(chunk.Data, []byte("\n"))
for i, line := range lines {
if bytes.Contains(line, result.Raw) {
// if the line contains the ignore tag, we should ignore the result
if bytes.Contains(line, []byte(ignoreTag)) {
return int64(i), true
}
return int64(i), false
}
}
lineNumber := int64(bytes.Count(before, []byte("\n")))
// If the line contains the ignore tag, we should ignore the result.
endLine := bytes.Index(after, []byte("\n"))
if endLine == -1 {
endLine = len(after)
}
if bytes.Contains(after[:endLine], []byte(ignoreTag)) {
return lineNumber, true
}
return lineNumber, false
return 0, false
}
// FragmentFirstLine returns the first line number of a fragment along with a pointer to the value to update in the
-71
View File
@@ -1,7 +1,6 @@
package engine
import (
"fmt"
"testing"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
@@ -51,39 +50,6 @@ func TestFragmentLineOffset(t *testing.T) {
expectedLine: 4,
ignore: false,
},
{
name: "match on consecutive lines",
chunk: &sources.Chunk{
Data: []byte("line1\nline2\ntrufflehog:ignore\nline4\nsecret\nhere\nline6"),
},
result: &detectors.Result{
Raw: []byte("secret\nhere"),
},
expectedLine: 4,
ignore: false,
},
{
name: "ignore on last consecutive lines",
chunk: &sources.Chunk{
Data: []byte("line1\nline2\nline3\nsecret\nhere // trufflehog:ignore\nline5"),
},
result: &detectors.Result{
Raw: []byte("secret\nhere"),
},
expectedLine: 3,
ignore: true,
},
{
name: "ignore on last line",
chunk: &sources.Chunk{
Data: []byte("line1\nline2\nline3\nsecret here // trufflehog:ignore"),
},
result: &detectors.Result{
Raw: []byte("secret here"),
},
expectedLine: 3,
ignore: true,
},
}
for _, tt := range tests {
@@ -99,43 +65,6 @@ func TestFragmentLineOffset(t *testing.T) {
}
}
func setupFragmentLineOffsetBench(totalLines, needleLine int) (*sources.Chunk, *detectors.Result) {
data := make([]byte, 0, 4096)
needle := []byte("needle")
for i := 0; i < totalLines; i++ {
if i != needleLine {
data = append(data, []byte(fmt.Sprintf("line%d\n", i))...)
continue
}
data = append(data, needle...)
data = append(data, '\n')
}
chunk := &sources.Chunk{Data: data}
result := &detectors.Result{Raw: needle}
return chunk, result
}
func BenchmarkFragmentLineOffsetStart(b *testing.B) {
chunk, result := setupFragmentLineOffsetBench(512, 2)
for i := 0; i < b.N; i++ {
_, _ = FragmentLineOffset(chunk, result)
}
}
func BenchmarkFragmentLineOffsetMiddle(b *testing.B) {
chunk, result := setupFragmentLineOffsetBench(512, 256)
for i := 0; i < b.N; i++ {
_, _ = FragmentLineOffset(chunk, result)
}
}
func BenchmarkFragmentLineOffsetEnd(b *testing.B) {
chunk, result := setupFragmentLineOffsetBench(512, 510)
for i := 0; i < b.N; i++ {
_, _ = FragmentLineOffset(chunk, result)
}
}
// Test to make sure that DefaultDecoders always returns the UTF8 decoder first.
// Technically a decoder test but we want this to run and fail in CI
func TestDefaultDecoders(t *testing.T) {
+18 -294
View File
@@ -6,13 +6,8 @@ import (
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/h2non/filetype"
"github.com/mholt/archiver/v4"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -31,18 +26,14 @@ var (
maxTimeout = time.Duration(30) * time.Second
)
// Ensure the Archive satisfies the interfaces at compile time.
var _ SpecializedHandler = (*Archive)(nil)
// Archive is a handler for extracting and decompressing archives.
type Archive struct {
size int
currentDepth int
size int
}
// New sets a default maximum size and current size counter.
func (a *Archive) New() {
a.size = 0
func (d *Archive) New() {
d.size = 0
}
// SetArchiveMaxSize sets the maximum size of the archive.
@@ -61,14 +52,14 @@ func SetArchiveMaxTimeout(timeout time.Duration) {
}
// FromFile extracts the files from an archive.
func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []byte {
archiveChan := make(chan []byte, 512)
func (d *Archive) FromFile(originalCtx context.Context, data io.Reader) chan ([]byte) {
archiveChan := make(chan ([]byte), 512)
go func() {
ctx, cancel := context.WithTimeout(originalCtx, maxTimeout)
logger := logContext.AddLogger(ctx).Logger()
defer cancel()
defer close(archiveChan)
err := a.openArchive(ctx, 0, data, archiveChan)
err := d.openArchive(ctx, 0, data, archiveChan)
if err != nil {
if errors.Is(err, archiver.ErrNoMatch) {
return
@@ -80,7 +71,7 @@ func (a *Archive) FromFile(originalCtx context.Context, data io.Reader) chan []b
}
// openArchive takes a reader and extracts the contents up to the maximum depth.
func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan []byte) error {
func (d *Archive) openArchive(ctx context.Context, depth int, reader io.Reader, archiveChan chan ([]byte)) error {
if depth >= maxDepth {
return fmt.Errorf("max archive depth reached")
}
@@ -106,14 +97,14 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
if err != nil {
return err
}
fileBytes, err := a.ReadToMax(ctx, compReader)
fileBytes, err := d.ReadToMax(ctx, compReader)
if err != nil {
return err
}
newReader := bytes.NewReader(fileBytes)
return a.openArchive(ctx, depth+1, newReader, archiveChan)
return d.openArchive(ctx, depth+1, newReader, archiveChan)
case archiver.Extractor:
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, a.extractorHandler(archiveChan))
err := archive.Extract(context.WithValue(ctx, depthKey, depth+1), reader, nil, d.extractorHandler(archiveChan))
if err != nil {
return err
}
@@ -123,7 +114,7 @@ func (a *Archive) openArchive(ctx context.Context, depth int, reader io.Reader,
}
// IsFiletype returns true if the provided reader is an archive.
func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
func (d *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader, bool) {
format, readerB, err := archiver.Identify("", reader)
if err != nil {
return readerB, false
@@ -138,7 +129,7 @@ func (a *Archive) IsFiletype(ctx context.Context, reader io.Reader) (io.Reader,
}
// extractorHandler is applied to each file in an archiver.Extractor file.
func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context, archiver.File) error {
func (d *Archive) extractorHandler(archiveChan chan ([]byte)) func(context.Context, archiver.File) error {
return func(ctx context.Context, f archiver.File) error {
logger := logContext.AddLogger(ctx).Logger()
logger.V(5).Info("Handling extracted file.", "filename", f.Name())
@@ -151,13 +142,13 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
if err != nil {
return err
}
fileBytes, err := a.ReadToMax(ctx, fReader)
fileBytes, err := d.ReadToMax(ctx, fReader)
if err != nil {
return err
}
fileContent := bytes.NewReader(fileBytes)
err = a.openArchive(ctx, depth, fileContent, archiveChan)
err = d.openArchive(ctx, depth, fileContent, archiveChan)
if err != nil {
return err
}
@@ -166,7 +157,7 @@ func (a *Archive) extractorHandler(archiveChan chan []byte) func(context.Context
}
// ReadToMax reads up to the max size.
func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) {
func (d *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte, err error) {
// Archiver v4 is in alpha and using an experimental version of
// rardecode. There is a bug somewhere with rar decoder format 29
// that can lead to a panic. An issue is open in rardecode repo
@@ -184,7 +175,7 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
}
}()
fileContent := bytes.Buffer{}
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-a.size)
logger.V(5).Info("Remaining buffer capacity", "bytes", maxSize-d.size)
for i := 0; i <= maxSize/512; i++ {
if common.IsDone(ctx) {
return nil, ctx.Err()
@@ -194,284 +185,17 @@ func (a *Archive) ReadToMax(ctx context.Context, reader io.Reader) (data []byte,
if err != nil && !errors.Is(err, io.EOF) {
return []byte{}, err
}
a.size += bRead
d.size += bRead
if len(fileChunk) > 0 {
fileContent.Write(fileChunk[0:bRead])
}
if bRead < 512 {
return fileContent.Bytes(), nil
}
if a.size >= maxSize && bRead == 512 {
if d.size >= maxSize && bRead == 512 {
logger.V(2).Info("Max archive size reached.")
return fileContent.Bytes(), nil
}
}
return fileContent.Bytes(), nil
}
const (
arMimeType = "application/x-unix-archive"
rpmMimeType = "application/x-rpm"
)
// Define a map of mime types to corresponding command-line tools
var mimeTools = map[string][]string{
arMimeType: {"ar"},
rpmMimeType: {"rpm2cpio", "cpio"},
}
// Check if the command-line tool is installed.
func isToolInstalled(tool string) bool {
_, err := exec.LookPath(tool)
return err == nil
}
// Ensure all tools are available for given mime type.
func ensureToolsForMimeType(mimeType string) error {
tools, exists := mimeTools[mimeType]
if !exists {
return fmt.Errorf("unsupported mime type")
}
for _, tool := range tools {
if !isToolInstalled(tool) {
return fmt.Errorf("Required tool " + tool + " is not installed")
}
}
return nil
}
// HandleSpecialized takes a file path and an io.Reader representing the input file,
// and processes it based on its extension, such as handling Debian (.deb) and RPM (.rpm) packages.
// It returns an io.Reader that can be used to read the processed content of the file,
// and an error if any issues occurred during processing.
// The caller is responsible for closing the returned reader.
func (a *Archive) HandleSpecialized(ctx context.Context, reader io.Reader) (io.Reader, bool, error) {
mimeType, reader, err := determineMimeType(reader)
if err != nil {
return nil, false, err
}
switch mimeType {
case arMimeType: // includes .deb files
if err := ensureToolsForMimeType(mimeType); err != nil {
return nil, false, err
}
reader, err = a.extractDebContent(ctx, reader)
case rpmMimeType:
if err := ensureToolsForMimeType(mimeType); err != nil {
return nil, false, err
}
reader, err = a.extractRpmContent(ctx, reader)
default:
return reader, false, nil
}
if err != nil {
return nil, false, fmt.Errorf("unable to extract file with MIME type %s: %w", mimeType, err)
}
return reader, true, nil
}
// extractDebContent takes a .deb file as an io.Reader, extracts its contents
// into a temporary directory, and returns a Reader for the extracted data archive.
// It handles the extraction process by using the 'ar' command and manages temporary
// files and directories for the operation.
// The caller is responsible for closing the returned reader.
func (a *Archive) extractDebContent(ctx context.Context, file io.Reader) (io.ReadCloser, error) {
if a.currentDepth >= maxDepth {
return nil, fmt.Errorf("max archive depth reached")
}
tmpEnv, err := a.createTempEnv(ctx, file)
if err != nil {
return nil, err
}
defer os.Remove(tmpEnv.tempFileName)
defer os.RemoveAll(tmpEnv.extractPath)
cmd := exec.Command("ar", "x", tmpEnv.tempFile.Name())
cmd.Dir = tmpEnv.extractPath
if err := executeCommand(cmd); err != nil {
return nil, err
}
handler := func(ctx context.Context, env tempEnv, file string) (string, error) {
if strings.HasPrefix(file, "data.tar.") {
return file, nil
}
return a.handleNestedFileMIME(ctx, env, file)
}
dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler)
if err != nil {
return nil, err
}
return openDataArchive(tmpEnv.extractPath, dataArchiveName)
}
// extractRpmContent takes an .rpm file as an io.Reader, extracts its contents
// into a temporary directory, and returns a Reader for the extracted data archive.
// It handles the extraction process by using the 'rpm2cpio' and 'cpio' commands and manages temporary
// files and directories for the operation.
// The caller is responsible for closing the returned reader.
func (a *Archive) extractRpmContent(ctx context.Context, file io.Reader) (io.ReadCloser, error) {
if a.currentDepth >= maxDepth {
return nil, fmt.Errorf("max archive depth reached")
}
tmpEnv, err := a.createTempEnv(ctx, file)
if err != nil {
return nil, err
}
defer os.Remove(tmpEnv.tempFileName)
defer os.RemoveAll(tmpEnv.extractPath)
// Use rpm2cpio to convert the RPM file to a cpio archive and then extract it using cpio command.
cmd := exec.Command("sh", "-c", "rpm2cpio "+tmpEnv.tempFile.Name()+" | cpio -id")
cmd.Dir = tmpEnv.extractPath
if err := executeCommand(cmd); err != nil {
return nil, err
}
handler := func(ctx context.Context, env tempEnv, file string) (string, error) {
if strings.HasSuffix(file, ".tar.gz") {
return file, nil
}
return a.handleNestedFileMIME(ctx, env, file)
}
dataArchiveName, err := a.handleExtractedFiles(ctx, tmpEnv, handler)
if err != nil {
return nil, err
}
return openDataArchive(tmpEnv.extractPath, dataArchiveName)
}
func (a *Archive) handleNestedFileMIME(ctx context.Context, tempEnv tempEnv, fileName string) (string, error) {
nestedFile, err := os.Open(filepath.Join(tempEnv.extractPath, fileName))
if err != nil {
return "", err
}
defer nestedFile.Close()
mimeType, reader, err := determineMimeType(nestedFile)
if err != nil {
return "", err
}
switch mimeType {
case arMimeType:
_, _, err = a.HandleSpecialized(ctx, reader)
case rpmMimeType:
_, _, err = a.HandleSpecialized(ctx, reader)
default:
return "", nil
}
if err != nil {
return "", err
}
return fileName, nil
}
// determineMimeType reads from the provided reader to detect the MIME type.
// It returns the detected MIME type and a new reader that includes the read portion.
func determineMimeType(reader io.Reader) (string, io.Reader, error) {
buffer := make([]byte, 512)
n, err := reader.Read(buffer)
if err != nil {
return "", nil, fmt.Errorf("unable to read file for MIME type detection: %w", err)
}
// Create a new reader that starts with the buffer we just read
// and continues with the rest of the original reader.
reader = io.MultiReader(bytes.NewReader(buffer[:n]), reader)
kind, err := filetype.Match(buffer)
if err != nil {
return "", nil, fmt.Errorf("unable to determine file type: %w", err)
}
return kind.MIME.Value, reader, nil
}
// handleExtractedFiles processes each file in the extracted directory using a provided handler function.
// The function iterates through the files, applying the handleFile function to each, and returns the name
// of the data archive it finds. This centralizes the logic for handling specialized files such as .deb and .rpm
// by using the appropriate handling function passed as an argument. This design allows for flexibility and reuse
// of this function across various extraction processes in the package.
func (a *Archive) handleExtractedFiles(ctx context.Context, env tempEnv, handleFile func(context.Context, tempEnv, string) (string, error)) (string, error) {
extractedFiles, err := os.ReadDir(env.extractPath)
if err != nil {
return "", fmt.Errorf("unable to read extracted directory: %w", err)
}
var dataArchiveName string
for _, file := range extractedFiles {
name, err := handleFile(ctx, env, file.Name())
if err != nil {
return "", err
}
if name != "" {
dataArchiveName = name
break
}
}
return dataArchiveName, nil
}
type tempEnv struct {
tempFile *os.File
tempFileName string
extractPath string
}
// createTempEnv creates a temporary file and a temporary directory for extracting archives.
// The caller is responsible for removing these temporary resources
// (both the file and directory) when they are no longer needed.
func (a *Archive) createTempEnv(ctx context.Context, file io.Reader) (tempEnv, error) {
tempFile, err := os.CreateTemp("", "tmp")
if err != nil {
return tempEnv{}, fmt.Errorf("unable to create temporary file: %w", err)
}
extractPath, err := os.MkdirTemp("", "tmp_archive")
if err != nil {
return tempEnv{}, fmt.Errorf("unable to create temporary directory: %w", err)
}
b, err := a.ReadToMax(ctx, file)
if err != nil {
return tempEnv{}, err
}
if _, err = tempFile.Write(b); err != nil {
return tempEnv{}, fmt.Errorf("unable to write to temporary file: %w", err)
}
return tempEnv{tempFile: tempFile, tempFileName: tempFile.Name(), extractPath: extractPath}, nil
}
func executeCommand(cmd *exec.Cmd) error {
var stderr bytes.Buffer
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
return fmt.Errorf("unable to execute command: %w; error: %s", err, stderr.String())
}
return nil
}
func openDataArchive(extractPath string, dataArchiveName string) (io.ReadCloser, error) {
dataArchivePath := filepath.Join(extractPath, dataArchiveName)
dataFile, err := os.Open(dataArchivePath)
if err != nil {
return nil, fmt.Errorf("unable to open file: %w", err)
}
return dataFile, nil
}
-38
View File
@@ -2,9 +2,7 @@ package handlers
import (
"context"
"io"
"net/http"
"os"
"regexp"
"strings"
"testing"
@@ -124,39 +122,3 @@ func TestHandleFile(t *testing.T) {
assert.True(t, HandleFile(context.Background(), reader, &sources.Chunk{}, ch))
assert.Equal(t, 1, len(ch))
}
func TestExtractDebContent(t *testing.T) {
// Open the sample .deb file from the testdata folder.
file, err := os.Open("testdata/test.deb")
assert.Nil(t, err)
defer file.Close()
ctx := context.Background()
a := &Archive{}
reader, err := a.extractDebContent(ctx, file)
assert.Nil(t, err)
content, err := io.ReadAll(reader)
assert.Nil(t, err)
expectedLength := 1015582
assert.Equal(t, expectedLength, len(string(content)))
}
func TestExtractRPMContent(t *testing.T) {
// Open the sample .rpm file from the testdata folder.
file, err := os.Open("testdata/test.rpm")
assert.Nil(t, err)
defer file.Close()
ctx := context.Background()
a := &Archive{}
reader, err := a.extractRpmContent(ctx, file)
assert.Nil(t, err)
content, err := io.ReadAll(reader)
assert.Nil(t, err)
expectedLength := 1822720
assert.Equal(t, expectedLength, len(string(content)))
}
+85 -32
View File
@@ -1,10 +1,20 @@
package handlers
import (
"archive/tar"
"context"
"fmt"
"io"
"math/rand"
"os"
"runtime"
"github.com/google/go-containerregistry/pkg/v1/tarball"
context2 "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/docker"
"google.golang.org/protobuf/types/known/anypb"
)
func DefaultHandlers() []Handler {
@@ -13,59 +23,67 @@ func DefaultHandlers() []Handler {
}
}
// SpecializedHandler defines the interface for handlers that can process specialized archives.
// It includes a method to handle specialized archives and determine if the file is of a special type.
type SpecializedHandler interface {
// HandleSpecialized examines the provided file reader within the context and determines if it is a specialized archive.
// It returns a reader with any necessary modifications, a boolean indicating if the file was specialized,
// and an error if something went wrong during processing.
HandleSpecialized(context.Context, io.Reader) (io.Reader, bool, error)
}
type Handler interface {
FromFile(context.Context, io.Reader) chan ([]byte)
IsFiletype(context.Context, io.Reader) (io.Reader, bool)
New()
}
// HandleFile processes a given file by selecting an appropriate handler from DefaultHandlers.
// It first checks if the handler implements SpecializedHandler for any special processing,
// then falls back to regular file type handling. If successful, it reads the file in chunks,
// packages them in the provided chunk skeleton, and sends them to chunksChan.
// The function returns true if processing was successful and false otherwise.
// Context is used for cancellation, and the caller is responsible for canceling it if needed.
func HandleFile(ctx context.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
func HandleFile(ctx context2.Context, file io.Reader, chunkSkel *sources.Chunk, chunksChan chan (*sources.Chunk)) bool {
var handler Handler
for _, h := range DefaultHandlers() {
h.New()
var (
isSpecial bool
err error
)
// Check if the handler implements SpecializedHandler and process accordingly.
if specialHandler, ok := h.(SpecializedHandler); ok {
if file, isSpecial, err = specialHandler.HandleSpecialized(ctx, file); isSpecial && err == nil {
return handleChunks(ctx, h.FromFile(ctx, file), chunkSkel, chunksChan)
}
}
var isType bool
if file, isType = h.IsFiletype(ctx, file); isType {
return handleChunks(ctx, h.FromFile(ctx, file), chunkSkel, chunksChan)
handler = h
break
}
}
return false
}
if handler == nil {
return false
}
func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sources.Chunk, chunksChan chan *sources.Chunk) bool {
if IsReaderTarball(ctx, file) {
isDocker, path := IsDockerImage(ctx, file)
if isDocker {
// if docker image then run
ctx.logger.V(3).Info("Docker image detected in tarball: " + path)
//scan with docker scanner
dockerConn := sourcespb.Docker{
Images: []string{"file://" + path},
Credential: &sourcespb.Docker_DockerKeychain{
DockerKeychain: true,
},
}
anyConn, err := anypb.New(&dockerConn)
if err != nil {
ctx.logger.V(3).Info("error marshalling Docker connection", "error", err)
}
dockerSource := docker.Source{}
if err := dockerSource.Init(ctx, "trufflehog - docker", 0, chunkSkel.SourceID, chunkSkel.Verify, anyConn, runtime.NumCPU()); err != nil {
// will need to pas this in later s.jobId
return false
}
if err := dockerSource.Chunks(ctx, chunksChan); err != nil {
return false
}
return true
}
}
// Process the file and read all []byte chunks from handlerChan.
handlerChan := handler.FromFile(ctx, file)
for {
select {
case data, open := <-handlerChan:
if !open {
// We finished reading everything from handlerChan.
return true
}
chunk := *chunkSkel
chunk.Data = data
// Send data on chunksChan.
select {
case chunksChan <- &chunk:
case <-ctx.Done():
@@ -76,3 +94,38 @@ func handleChunks(ctx context.Context, handlerChan chan []byte, chunkSkel *sourc
}
}
}
func IsReaderTarball(ctx context2.Context, reader io.Reader) bool {
// Check if file is a tarball, if so, check if it is a Docker image.
_, err := tar.NewReader(reader).Next()
if err != nil {
return false
}
return true
}
func IsDockerImage(ctx context2.Context, file io.Reader) (isDocker bool, tmpFilename string) {
// Generate a random name for the temporary file.
tmpFilename = fmt.Sprintf("%s/temp_%d.tar", os.TempDir(), rand.Int())
// Create a temporary file to write image data from ReadCloser.
tmpFile, err := os.Create(tmpFilename)
if err != nil {
fmt.Printf("Error creating temporary file: %s\n", err)
return false, ""
}
defer os.Remove(tmpFilename)
// Copy data from ReadCloser to the temporary file.
if _, err := io.Copy(tmpFile, file); err != nil {
fmt.Printf("Error copying data to temporary file: %s\n", err)
return false, ""
}
tmpFile.Close()
_, err = tarball.ImageFromPath(tmpFilename, nil)
if err != nil {
return false, ""
}
return true, tmpFilename
}
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
+6 -11
View File
@@ -996,8 +996,6 @@ const (
DetectorType_TrufflehogEnterprise DetectorType = 922
DetectorType_EnvoyApiKey DetectorType = 923
DetectorType_GitHubOauth2 DetectorType = 924
// 925 reserved for Salesforce
DetectorType_HuggingFace DetectorType = 926
)
// Enum value maps for DetectorType.
@@ -1924,7 +1922,6 @@ var (
922: "TrufflehogEnterprise",
923: "EnvoyApiKey",
924: "GitHubOauth2",
926: "HuggingFace",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2848,7 +2845,6 @@ var (
"TrufflehogEnterprise": 922,
"EnvoyApiKey": 923,
"GitHubOauth2": 924,
"HuggingFace": 926,
}
)
@@ -3227,7 +3223,7 @@ var file_detectors_proto_rawDesc = []byte{
0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55,
0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c, 0x41, 0x49,
0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10, 0x02, 0x12,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xeb, 0x73, 0x0a, 0x0c, 0x44,
0x09, 0x0a, 0x05, 0x55, 0x54, 0x46, 0x31, 0x36, 0x10, 0x03, 0x2a, 0xd9, 0x73, 0x0a, 0x0c, 0x44,
0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41,
0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50,
0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41,
@@ -4153,12 +4149,11 @@ var file_detectors_proto_rawDesc = []byte{
0x65, 0x68, 0x6f, 0x67, 0x45, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x69, 0x73, 0x65, 0x10, 0x9a,
0x07, 0x12, 0x10, 0x0a, 0x0b, 0x45, 0x6e, 0x76, 0x6f, 0x79, 0x41, 0x70, 0x69, 0x4b, 0x65, 0x79,
0x10, 0x9b, 0x07, 0x12, 0x11, 0x0a, 0x0c, 0x47, 0x69, 0x74, 0x48, 0x75, 0x62, 0x4f, 0x61, 0x75,
0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x48, 0x75, 0x67, 0x67, 0x69, 0x6e,
0x67, 0x46, 0x61, 0x63, 0x65, 0x10, 0x9e, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65,
0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f,
0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65,
0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x74, 0x68, 0x32, 0x10, 0x9c, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75,
0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f,
0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74,
0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
+1 -1
View File
@@ -21,7 +21,7 @@ func GetFields() githubCmdModel {
Key: "org",
Required: false,
Help: "GitHub organization to scan.",
Placeholder: "trufflesecurity",
Placeholder: "https://github.com/trufflesecurity",
}
repo := textinputs.InputConfig{
-2
View File
@@ -933,8 +933,6 @@ enum DetectorType {
TrufflehogEnterprise = 922;
EnvoyApiKey = 923;
GitHubOauth2 = 924;
// 925 reserved for Salesforce
HuggingFace = 926;
}
message Result {