Trufflehog integration #11

Signed-off-by: Hadi <[email protected]>

README.md

@@ -119,6 +119,7 @@ github-recon [--flags value] target_username_or_email
 -r, --refresh                Refresh the cache (only for deep scan)
 -s, --show-source            Show where the information (authors, emails, etc) were found (only for deep scan)
 -m, --max-distance int       Maximum Levenshtein distance for matching usernames & emails (only for deep scan) (default 20)
+    --trufflehog             Run trufflehog on cloned repositories (only for deep scan) (default true)
 -S, --silent                 Suppress all non-essential output
     --spoof-email            Spoof email (only for email mode) (default true)
 -a, --hide-avatar            Hide the avatar in the output

cmd/print.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"reflect"
+	"sort"
 	"strings"
 
 	github_recon_settings "github.com/anotherhadi/github-recon/settings"
@@ -27,11 +28,17 @@ func printStruct(settings github_recon_settings.Settings, s any, indent int) {
 	prefix := strings.Repeat("  ", indent)
 
 	v := reflect.ValueOf(s)
+	if !v.IsValid() {
+		return
+	}
 	t := reflect.TypeOf(s)
 
-	if v.Kind() == reflect.Ptr {
+	for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface {
+		if v.IsNil() {
+			return
+		}
 		v = v.Elem()
-		t = t.Elem()
+		t = v.Type()
 	}
 
 	switch v.Kind() {
@@ -43,7 +50,6 @@ func printStruct(settings github_recon_settings.Settings, s any, indent int) {
 		}
 
 		printed := 0
-
 		for i := 0; i < v.NumField(); i++ {
 			field := t.Field(i).Name
 			value := v.Field(i)
@@ -60,11 +66,13 @@ func printStruct(settings github_recon_settings.Settings, s any, indent int) {
 			printed++
 
 			switch value.Kind() {
-			case reflect.Struct, reflect.Slice, reflect.Array, reflect.Ptr:
+			case reflect.Struct, reflect.Slice, reflect.Array, reflect.Ptr, reflect.Map, reflect.Interface:
 				fmt.Println(prefix + greyStyle.Render(field+":"))
 				printStruct(settings, value.Interface(), indent+1)
+
 			case reflect.String:
 				fmt.Printf("%s%s %s\n", prefix, greyStyle.Render(field+":"), greenStyle.Render(fmt.Sprintf("%q", value.Interface())))
+
 			default:
 				fmt.Printf("%s%s %s\n", prefix, greyStyle.Render(field+":"), greenStyle.Render(fmt.Sprintf("%v", value.Interface())))
 			}
@@ -84,9 +92,31 @@ func printStruct(settings github_recon_settings.Settings, s any, indent int) {
 			printStruct(settings, v.Index(i).Interface(), indent)
 		}
 
+	case reflect.Map:
+		if v.Len() == 0 {
+			fmt.Println(prefix + greyStyle.Render("No data found"))
+			return
+		}
+
+		keys := v.MapKeys()
+		keyStrs := make([]string, len(keys))
+		for i, k := range keys {
+			keyStrs[i] = fmt.Sprintf("%v", k.Interface())
+		}
+		sort.Strings(keyStrs)
+
+		for _, keyStr := range keyStrs {
+			for _, k := range keys {
+				if fmt.Sprintf("%v", k.Interface()) == keyStr {
+					val := v.MapIndex(k)
+					fmt.Println(prefix + greyStyle.Render(fmt.Sprintf("%v:", k.Interface())))
+					printStruct(settings, val.Interface(), indent+1)
+				}
+			}
+		}
+
 	default:
 		fmt.Println(prefix + greenStyle.Render(fmt.Sprintf("%v", v.Interface())))
-		fmt.Println("")
 	}
 }
 

github-recon/username/deep.go

@@ -1,6 +1,8 @@
 package recon
 
 import (
+	"encoding/json"
+	"fmt"
 	"io/fs"
 	"os"
 	"os/exec"
@@ -14,26 +16,34 @@ import (
 	"github.com/google/go-github/v72/github"
 )
 
-type Authors []AuthorOccurrence
+type Authors []Author
 
-type AuthorOccurrence struct {
+type Author struct {
 	Name        string
 	Levenshtein int
 	Email       string
 	FoundIn     []string
 }
 
-type Emails []EmailOccurrence
+type Emails []Email
 
-type EmailOccurrence struct {
+type Email struct {
 	Email       string
 	Levenshtein int
 	FoundIn     []string
 }
 
+type Secrets []Secret
+
+type Secret struct {
+	Repositorie string
+	Raw         map[string]any
+}
+
 type DeepScanResult struct {
-	Authors []AuthorOccurrence
+	Authors Authors
-	Emails  []EmailOccurrence
+	Emails  Emails
+	Secrets Secrets
 }
 
 type Repositorie struct {
@@ -43,62 +53,6 @@ type Repositorie struct {
 	Size       int
 }
 
-func findEmailsAndOccurrencesInDir(rootPath string, username string) (Emails, error) {
-	emailLocations := make(map[string]map[string]bool)
-	emailRegex := regexp.MustCompile(`[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}`)
-	normalizedRootPath := filepath.Clean(rootPath)
-
-	err := filepath.WalkDir(rootPath, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-		if !d.IsDir() {
-			if strings.Contains(path, ".git/logs/") {
-				return nil
-			}
-			content, err := os.ReadFile(path)
-			if err != nil {
-				return err
-			}
-
-			currentFileEmails := emailRegex.FindAllString(string(content), -1)
-			if len(currentFileEmails) > 0 {
-				relativePath, errRel := filepath.Rel(normalizedRootPath, path)
-				if errRel != nil {
-					relativePath = path
-				}
-
-				for _, email := range currentFileEmails {
-					if len(email) > 12 {
-						if _, ok := emailLocations[email]; !ok {
-							emailLocations[email] = make(map[string]bool)
-						}
-						emailLocations[email][relativePath] = true
-					}
-				}
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var results []EmailOccurrence
-	for email, pathSet := range emailLocations {
-		var paths []string
-		for path := range pathSet {
-			paths = append(paths, path)
-		}
-		results = append(results, EmailOccurrence{
-			Email: email, FoundIn: paths,
-			Levenshtein: utils.LevenshteinDistance(username, strings.SplitN(email, "@", 2)[0]),
-		})
-	}
-
-	return results, nil
-}
-
 func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) {
 	repositories := []Repositorie{}
 	repos, resp, err := s.Client.Repositories.ListByUser(
@@ -185,7 +139,7 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) {
 	}
 	s.Logger.Info("Cloned all repositories", "path", tmp_folder)
 
-	authorOccurrences := []AuthorOccurrence{}
+	authorOccurrences := Authors{}
 	mapAuthorToIndex := make(map[string]int)
 	for _, repo := range repositories {
 		destination := tmp_folder + "/" + repo.Owner + "/" + repo.Name
@@ -244,7 +198,7 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) {
 							continue
 						}
 
-						authorOccurrences = append(authorOccurrences, AuthorOccurrence{
+						authorOccurrences = append(authorOccurrences, Author{
 							Name:        authorName,
 							Email:       authorEmail,
 							FoundIn:     []string{repoIdentifier},
@@ -256,14 +210,14 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) {
 			}
 		}
 	}
-	slices.SortFunc(authorOccurrences, func(a, b AuthorOccurrence) int {
+	slices.SortFunc(authorOccurrences, func(a, b Author) int {
 		if a.Levenshtein != b.Levenshtein {
 			return a.Levenshtein - b.Levenshtein
 		}
 		return 1
 	})
 
-	authors := []AuthorOccurrence{}
+	authors := Authors{}
 	for _, author := range authorOccurrences {
 		if author.Levenshtein > s.MaxDistance {
 			continue
@@ -280,14 +234,14 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) {
 		s.Logger.Error("Failed to find emails in directory", "err", err)
 		return
 	}
-	slices.SortFunc(emailsFound, func(a, b EmailOccurrence) int {
+	slices.SortFunc(emailsFound, func(a, b Email) int {
 		if a.Levenshtein != b.Levenshtein {
 			return a.Levenshtein - b.Levenshtein
 		}
 		return 1
 	})
 
-	emails := []EmailOccurrence{}
+	emails := Emails{}
 	for _, email := range emailsFound {
 		if email.Levenshtein > s.MaxDistance {
 			continue
@@ -298,5 +252,131 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) {
 	response.Authors = authors
 	response.Emails = emails
 
+	s.Logger.Info("Searching for secrets in cloned repositories", "path", tmp_folder)
+	if s.Trufflehog {
+		cmd := exec.Command("trufflehog", "--version")
+		if err := cmd.Run(); err != nil {
+			s.Logger.Warn("Trufflehog is not installed, skipping secret scanning.")
+		} else {
+			secrets, err := truffleHog(tmp_folder)
+			if err != nil {
+				s.Logger.Error("Failed to run trufflehog", "err", err)
+			} else {
+				response.Secrets = secrets
+			}
+		}
+	}
+
 	return
 }
+
+func truffleHog(tmpFolder string) (Secrets, error) {
+	allSecrets := Secrets{}
+
+	directories, err := os.ReadDir(tmpFolder)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read tmp folder: %w", err)
+	}
+
+	for _, dir := range directories {
+		if !dir.IsDir() {
+			continue
+		}
+
+		innerPath := filepath.Join(tmpFolder, dir.Name())
+		innerDirectories, err := os.ReadDir(innerPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read inner tmp folder: %w", err)
+		}
+
+		for _, innerDir := range innerDirectories {
+			if !innerDir.IsDir() {
+				continue
+			}
+
+			repoPath := filepath.Join(innerPath, innerDir.Name())
+			cmd := exec.Command("trufflehog", "git", "file://"+repoPath, "--json", "--log-level=-1", "--results=verified")
+			output, err := cmd.Output()
+
+			if err != nil {
+				if exitErr, ok := err.(*exec.ExitError); ok {
+					if exitErr.ExitCode() > 1 {
+						return nil, fmt.Errorf("failed to execute trufflehog (ExitError): %s", string(exitErr.Stderr))
+					}
+				} else {
+					return nil, fmt.Errorf("failed to execute trufflehog: %w", err)
+				}
+			}
+
+			decoder := json.NewDecoder(strings.NewReader(string(output)))
+			for decoder.More() {
+				var result map[string]any
+				if err := decoder.Decode(&result); err != nil {
+					return nil, fmt.Errorf("failed to parse trufflehog output: %w", err)
+				}
+				allSecrets = append(allSecrets, Secret{
+					Repositorie: dir.Name() + "/" + innerDir.Name(),
+					Raw:         result,
+				})
+			}
+		}
+	}
+
+	return allSecrets, nil
+}
+
+func findEmailsAndOccurrencesInDir(rootPath string, username string) (Emails, error) {
+	emailLocations := make(map[string]map[string]bool)
+	emailRegex := regexp.MustCompile(`[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}`)
+	normalizedRootPath := filepath.Clean(rootPath)
+
+	err := filepath.WalkDir(rootPath, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !d.IsDir() {
+			if strings.Contains(path, ".git/logs/") {
+				return nil
+			}
+			content, err := os.ReadFile(path)
+			if err != nil {
+				return err
+			}
+
+			currentFileEmails := emailRegex.FindAllString(string(content), -1)
+			if len(currentFileEmails) > 0 {
+				relativePath, errRel := filepath.Rel(normalizedRootPath, path)
+				if errRel != nil {
+					relativePath = path
+				}
+
+				for _, email := range currentFileEmails {
+					if len(email) > 12 {
+						if _, ok := emailLocations[email]; !ok {
+							emailLocations[email] = make(map[string]bool)
+						}
+						emailLocations[email][relativePath] = true
+					}
+				}
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var results Emails
+	for email, pathSet := range emailLocations {
+		var paths []string
+		for path := range pathSet {
+			paths = append(paths, path)
+		}
+		results = append(results, Email{
+			Email: email, FoundIn: paths,
+			Levenshtein: utils.LevenshteinDistance(username, strings.SplitN(email, "@", 2)[0]),
+		})
+	}
+
+	return results, nil
+}

settings/settings.go

@@ -34,6 +34,7 @@ type Settings struct {
 	MaxDistance   int
 	HideAvatar    bool
 	SpoofEmail    bool
+	Trufflehog    bool
 
 	// Internal
 	Client *github.Client
@@ -93,6 +94,12 @@ func GetSettings() (settings Settings) {
 		20,
 		"Maximum Levenshtein distance for matching usernames & emails (only for deep scan)",
 	)
+	flag.BoolVar(
+		&settings.Trufflehog,
+		"trufflehog",
+		true,
+		"Run trufflehog on cloned repositories (only for deep scan)",
+	)
 
 	flag.BoolVarP(&settings.Silent, "silent", "S", false, "Suppress all non-essential output")
 	flag.BoolVarP(&settings.SpoofEmail, "spoof-email", "", true, "Spoof email (only for email mode)")