From c84b2b0458edfcf77e0419918c82ffd43cfe5059 Mon Sep 17 00:00:00 2001 From: Hadi <112569860+anotherhadi@users.noreply.github.com> Date: Mon, 1 Sep 2025 15:17:05 +0200 Subject: [PATCH] Trufflehog integration #11 Signed-off-by: Hadi <112569860+anotherhadi@users.noreply.github.com> --- README.md | 1 + cmd/print.go | 40 ++++++- github-recon/username/deep.go | 216 +++++++++++++++++++++++----------- settings/settings.go | 7 ++ 4 files changed, 191 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index f11e2eb..c158c04 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ github-recon [--flags value] target_username_or_email -r, --refresh Refresh the cache (only for deep scan) -s, --show-source Show where the information (authors, emails, etc) were found (only for deep scan) -m, --max-distance int Maximum Levenshtein distance for matching usernames & emails (only for deep scan) (default 20) + --trufflehog Run trufflehog on cloned repositories (only for deep scan) (default true) -S, --silent Suppress all non-essential output --spoof-email Spoof email (only for email mode) (default true) -a, --hide-avatar Hide the avatar in the output diff --git a/cmd/print.go b/cmd/print.go index 3b78db9..a0d7918 100644 --- a/cmd/print.go +++ b/cmd/print.go @@ -3,6 +3,7 @@ package main import ( "fmt" "reflect" + "sort" "strings" github_recon_settings "github.com/anotherhadi/github-recon/settings" @@ -27,11 +28,17 @@ func printStruct(settings github_recon_settings.Settings, s any, indent int) { prefix := strings.Repeat(" ", indent) v := reflect.ValueOf(s) + if !v.IsValid() { + return + } t := reflect.TypeOf(s) - if v.Kind() == reflect.Ptr { + for v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface { + if v.IsNil() { + return + } v = v.Elem() - t = t.Elem() + t = v.Type() } switch v.Kind() { @@ -43,7 +50,6 @@ func printStruct(settings github_recon_settings.Settings, s any, indent int) { } printed := 0 - for i := 0; i < v.NumField(); i++ { field := t.Field(i).Name value := v.Field(i) @@ -60,11 +66,13 @@ func printStruct(settings github_recon_settings.Settings, s any, indent int) { printed++ switch value.Kind() { - case reflect.Struct, reflect.Slice, reflect.Array, reflect.Ptr: + case reflect.Struct, reflect.Slice, reflect.Array, reflect.Ptr, reflect.Map, reflect.Interface: fmt.Println(prefix + greyStyle.Render(field+":")) printStruct(settings, value.Interface(), indent+1) + case reflect.String: fmt.Printf("%s%s %s\n", prefix, greyStyle.Render(field+":"), greenStyle.Render(fmt.Sprintf("%q", value.Interface()))) + default: fmt.Printf("%s%s %s\n", prefix, greyStyle.Render(field+":"), greenStyle.Render(fmt.Sprintf("%v", value.Interface()))) } @@ -84,9 +92,31 @@ func printStruct(settings github_recon_settings.Settings, s any, indent int) { printStruct(settings, v.Index(i).Interface(), indent) } + case reflect.Map: + if v.Len() == 0 { + fmt.Println(prefix + greyStyle.Render("No data found")) + return + } + + keys := v.MapKeys() + keyStrs := make([]string, len(keys)) + for i, k := range keys { + keyStrs[i] = fmt.Sprintf("%v", k.Interface()) + } + sort.Strings(keyStrs) + + for _, keyStr := range keyStrs { + for _, k := range keys { + if fmt.Sprintf("%v", k.Interface()) == keyStr { + val := v.MapIndex(k) + fmt.Println(prefix + greyStyle.Render(fmt.Sprintf("%v:", k.Interface()))) + printStruct(settings, val.Interface(), indent+1) + } + } + } + default: fmt.Println(prefix + greenStyle.Render(fmt.Sprintf("%v", v.Interface()))) - fmt.Println("") } } diff --git a/github-recon/username/deep.go b/github-recon/username/deep.go index 27baeb3..a1278dd 100644 --- a/github-recon/username/deep.go +++ b/github-recon/username/deep.go @@ -1,6 +1,8 @@ package recon import ( + "encoding/json" + "fmt" "io/fs" "os" "os/exec" @@ -14,26 +16,34 @@ import ( "github.com/google/go-github/v72/github" ) -type Authors []AuthorOccurrence +type Authors []Author -type AuthorOccurrence struct { +type Author struct { Name string Levenshtein int Email string FoundIn []string } -type Emails []EmailOccurrence +type Emails []Email -type EmailOccurrence struct { +type Email struct { Email string Levenshtein int FoundIn []string } +type Secrets []Secret + +type Secret struct { + Repositorie string + Raw map[string]any +} + type DeepScanResult struct { - Authors []AuthorOccurrence - Emails []EmailOccurrence + Authors Authors + Emails Emails + Secrets Secrets } type Repositorie struct { @@ -43,62 +53,6 @@ type Repositorie struct { Size int } -func findEmailsAndOccurrencesInDir(rootPath string, username string) (Emails, error) { - emailLocations := make(map[string]map[string]bool) - emailRegex := regexp.MustCompile(`[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}`) - normalizedRootPath := filepath.Clean(rootPath) - - err := filepath.WalkDir(rootPath, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if !d.IsDir() { - if strings.Contains(path, ".git/logs/") { - return nil - } - content, err := os.ReadFile(path) - if err != nil { - return err - } - - currentFileEmails := emailRegex.FindAllString(string(content), -1) - if len(currentFileEmails) > 0 { - relativePath, errRel := filepath.Rel(normalizedRootPath, path) - if errRel != nil { - relativePath = path - } - - for _, email := range currentFileEmails { - if len(email) > 12 { - if _, ok := emailLocations[email]; !ok { - emailLocations[email] = make(map[string]bool) - } - emailLocations[email][relativePath] = true - } - } - } - } - return nil - }) - if err != nil { - return nil, err - } - - var results []EmailOccurrence - for email, pathSet := range emailLocations { - var paths []string - for path := range pathSet { - paths = append(paths, path) - } - results = append(results, EmailOccurrence{ - Email: email, FoundIn: paths, - Levenshtein: utils.LevenshteinDistance(username, strings.SplitN(email, "@", 2)[0]), - }) - } - - return results, nil -} - func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) { repositories := []Repositorie{} repos, resp, err := s.Client.Repositories.ListByUser( @@ -185,7 +139,7 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) { } s.Logger.Info("Cloned all repositories", "path", tmp_folder) - authorOccurrences := []AuthorOccurrence{} + authorOccurrences := Authors{} mapAuthorToIndex := make(map[string]int) for _, repo := range repositories { destination := tmp_folder + "/" + repo.Owner + "/" + repo.Name @@ -244,7 +198,7 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) { continue } - authorOccurrences = append(authorOccurrences, AuthorOccurrence{ + authorOccurrences = append(authorOccurrences, Author{ Name: authorName, Email: authorEmail, FoundIn: []string{repoIdentifier}, @@ -256,14 +210,14 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) { } } } - slices.SortFunc(authorOccurrences, func(a, b AuthorOccurrence) int { + slices.SortFunc(authorOccurrences, func(a, b Author) int { if a.Levenshtein != b.Levenshtein { return a.Levenshtein - b.Levenshtein } return 1 }) - authors := []AuthorOccurrence{} + authors := Authors{} for _, author := range authorOccurrences { if author.Levenshtein > s.MaxDistance { continue @@ -280,14 +234,14 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) { s.Logger.Error("Failed to find emails in directory", "err", err) return } - slices.SortFunc(emailsFound, func(a, b EmailOccurrence) int { + slices.SortFunc(emailsFound, func(a, b Email) int { if a.Levenshtein != b.Levenshtein { return a.Levenshtein - b.Levenshtein } return 1 }) - emails := []EmailOccurrence{} + emails := Emails{} for _, email := range emailsFound { if email.Levenshtein > s.MaxDistance { continue @@ -298,5 +252,131 @@ func DeepScan(s github_recon_settings.Settings) (response DeepScanResult) { response.Authors = authors response.Emails = emails + s.Logger.Info("Searching for secrets in cloned repositories", "path", tmp_folder) + if s.Trufflehog { + cmd := exec.Command("trufflehog", "--version") + if err := cmd.Run(); err != nil { + s.Logger.Warn("Trufflehog is not installed, skipping secret scanning.") + } else { + secrets, err := truffleHog(tmp_folder) + if err != nil { + s.Logger.Error("Failed to run trufflehog", "err", err) + } else { + response.Secrets = secrets + } + } + } + return } + +func truffleHog(tmpFolder string) (Secrets, error) { + allSecrets := Secrets{} + + directories, err := os.ReadDir(tmpFolder) + if err != nil { + return nil, fmt.Errorf("failed to read tmp folder: %w", err) + } + + for _, dir := range directories { + if !dir.IsDir() { + continue + } + + innerPath := filepath.Join(tmpFolder, dir.Name()) + innerDirectories, err := os.ReadDir(innerPath) + if err != nil { + return nil, fmt.Errorf("failed to read inner tmp folder: %w", err) + } + + for _, innerDir := range innerDirectories { + if !innerDir.IsDir() { + continue + } + + repoPath := filepath.Join(innerPath, innerDir.Name()) + cmd := exec.Command("trufflehog", "git", "file://"+repoPath, "--json", "--log-level=-1", "--results=verified") + output, err := cmd.Output() + + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + if exitErr.ExitCode() > 1 { + return nil, fmt.Errorf("failed to execute trufflehog (ExitError): %s", string(exitErr.Stderr)) + } + } else { + return nil, fmt.Errorf("failed to execute trufflehog: %w", err) + } + } + + decoder := json.NewDecoder(strings.NewReader(string(output))) + for decoder.More() { + var result map[string]any + if err := decoder.Decode(&result); err != nil { + return nil, fmt.Errorf("failed to parse trufflehog output: %w", err) + } + allSecrets = append(allSecrets, Secret{ + Repositorie: dir.Name() + "/" + innerDir.Name(), + Raw: result, + }) + } + } + } + + return allSecrets, nil +} + +func findEmailsAndOccurrencesInDir(rootPath string, username string) (Emails, error) { + emailLocations := make(map[string]map[string]bool) + emailRegex := regexp.MustCompile(`[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}`) + normalizedRootPath := filepath.Clean(rootPath) + + err := filepath.WalkDir(rootPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + if strings.Contains(path, ".git/logs/") { + return nil + } + content, err := os.ReadFile(path) + if err != nil { + return err + } + + currentFileEmails := emailRegex.FindAllString(string(content), -1) + if len(currentFileEmails) > 0 { + relativePath, errRel := filepath.Rel(normalizedRootPath, path) + if errRel != nil { + relativePath = path + } + + for _, email := range currentFileEmails { + if len(email) > 12 { + if _, ok := emailLocations[email]; !ok { + emailLocations[email] = make(map[string]bool) + } + emailLocations[email][relativePath] = true + } + } + } + } + return nil + }) + if err != nil { + return nil, err + } + + var results Emails + for email, pathSet := range emailLocations { + var paths []string + for path := range pathSet { + paths = append(paths, path) + } + results = append(results, Email{ + Email: email, FoundIn: paths, + Levenshtein: utils.LevenshteinDistance(username, strings.SplitN(email, "@", 2)[0]), + }) + } + + return results, nil +} diff --git a/settings/settings.go b/settings/settings.go index f53452a..b8d47e6 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -34,6 +34,7 @@ type Settings struct { MaxDistance int HideAvatar bool SpoofEmail bool + Trufflehog bool // Internal Client *github.Client @@ -93,6 +94,12 @@ func GetSettings() (settings Settings) { 20, "Maximum Levenshtein distance for matching usernames & emails (only for deep scan)", ) + flag.BoolVar( + &settings.Trufflehog, + "trufflehog", + true, + "Run trufflehog on cloned repositories (only for deep scan)", + ) flag.BoolVarP(&settings.Silent, "silent", "S", false, "Suppress all non-essential output") flag.BoolVarP(&settings.SpoofEmail, "spoof-email", "", true, "Spoof email (only for email mode)")