init
This commit is contained in:
Hadi
191
back/search/dataleak/dataleak.go
Normal file
191
back/search/dataleak/dataleak.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package dataleak
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/backend/server"
|
||||
"github.com/charmbracelet/log"
|
||||
)
|
||||
|
||||
type LeakResult struct {
|
||||
Duration time.Duration
|
||||
Rows []map[string]string
|
||||
Error string
|
||||
}
|
||||
|
||||
func Search(s *server.Server, queryText, column string, exactMatch bool) LeakResult {
|
||||
if len(*(s.Dataleaks)) == 0 {
|
||||
return LeakResult{
|
||||
Error: "No dataleak configured",
|
||||
}
|
||||
}
|
||||
now := time.Now()
|
||||
result := LeakResult{}
|
||||
|
||||
sqlQuery := buildSqlQuery(s, queryText, column, exactMatch)
|
||||
|
||||
if s.Settings.Debug {
|
||||
log.Info("New query:", "query", sqlQuery)
|
||||
}
|
||||
rows, err := s.Duckdb.Query(sqlQuery)
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
rawResult := make([][]byte, len(cols))
|
||||
dest := make([]any, len(cols))
|
||||
for i := range rawResult {
|
||||
dest[i] = &rawResult[i]
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(dest...)
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
rowMap := make(map[string]string)
|
||||
for i, colName := range cols {
|
||||
if rawResult[i] == nil || colName == "" {
|
||||
continue
|
||||
}
|
||||
if colName == "filename" {
|
||||
rowMap["source"] = server.FormatParquetName(string(rawResult[i]))
|
||||
continue
|
||||
}
|
||||
rowMap[colName] = string(rawResult[i])
|
||||
}
|
||||
result.Rows = append(result.Rows, rowMap)
|
||||
}
|
||||
|
||||
if err = rows.Err(); err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
result.Rows = removeDuplicateMaps(result.Rows)
|
||||
|
||||
result.Duration = time.Since(now)
|
||||
return result
|
||||
}
|
||||
|
||||
func removeDuplicateMaps(maps []map[string]string) []map[string]string {
|
||||
seen := make(map[string]struct{})
|
||||
result := []map[string]string{}
|
||||
|
||||
for _, m := range maps {
|
||||
// Create a unique key for the map by concatenating its key-value pairs
|
||||
var sb strings.Builder
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
slices.Sort(keys) // Sort keys to ensure consistent order
|
||||
for _, k := range keys {
|
||||
sb.WriteString(k)
|
||||
sb.WriteString("=")
|
||||
sb.WriteString(m[k])
|
||||
sb.WriteString(";")
|
||||
}
|
||||
key := sb.String()
|
||||
|
||||
if _, exists := seen[key]; !exists {
|
||||
seen[key] = struct{}{}
|
||||
result = append(result, m)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func buildSqlQuery(s *server.Server, queryText, column string, exactMatch bool) string {
|
||||
limit := strconv.Itoa(s.Settings.Limit)
|
||||
from := getFromClause(s)
|
||||
if column == "name" {
|
||||
column = "full_name"
|
||||
}
|
||||
columns := []string{column}
|
||||
if column == "all" || column == "" {
|
||||
columns = s.Settings.BaseColumns
|
||||
}
|
||||
columnsFiltered := []string{}
|
||||
allColumns := []string{}
|
||||
// TODO: Add columns that ends with _col aswell
|
||||
for _, dataleak := range *s.Dataleaks {
|
||||
for _, col := range dataleak.Columns {
|
||||
if !slices.Contains(allColumns, col) {
|
||||
allColumns = append(allColumns, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
if column == "full_text" {
|
||||
columnsFiltered = allColumns
|
||||
} else {
|
||||
for _, col := range columns {
|
||||
if slices.Contains(allColumns, col) {
|
||||
columnsFiltered = append(columnsFiltered, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(columnsFiltered) == 0 {
|
||||
return fmt.Sprintf("SELECT * FROM %s LIMIT %s", from, limit)
|
||||
}
|
||||
|
||||
where := getWhereClause(queryText, columnsFiltered, exactMatch)
|
||||
return fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT %s", from, where, limit)
|
||||
}
|
||||
|
||||
func getWhereClause(queryText string, columns []string, exactMatch bool) string {
|
||||
terms := strings.Fields(queryText)
|
||||
var andClauses []string
|
||||
|
||||
for _, term := range terms {
|
||||
var orClausesForTerm []string
|
||||
termEscaped := strings.ReplaceAll(term, "'", "''")
|
||||
|
||||
for _, col := range columns {
|
||||
if exactMatch {
|
||||
termEscapedILike := strings.ReplaceAll(termEscaped, "_", "\\_")
|
||||
termEscapedILike = strings.ReplaceAll(termEscapedILike, "%", "\\%")
|
||||
orClausesForTerm = append(orClausesForTerm, fmt.Sprintf("\"%s\" ILIKE '%s' ESCAPE '\\'", col, strings.ToLower(termEscapedILike)))
|
||||
} else {
|
||||
// Escape characters for ILIKE
|
||||
termEscapedILike := strings.ReplaceAll(termEscaped, "_", "\\_")
|
||||
termEscapedILike = strings.ReplaceAll(termEscapedILike, "%", "\\%")
|
||||
orClausesForTerm = append(orClausesForTerm, fmt.Sprintf("\"%s\" ILIKE '%%%s%%' ESCAPE '\\'", col, strings.ToLower(termEscapedILike)))
|
||||
}
|
||||
}
|
||||
andClauses = append(andClauses, "("+strings.Join(orClausesForTerm, " OR ")+")")
|
||||
}
|
||||
return strings.Join(andClauses, " AND ")
|
||||
}
|
||||
|
||||
func getFromClause(s *server.Server) string {
|
||||
parquets := []string{}
|
||||
for _, dataleak := range *s.Dataleaks {
|
||||
parquets = append(parquets, "'"+dataleak.Path+"'")
|
||||
}
|
||||
return fmt.Sprintf("read_parquet([%s], union_by_name=true, filename=true)", strings.Join(parquets, ", "))
|
||||
}
|
||||
|
||||
func castAllColumns(cols []string) []string {
|
||||
casted := make([]string, len(cols))
|
||||
for i, col := range cols {
|
||||
casted[i] = fmt.Sprintf("cast(%s as text)", col)
|
||||
}
|
||||
return casted
|
||||
}
|
||||
90
back/search/osint/github.go
Normal file
90
back/search/osint/github.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package osint
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/backend/server"
|
||||
|
||||
recon_email "github.com/anotherhadi/github-recon/github-recon/email"
|
||||
recon_username "github.com/anotherhadi/github-recon/github-recon/username"
|
||||
github_recon_settings "github.com/anotherhadi/github-recon/settings"
|
||||
)
|
||||
|
||||
type GithubResult struct {
|
||||
Duration time.Duration
|
||||
Error string
|
||||
|
||||
UsernameResult *recon_username.UsernameResult
|
||||
EmailResult *recon_email.EmailResult
|
||||
}
|
||||
|
||||
func Search(s *server.Server, queryText, column string) *GithubResult {
|
||||
if !s.Settings.GithubRecon {
|
||||
return nil
|
||||
}
|
||||
gr := GithubResult{}
|
||||
now := time.Now()
|
||||
settings := github_recon_settings.GetDefaultSettings()
|
||||
settings.Token = s.Settings.GithubToken
|
||||
settings.DeepScan = s.Settings.GithubDeepMode
|
||||
if settings.Token != "null" && strings.TrimSpace(settings.Token) != "" {
|
||||
settings.Client = settings.Client.WithAuthToken(settings.Token)
|
||||
}
|
||||
settings.Silent = true
|
||||
|
||||
queryText = strings.TrimSpace(queryText)
|
||||
|
||||
if column == "email" || strings.HasSuffix(column, "_email") ||
|
||||
column == "username" || strings.HasSuffix(column, "_username") ||
|
||||
column == "" || column == "all" {
|
||||
if isValidEmail(queryText) {
|
||||
settings.Target = queryText
|
||||
settings.TargetType = github_recon_settings.TargetEmail
|
||||
result := recon_email.Email(settings)
|
||||
gr.EmailResult = &result
|
||||
} else if isValidUsername(queryText) {
|
||||
settings.Target = queryText
|
||||
settings.TargetType = github_recon_settings.TargetUsername
|
||||
result, err := recon_username.Username(settings)
|
||||
if err != nil {
|
||||
gr.Error = err.Error()
|
||||
}
|
||||
if result.User.Username == "" {
|
||||
gr.UsernameResult = nil
|
||||
} else {
|
||||
gr.UsernameResult = &result
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
|
||||
gr.Duration = time.Since(now)
|
||||
return &gr
|
||||
}
|
||||
|
||||
func isValidEmail(email string) bool {
|
||||
if !strings.Contains(email, "@") || !strings.Contains(email, ".") {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(email, "@") || strings.HasSuffix(email, "@") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(email, " ") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isValidUsername(username string) bool {
|
||||
if len(username) < 1 || len(username) > 39 {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(username, " ") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
68
back/search/search.go
Normal file
68
back/search/search.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/backend/search/dataleak"
|
||||
"github.com/anotherhadi/eleakxir/backend/search/osint"
|
||||
"github.com/anotherhadi/eleakxir/backend/server"
|
||||
)
|
||||
|
||||
type Query struct {
|
||||
Text string
|
||||
Column string // The column to search in (e.g., "email", "password", etc.
|
||||
ExactMatch bool // Whether to search for an exact match
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Id string
|
||||
Date time.Time
|
||||
Status string // "pending", "completed"
|
||||
Query Query
|
||||
|
||||
LeakResult dataleak.LeakResult
|
||||
GithubResult osint.GithubResult
|
||||
}
|
||||
|
||||
func Search(s *server.Server, q Query, r *Result, mu *sync.RWMutex) {
|
||||
var wg sync.WaitGroup
|
||||
|
||||
mu.Lock()
|
||||
r.Date = time.Now()
|
||||
r.Status = "pending"
|
||||
r.Query = q
|
||||
mu.Unlock()
|
||||
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
leakResult := dataleak.Search(s, q.Text, q.Column, q.ExactMatch)
|
||||
mu.Lock()
|
||||
r.LeakResult = leakResult
|
||||
mu.Unlock()
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
githubResult := osint.Search(s, q.Text, q.Column)
|
||||
mu.Lock()
|
||||
r.GithubResult = *githubResult
|
||||
mu.Unlock()
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
mu.Lock()
|
||||
r.Status = "completed"
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
func EncodeQueryID(q Query, dataleaksCount uint64) string {
|
||||
raw, _ := json.Marshal(q)
|
||||
return fmt.Sprintf("%d:%s", dataleaksCount, base64.URLEncoding.EncodeToString(raw))
|
||||
}
|
||||
Reference in New Issue
Block a user