This commit is contained in:
Hadi
2025-09-24 17:20:03 +02:00
commit b9fbed9a54
83 changed files with 6241 additions and 0 deletions

21
back/.air.toml Normal file
View File

@@ -0,0 +1,21 @@
root = "."
tmp_dir = "tmp"
[build]
cmd = "go build -o ./tmp/main ./cmd/main.go"
full_bin = "DEBUG=true DATALEAKS_FOLDERS=../testdata ./tmp/main"
args = []
exclude_dir = ["tmp", "vendor"]
exclude_file = []
exclude_regex = ["_test.go"]
exclude_unchanged = false
follow_symlink = false
send_interrupt = false
delay = 1000 # ms
stop_on_error = true
[log]
time = true
[misc]
clean_on_exit = true

1
back/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
tmp/

174
back/api/api.go Normal file
View File

@@ -0,0 +1,174 @@
package api
import (
"net/http"
"strings"
"time"
"github.com/anotherhadi/eleakxir/backend/search"
"github.com/anotherhadi/eleakxir/backend/server"
"github.com/gin-gonic/gin"
)
// TODO: We need to know when we hit the LIMIT
func routes(s *server.Server, cache *map[string]*search.Result) {
s.Router.Use(
func(c *gin.Context) {
if s.Settings.Password != "" {
password := c.GetHeader("X-Password")
if password != s.Settings.Password {
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
return
}
}
c.Next()
},
)
s.Router.GET("/", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{
"Settings": s.Settings,
"Dataleaks": s.Dataleaks,
"TotalDataleaks": s.TotalDataleaks,
"TotalRows": s.TotalRows,
"TotalSize": s.TotalSize,
})
})
s.Router.GET("/history", func(c *gin.Context) {
type historyItem struct {
Id string
Status string
Date time.Time
Query search.Query
Results int
}
var history []historyItem
s.Mu.RLock()
for _, r := range *cache {
history = append(history, historyItem{
Id: r.Id,
Status: r.Status,
Date: r.Date,
Query: r.Query,
Results: len(r.LeakResult.Rows),
})
}
s.Mu.RUnlock()
for i := 0; i < len(history)-1; i++ {
for j := 0; j < len(history)-i-1; j++ {
if history[j].Date.Before(history[j+1].Date) {
history[j], history[j+1] = history[j+1], history[j]
}
}
}
c.JSON(http.StatusOK, gin.H{
"History": history,
})
})
s.Router.POST("/search", func(c *gin.Context) {
var query search.Query
if err := c.BindJSON(&query); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"Error": "invalid JSON"})
return
}
query = cleanQuery(query)
if len(query.Text) <= s.Settings.MinimumQueryLength {
c.JSON(http.StatusBadRequest, gin.H{"Error": "query too short"})
return
}
id := search.EncodeQueryID(query, *s.TotalDataleaks)
s.Mu.RLock()
_, exists := (*cache)[id]
s.Mu.RUnlock()
if exists {
c.JSON(http.StatusOK, gin.H{"Id": id})
return
}
r := search.Result{
Id: id,
}
go search.Search(s, query, &r, s.Mu)
s.Mu.Lock()
(*cache)[id] = &r
s.Mu.Unlock()
c.JSON(http.StatusOK, gin.H{"Id": id})
})
s.Router.GET("/search/:id", func(c *gin.Context) {
id := c.Param("id")
s.Mu.RLock()
r, exists := (*cache)[id]
s.Mu.RUnlock()
if !exists {
c.JSON(http.StatusNotFound, gin.H{"Error": "not found"})
return
}
c.JSON(http.StatusOK, r)
})
}
func Init(s *server.Server) {
if !s.Settings.Debug {
gin.SetMode(gin.ReleaseMode)
}
s.Router = gin.Default()
s.Router.Use(CORSMiddleware())
cache := make(map[string]*search.Result)
go func() {
for {
time.Sleep(time.Minute)
deleteOldCache(s, &cache)
}
}()
routes(s, &cache)
}
func deleteOldCache(s *server.Server, cache *map[string]*search.Result) {
s.Mu.Lock()
defer s.Mu.Unlock()
now := time.Now()
for id, r := range *cache {
if now.Sub(r.Date) > s.Settings.MaxCacheDuration {
delete(*cache, id)
}
}
}
func cleanQuery(q search.Query) search.Query {
q.Column = strings.ToLower(strings.TrimSpace(q.Column))
q.Column = strings.Join(strings.Fields(q.Column), " ")
q.Column = strings.ReplaceAll(q.Column, "`", "")
q.Column = strings.ReplaceAll(q.Column, "'", "")
q.Column = strings.ReplaceAll(q.Column, "-", "_")
q.Column = strings.ReplaceAll(q.Column, " ", "_")
q.Column = strings.ReplaceAll(q.Column, "\"", "")
q.Text = strings.TrimSpace(q.Text)
q.Text = strings.Join(strings.Fields(q.Text), " ")
return q
}
func CORSMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
c.Writer.Header().
Set("Access-Control-Allow-Headers", "X-Password, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
if c.Request.Method == "OPTIONS" {
c.AbortWithStatus(204)
return
}
c.Next()
}
}

21
back/cmd/main.go Normal file
View File

@@ -0,0 +1,21 @@
package main
import (
"fmt"
"strconv"
"github.com/anotherhadi/eleakxir/backend/api"
"github.com/anotherhadi/eleakxir/backend/server"
)
func main() {
server := server.NewServer()
fmt.Println("Starting the server.")
api.Init(server)
err := server.Router.Run(":" + strconv.Itoa(server.Settings.Port))
if err != nil {
panic(err)
}
}

70
back/go.mod Normal file
View File

@@ -0,0 +1,70 @@
module github.com/anotherhadi/eleakxir/backend
go 1.25.0
require (
github.com/anotherhadi/github-recon v1.5.6
github.com/charmbracelet/log v0.4.2
github.com/gin-gonic/gin v1.10.1
github.com/marcboeker/go-duckdb v1.8.5
)
require (
github.com/apache/arrow-go/v18 v18.1.0 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/bytedance/sonic v1.11.6 // indirect
github.com/bytedance/sonic/loader v0.1.1 // indirect
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
github.com/charmbracelet/lipgloss v1.1.0 // indirect
github.com/charmbracelet/x/ansi v0.8.0 // indirect
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/cloudwego/base64x v0.1.4 // indirect
github.com/cloudwego/iasm v0.2.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.20.0 // indirect
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
github.com/goccy/go-json v0.10.5 // indirect
github.com/google/flatbuffers v25.1.24+incompatible // indirect
github.com/google/go-github/v72 v72.0.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pierrec/lz4/v4 v4.1.22 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/saran13raj/go-pixels v0.0.0-20250629121333-58b240a3ae51 // indirect
github.com/spf13/pflag v1.0.7 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
golang.org/x/arch v0.8.0 // indirect
golang.org/x/crypto v0.38.0 // indirect
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect
golang.org/x/image v0.28.0 // indirect
golang.org/x/mod v0.25.0 // indirect
golang.org/x/net v0.40.0 // indirect
golang.org/x/sync v0.15.0 // indirect
golang.org/x/sys v0.33.0 // indirect
golang.org/x/text v0.26.0 // indirect
golang.org/x/tools v0.33.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/protobuf v1.36.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

179
back/go.sum Normal file
View File

@@ -0,0 +1,179 @@
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
github.com/anotherhadi/github-recon v1.5.6 h1:IN3lQZRqqNbPpSyP5fvNoJrYODbM2tNwS5tiRgD+i1s=
github.com/anotherhadi/github-recon v1.5.6/go.mod h1:E2tmCmjEZdJeBx8u1J8sSMtnmU8aDQ6IjCoq3ykoHtY=
github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y=
github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0=
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs=
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
github.com/charmbracelet/log v0.4.2 h1:hYt8Qj6a8yLnvR+h7MwsJv/XvmBJXiueUcI3cIxsyig=
github.com/charmbracelet/log v0.4.2/go.mod h1:qifHGX/tc7eluv2R6pWIpyHDDrrb/AG71Pf2ysQu5nw=
github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE=
github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q=
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8=
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o=
github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM=
github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/marcboeker/go-duckdb v1.8.5 h1:tkYp+TANippy0DaIOP5OEfBEwbUINqiFqgwMQ44jME0=
github.com/marcboeker/go-duckdb v1.8.5/go.mod h1:6mK7+WQE4P4u5AFLvVBmhFxY5fvhymFptghgJX6B+/8=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/saran13raj/go-pixels v0.0.0-20250629121333-58b240a3ae51 h1:H/XUfYcLxI3CBmDlgBpnOeTntRgqWvIoUXnqhCF5a0s=
github.com/saran13raj/go-pixels v0.0.0-20250629121333-58b240a3ae51/go.mod h1:sqhdZVLvqzTEBtmZBuTnFDUW0Lsryw2X2/wrLgqLEYg=
github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M=
github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc=
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE=
golang.org/x/image v0.28.0/go.mod h1:GUJYXtnGKEUgggyzh+Vxt+AviiCcyiwpsl8iQ8MvwGY=
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0=
gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o=
google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=

View File

@@ -0,0 +1,191 @@
package dataleak
import (
"fmt"
"slices"
"strconv"
"strings"
"time"
"github.com/anotherhadi/eleakxir/backend/server"
"github.com/charmbracelet/log"
)
type LeakResult struct {
Duration time.Duration
Rows []map[string]string
Error string
}
func Search(s *server.Server, queryText, column string, exactMatch bool) LeakResult {
if len(*(s.Dataleaks)) == 0 {
return LeakResult{
Error: "No dataleak configured",
}
}
now := time.Now()
result := LeakResult{}
sqlQuery := buildSqlQuery(s, queryText, column, exactMatch)
if s.Settings.Debug {
log.Info("New query:", "query", sqlQuery)
}
rows, err := s.Duckdb.Query(sqlQuery)
if err != nil {
result.Error = err.Error()
return result
}
defer rows.Close()
cols, err := rows.Columns()
if err != nil {
result.Error = err.Error()
return result
}
rawResult := make([][]byte, len(cols))
dest := make([]any, len(cols))
for i := range rawResult {
dest[i] = &rawResult[i]
}
for rows.Next() {
err := rows.Scan(dest...)
if err != nil {
result.Error = err.Error()
return result
}
rowMap := make(map[string]string)
for i, colName := range cols {
if rawResult[i] == nil || colName == "" {
continue
}
if colName == "filename" {
rowMap["source"] = server.FormatParquetName(string(rawResult[i]))
continue
}
rowMap[colName] = string(rawResult[i])
}
result.Rows = append(result.Rows, rowMap)
}
if err = rows.Err(); err != nil {
result.Error = err.Error()
return result
}
result.Rows = removeDuplicateMaps(result.Rows)
result.Duration = time.Since(now)
return result
}
func removeDuplicateMaps(maps []map[string]string) []map[string]string {
seen := make(map[string]struct{})
result := []map[string]string{}
for _, m := range maps {
// Create a unique key for the map by concatenating its key-value pairs
var sb strings.Builder
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
slices.Sort(keys) // Sort keys to ensure consistent order
for _, k := range keys {
sb.WriteString(k)
sb.WriteString("=")
sb.WriteString(m[k])
sb.WriteString(";")
}
key := sb.String()
if _, exists := seen[key]; !exists {
seen[key] = struct{}{}
result = append(result, m)
}
}
return result
}
func buildSqlQuery(s *server.Server, queryText, column string, exactMatch bool) string {
limit := strconv.Itoa(s.Settings.Limit)
from := getFromClause(s)
if column == "name" {
column = "full_name"
}
columns := []string{column}
if column == "all" || column == "" {
columns = s.Settings.BaseColumns
}
columnsFiltered := []string{}
allColumns := []string{}
// TODO: Add columns that ends with _col aswell
for _, dataleak := range *s.Dataleaks {
for _, col := range dataleak.Columns {
if !slices.Contains(allColumns, col) {
allColumns = append(allColumns, col)
}
}
}
if column == "full_text" {
columnsFiltered = allColumns
} else {
for _, col := range columns {
if slices.Contains(allColumns, col) {
columnsFiltered = append(columnsFiltered, col)
}
}
}
if len(columnsFiltered) == 0 {
return fmt.Sprintf("SELECT * FROM %s LIMIT %s", from, limit)
}
where := getWhereClause(queryText, columnsFiltered, exactMatch)
return fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT %s", from, where, limit)
}
func getWhereClause(queryText string, columns []string, exactMatch bool) string {
terms := strings.Fields(queryText)
var andClauses []string
for _, term := range terms {
var orClausesForTerm []string
termEscaped := strings.ReplaceAll(term, "'", "''")
for _, col := range columns {
if exactMatch {
termEscapedILike := strings.ReplaceAll(termEscaped, "_", "\\_")
termEscapedILike = strings.ReplaceAll(termEscapedILike, "%", "\\%")
orClausesForTerm = append(orClausesForTerm, fmt.Sprintf("\"%s\" ILIKE '%s' ESCAPE '\\'", col, strings.ToLower(termEscapedILike)))
} else {
// Escape characters for ILIKE
termEscapedILike := strings.ReplaceAll(termEscaped, "_", "\\_")
termEscapedILike = strings.ReplaceAll(termEscapedILike, "%", "\\%")
orClausesForTerm = append(orClausesForTerm, fmt.Sprintf("\"%s\" ILIKE '%%%s%%' ESCAPE '\\'", col, strings.ToLower(termEscapedILike)))
}
}
andClauses = append(andClauses, "("+strings.Join(orClausesForTerm, " OR ")+")")
}
return strings.Join(andClauses, " AND ")
}
func getFromClause(s *server.Server) string {
parquets := []string{}
for _, dataleak := range *s.Dataleaks {
parquets = append(parquets, "'"+dataleak.Path+"'")
}
return fmt.Sprintf("read_parquet([%s], union_by_name=true, filename=true)", strings.Join(parquets, ", "))
}
func castAllColumns(cols []string) []string {
casted := make([]string, len(cols))
for i, col := range cols {
casted[i] = fmt.Sprintf("cast(%s as text)", col)
}
return casted
}

View File

@@ -0,0 +1,90 @@
package osint
import (
"strings"
"time"
"github.com/anotherhadi/eleakxir/backend/server"
recon_email "github.com/anotherhadi/github-recon/github-recon/email"
recon_username "github.com/anotherhadi/github-recon/github-recon/username"
github_recon_settings "github.com/anotherhadi/github-recon/settings"
)
type GithubResult struct {
Duration time.Duration
Error string
UsernameResult *recon_username.UsernameResult
EmailResult *recon_email.EmailResult
}
func Search(s *server.Server, queryText, column string) *GithubResult {
if !s.Settings.GithubRecon {
return nil
}
gr := GithubResult{}
now := time.Now()
settings := github_recon_settings.GetDefaultSettings()
settings.Token = s.Settings.GithubToken
settings.DeepScan = s.Settings.GithubDeepMode
if settings.Token != "null" && strings.TrimSpace(settings.Token) != "" {
settings.Client = settings.Client.WithAuthToken(settings.Token)
}
settings.Silent = true
queryText = strings.TrimSpace(queryText)
if column == "email" || strings.HasSuffix(column, "_email") ||
column == "username" || strings.HasSuffix(column, "_username") ||
column == "" || column == "all" {
if isValidEmail(queryText) {
settings.Target = queryText
settings.TargetType = github_recon_settings.TargetEmail
result := recon_email.Email(settings)
gr.EmailResult = &result
} else if isValidUsername(queryText) {
settings.Target = queryText
settings.TargetType = github_recon_settings.TargetUsername
result, err := recon_username.Username(settings)
if err != nil {
gr.Error = err.Error()
}
if result.User.Username == "" {
gr.UsernameResult = nil
} else {
gr.UsernameResult = &result
}
} else {
return nil
}
} else {
return nil
}
gr.Duration = time.Since(now)
return &gr
}
func isValidEmail(email string) bool {
if !strings.Contains(email, "@") || !strings.Contains(email, ".") {
return false
}
if strings.HasPrefix(email, "@") || strings.HasSuffix(email, "@") {
return false
}
if strings.Contains(email, " ") {
return false
}
return true
}
func isValidUsername(username string) bool {
if len(username) < 1 || len(username) > 39 {
return false
}
if strings.Contains(username, " ") {
return false
}
return true
}

68
back/search/search.go Normal file
View File

@@ -0,0 +1,68 @@
package search
import (
"encoding/base64"
"encoding/json"
"fmt"
"sync"
"time"
"github.com/anotherhadi/eleakxir/backend/search/dataleak"
"github.com/anotherhadi/eleakxir/backend/search/osint"
"github.com/anotherhadi/eleakxir/backend/server"
)
type Query struct {
Text string
Column string // The column to search in (e.g., "email", "password", etc.
ExactMatch bool // Whether to search for an exact match
}
type Result struct {
Id string
Date time.Time
Status string // "pending", "completed"
Query Query
LeakResult dataleak.LeakResult
GithubResult osint.GithubResult
}
func Search(s *server.Server, q Query, r *Result, mu *sync.RWMutex) {
var wg sync.WaitGroup
mu.Lock()
r.Date = time.Now()
r.Status = "pending"
r.Query = q
mu.Unlock()
wg.Add(2)
go func() {
leakResult := dataleak.Search(s, q.Text, q.Column, q.ExactMatch)
mu.Lock()
r.LeakResult = leakResult
mu.Unlock()
wg.Done()
}()
go func() {
githubResult := osint.Search(s, q.Text, q.Column)
mu.Lock()
r.GithubResult = *githubResult
mu.Unlock()
wg.Done()
}()
wg.Wait()
mu.Lock()
r.Status = "completed"
mu.Unlock()
}
func EncodeQueryID(q Query, dataleaksCount uint64) string {
raw, _ := json.Marshal(q)
return fmt.Sprintf("%d:%s", dataleaksCount, base64.URLEncoding.EncodeToString(raw))
}

111
back/server/dataleak.go Normal file
View File

@@ -0,0 +1,111 @@
package server
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"github.com/charmbracelet/log"
)
type Dataleak struct {
Path string
Name string
Columns []string
Length uint64
Size uint64
}
const CACHE_FILENAME = "dataleaks_cache.json"
// TODO: check os.FileInfo.ModTime() to see if the file has changed since last cache update
func Cache(s *Server) error {
if len(s.Settings.Folders) == 0 {
return nil
}
if s.Settings.CacheFolder == "" {
s.Settings.CacheFolder = s.Settings.Folders[0]
}
if err := createDirectoryIfNotExists(s.Settings.CacheFolder); err != nil {
return err
}
cacheFile := filepath.Join(s.Settings.CacheFolder, CACHE_FILENAME)
dataleaks := []Dataleak{}
data, err := os.ReadFile(cacheFile)
if err == nil {
if err := json.Unmarshal(data, &dataleaks); err != nil {
log.Warn("Failed to unmarshal dataleaks cache", "error", err)
}
} else {
log.Warn("Failed to read dataleaks cache file", "error", err)
}
// Filter out non-existent files
filteredDataleaks := []Dataleak{}
writeOutput := false
for _, d := range dataleaks {
if _, err := os.Stat(d.Path); err == nil {
filteredDataleaks = append(filteredDataleaks, d)
} else if os.IsNotExist(err) {
log.Info("Removing non-existent file from cache", "path", d.Path)
writeOutput = true
} else {
log.Error("Error checking file existence", "path", d.Path, "error", err)
}
}
dataleaks = filteredDataleaks
// Create a map for quick lookups
dataleakMap := make(map[string]struct{}, len(dataleaks))
for _, d := range dataleaks {
dataleakMap[d.Path] = struct{}{}
}
// Add new files
parquetFiles := getAllParquetFiles(s.Settings.Folders)
for _, p := range parquetFiles {
if _, found := dataleakMap[p]; found {
continue
}
writeOutput = true
dataleaks = append(dataleaks, getDataleak(*s, p))
}
if writeOutput {
data, err := json.MarshalIndent(dataleaks, "", " ")
if err != nil {
return fmt.Errorf("error marshalling cache: %w", err)
}
if err := os.WriteFile(cacheFile, data, 0644); err != nil {
return fmt.Errorf("error writing cache: %w", err)
}
}
s.Dataleaks = &dataleaks
totalDataleaks := uint64(len(dataleaks))
totalRows := uint64(0)
totalSize := uint64(0)
for _, d := range dataleaks {
totalRows += d.Length
totalSize += d.Size
}
s.TotalDataleaks = &totalDataleaks
s.TotalSize = &totalSize
s.TotalRows = &totalRows
return nil
}
func getDataleak(s Server, path string) Dataleak {
return Dataleak{
Path: path,
Name: FormatParquetName(path),
Columns: getParquetColumns(s, path),
Length: getParquetLength(s, path),
Size: getFileSize(path),
}
}

61
back/server/server.go Normal file
View File

@@ -0,0 +1,61 @@
package server
import (
"database/sql"
"sync"
"time"
"github.com/charmbracelet/log"
"github.com/gin-gonic/gin"
_ "github.com/marcboeker/go-duckdb"
)
type Server struct {
Settings ServerSettings
Dataleaks *[]Dataleak
TotalRows *uint64
TotalDataleaks *uint64
TotalSize *uint64 // MB
Router *gin.Engine
Duckdb *sql.DB
Mu *sync.RWMutex
}
func NewServer() *Server {
zero := uint64(0)
emptyDataleak := []Dataleak{}
s := &Server{
Settings: LoadServerSettings(),
Mu: &sync.RWMutex{},
TotalDataleaks: &zero,
TotalRows: &zero,
TotalSize: &zero,
Dataleaks: &emptyDataleak,
}
var err error
s.Duckdb, err = sql.Open("duckdb", "")
if err != nil {
panic(err)
}
err = Cache(s)
if err != nil {
panic(err)
}
go func() {
for {
time.Sleep(s.Settings.ReloadDataleaksInterval)
err := Cache(s)
if err != nil {
log.Error(err)
}
}
}()
return s
}

129
back/server/settings.go Normal file
View File

@@ -0,0 +1,129 @@
package server
import (
"os"
"strconv"
"strings"
"time"
github_recon_settings "github.com/anotherhadi/github-recon/settings"
)
type ServerSettings struct {
Port int `json:"-"` // Port to run the server on
Debug bool
Password string `json:"-"` // Do not expose the password in JSON
MinimumQueryLength int
MaxCacheDuration time.Duration // Delete a search from the cache after this duration
// Dataleaks
Folders []string // Folders to search in for parquets, recursive
CacheFolder string
BaseColumns []string // Use these columns when column="all"
Limit int // Limit number of rows returned
ReloadDataleaksInterval time.Duration // Reload dataleaks files from disk every X
// OSINT Tools
GithubRecon bool // Activate github-recon OSINT tool
GithubToken string `json:"-"` // Github token for github-recon
GithubTokenLoaded bool
GithubDeepMode bool // Deep mode for github-recon
}
func LoadServerSettings() ServerSettings {
ss := ServerSettings{
Port: getEnvPortOrDefault("PORT", 9198),
Debug: getEnvBoolOrDefault("DEBUG", false),
Password: getEnvStringOrDefault("PASSWORD", ""),
MinimumQueryLength: getEnvIntOrDefault("MINIMUM_QUERY_LENGTH", 3),
MaxCacheDuration: getEnvDurationOrDefault("MAX_CACHE_DURATION", 24*time.Hour),
// Dataleaks
Folders: getEnvStringListOrDefault("DATALEAKS_FOLDERS", []string{}),
CacheFolder: getEnvStringOrDefault("DATALEAKS_CACHE_FOLDER", ""),
BaseColumns: getEnvStringListOrDefault("BASE_COLUMNS", []string{"email", "username", "password", "full_name", "phone", "url"}),
Limit: getEnvIntOrDefault("LIMIT", 100),
ReloadDataleaksInterval: getEnvDurationOrDefault("RELOAD_DATALEAKS_INTERVAL", 20*time.Minute),
// OSINT Tools
GithubRecon: getEnvBoolOrDefault("GITHUB_RECON", true),
GithubToken: getEnvStringOrDefault("GITHUB_TOKEN", "null"),
GithubDeepMode: getEnvBoolOrDefault("GITHUB_DEEP_MODE", false),
}
if ss.GithubToken == "null" || strings.TrimSpace(ss.GithubToken) == "" {
ss.GithubToken = github_recon_settings.GetToken()
}
if ss.GithubToken != "null" && strings.TrimSpace(ss.GithubToken) != "" {
ss.GithubTokenLoaded = true
}
return ss
}
func getEnvStringOrDefault(envKey, defaultValue string) string {
value := strings.TrimSpace(os.Getenv(envKey))
if value == "" {
return defaultValue
}
return value
}
func getEnvBoolOrDefault(envKey string, defaultValue bool) bool {
value := strings.TrimSpace(os.Getenv(envKey))
if value == "" {
return defaultValue
}
value = strings.ToLower(value)
if value == "true" || value == "1" {
return true
} else if value == "false" || value == "0" {
return false
}
return defaultValue
}
func getEnvDurationOrDefault(envKey string, defaultValue time.Duration) time.Duration {
v := getEnvStringOrDefault(envKey, "")
if v == "" {
return defaultValue
}
t, err := time.ParseDuration(v)
if err != nil {
return defaultValue
}
return t
}
func getEnvStringListOrDefault(envKey string, defaultValue []string) []string {
value := strings.TrimSpace(os.Getenv(envKey))
if value == "" {
return defaultValue
}
l := strings.Split(value, ",")
for i := range l {
l[i] = strings.TrimSpace(l[i])
}
return l
}
func getEnvIntOrDefault(envKey string, defaultValue int) int {
value := strings.TrimSpace(os.Getenv(envKey))
if value == "" {
return defaultValue
}
i, err := strconv.Atoi(value)
if err != nil {
return defaultValue
}
return i
}
func getEnvPortOrDefault(envKey string, defaultValue int) int {
p := getEnvIntOrDefault(envKey, defaultValue)
if p <= 0 || p >= 65534 {
return defaultValue
}
return p
}

131
back/server/utils.go Normal file
View File

@@ -0,0 +1,131 @@
package server
import (
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
)
func getParquetColumns(s Server, path string) []string {
query := fmt.Sprintf("DESCRIBE SELECT * FROM '%s';", path)
rows, err := s.Duckdb.Query(query)
if err != nil {
return []string{}
}
defer rows.Close()
var columns []string
for rows.Next() {
var columnName string
var columnType string
var nullable string
var key sql.NullString
var defaultValue sql.NullString
var extra sql.NullString
if err := rows.Scan(&columnName, &columnType, &nullable, &key, &defaultValue, &extra); err != nil {
return []string{}
}
columns = append(columns, columnName)
}
if err = rows.Err(); err != nil {
return []string{}
}
if len(columns) == 0 {
return []string{}
}
return columns
}
func getParquetLength(s Server, path string) uint64 {
query := fmt.Sprintf("SELECT COUNT(*) FROM '%s';", path)
row := s.Duckdb.QueryRow(query)
var count uint64
if err := row.Scan(&count); err != nil {
return 0
}
return count
}
// Walk through the given folder and its subfolders to find all parquet files
// Return a list of path
func getAllParquetFiles(folders []string) []string {
var paths []string
for _, baseDir := range folders {
_ = filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() || !strings.HasSuffix(info.Name(), ".parquet") {
return err
}
paths = append(paths, path)
return nil
})
}
return paths
}
func getFileSize(path string) uint64 {
info, err := os.Stat(path)
if err != nil {
return 0
}
return uint64(info.Size() / (1024 * 1024)) // MB
}
func FormatParquetName(path string) string {
_, file := filepath.Split(path)
fileName := strings.TrimSuffix(file, ".parquet")
parts := strings.Split(fileName, "-")
sourceName := parts[0]
var blocks []string
for _, part := range parts[1:] {
if strings.HasPrefix(part, "date_") {
dateStr := strings.TrimPrefix(part, "date_")
dateStr = strings.ReplaceAll(dateStr, "_", "/")
blocks = append(blocks, fmt.Sprintf("date: %s", dateStr))
} else if strings.HasPrefix(part, "source_") {
sourceStr := strings.TrimPrefix(part, "source_")
blocks = append(blocks, fmt.Sprintf("source: %s", sourceStr))
} else if strings.HasPrefix(part, "notes_") {
noteStr := strings.TrimPrefix(part, "notes_")
noteStr = strings.ReplaceAll(noteStr, "_", " ")
blocks = append(blocks, noteStr)
}
}
sourceName = strings.ReplaceAll(sourceName, "_", " ")
sourceWords := strings.Fields(sourceName)
for i, word := range sourceWords {
if len(word) > 0 {
sourceWords[i] = strings.ToUpper(string(word[0])) + word[1:]
}
}
formattedSourceName := strings.Join(sourceWords, " ")
if len(blocks) > 0 {
return fmt.Sprintf("%s (%s)", formattedSourceName, strings.Join(blocks, ", "))
}
return formattedSourceName
}
func createDirectoryIfNotExists(path string) error {
if _, err := os.Stat(path); os.IsNotExist(err) {
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
}
return nil
}