init
This commit is contained in:
21
back/.air.toml
Normal file
21
back/.air.toml
Normal file
@@ -0,0 +1,21 @@
|
||||
root = "."
|
||||
tmp_dir = "tmp"
|
||||
|
||||
[build]
|
||||
cmd = "go build -o ./tmp/main ./cmd/main.go"
|
||||
full_bin = "DEBUG=true DATALEAKS_FOLDERS=../testdata ./tmp/main"
|
||||
args = []
|
||||
exclude_dir = ["tmp", "vendor"]
|
||||
exclude_file = []
|
||||
exclude_regex = ["_test.go"]
|
||||
exclude_unchanged = false
|
||||
follow_symlink = false
|
||||
send_interrupt = false
|
||||
delay = 1000 # ms
|
||||
stop_on_error = true
|
||||
|
||||
[log]
|
||||
time = true
|
||||
|
||||
[misc]
|
||||
clean_on_exit = true
|
||||
1
back/.gitignore
vendored
Normal file
1
back/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
tmp/
|
||||
174
back/api/api.go
Normal file
174
back/api/api.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/backend/search"
|
||||
"github.com/anotherhadi/eleakxir/backend/server"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// TODO: We need to know when we hit the LIMIT
|
||||
|
||||
func routes(s *server.Server, cache *map[string]*search.Result) {
|
||||
s.Router.Use(
|
||||
func(c *gin.Context) {
|
||||
if s.Settings.Password != "" {
|
||||
password := c.GetHeader("X-Password")
|
||||
if password != s.Settings.Password {
|
||||
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
|
||||
return
|
||||
}
|
||||
}
|
||||
c.Next()
|
||||
},
|
||||
)
|
||||
|
||||
s.Router.GET("/", func(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"Settings": s.Settings,
|
||||
"Dataleaks": s.Dataleaks,
|
||||
"TotalDataleaks": s.TotalDataleaks,
|
||||
"TotalRows": s.TotalRows,
|
||||
"TotalSize": s.TotalSize,
|
||||
})
|
||||
})
|
||||
|
||||
s.Router.GET("/history", func(c *gin.Context) {
|
||||
type historyItem struct {
|
||||
Id string
|
||||
Status string
|
||||
Date time.Time
|
||||
Query search.Query
|
||||
Results int
|
||||
}
|
||||
var history []historyItem
|
||||
s.Mu.RLock()
|
||||
for _, r := range *cache {
|
||||
history = append(history, historyItem{
|
||||
Id: r.Id,
|
||||
Status: r.Status,
|
||||
Date: r.Date,
|
||||
Query: r.Query,
|
||||
Results: len(r.LeakResult.Rows),
|
||||
})
|
||||
}
|
||||
s.Mu.RUnlock()
|
||||
for i := 0; i < len(history)-1; i++ {
|
||||
for j := 0; j < len(history)-i-1; j++ {
|
||||
if history[j].Date.Before(history[j+1].Date) {
|
||||
history[j], history[j+1] = history[j+1], history[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"History": history,
|
||||
})
|
||||
})
|
||||
|
||||
s.Router.POST("/search", func(c *gin.Context) {
|
||||
var query search.Query
|
||||
if err := c.BindJSON(&query); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"Error": "invalid JSON"})
|
||||
return
|
||||
}
|
||||
query = cleanQuery(query)
|
||||
if len(query.Text) <= s.Settings.MinimumQueryLength {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"Error": "query too short"})
|
||||
return
|
||||
}
|
||||
id := search.EncodeQueryID(query, *s.TotalDataleaks)
|
||||
s.Mu.RLock()
|
||||
_, exists := (*cache)[id]
|
||||
s.Mu.RUnlock()
|
||||
if exists {
|
||||
c.JSON(http.StatusOK, gin.H{"Id": id})
|
||||
return
|
||||
}
|
||||
r := search.Result{
|
||||
Id: id,
|
||||
}
|
||||
go search.Search(s, query, &r, s.Mu)
|
||||
s.Mu.Lock()
|
||||
(*cache)[id] = &r
|
||||
s.Mu.Unlock()
|
||||
c.JSON(http.StatusOK, gin.H{"Id": id})
|
||||
})
|
||||
|
||||
s.Router.GET("/search/:id", func(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
s.Mu.RLock()
|
||||
r, exists := (*cache)[id]
|
||||
s.Mu.RUnlock()
|
||||
if !exists {
|
||||
c.JSON(http.StatusNotFound, gin.H{"Error": "not found"})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, r)
|
||||
})
|
||||
}
|
||||
|
||||
func Init(s *server.Server) {
|
||||
if !s.Settings.Debug {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
|
||||
s.Router = gin.Default()
|
||||
s.Router.Use(CORSMiddleware())
|
||||
|
||||
cache := make(map[string]*search.Result)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
time.Sleep(time.Minute)
|
||||
deleteOldCache(s, &cache)
|
||||
}
|
||||
}()
|
||||
|
||||
routes(s, &cache)
|
||||
}
|
||||
|
||||
func deleteOldCache(s *server.Server, cache *map[string]*search.Result) {
|
||||
s.Mu.Lock()
|
||||
defer s.Mu.Unlock()
|
||||
now := time.Now()
|
||||
for id, r := range *cache {
|
||||
if now.Sub(r.Date) > s.Settings.MaxCacheDuration {
|
||||
delete(*cache, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cleanQuery(q search.Query) search.Query {
|
||||
q.Column = strings.ToLower(strings.TrimSpace(q.Column))
|
||||
q.Column = strings.Join(strings.Fields(q.Column), " ")
|
||||
q.Column = strings.ReplaceAll(q.Column, "`", "")
|
||||
q.Column = strings.ReplaceAll(q.Column, "'", "")
|
||||
q.Column = strings.ReplaceAll(q.Column, "-", "_")
|
||||
q.Column = strings.ReplaceAll(q.Column, " ", "_")
|
||||
q.Column = strings.ReplaceAll(q.Column, "\"", "")
|
||||
|
||||
q.Text = strings.TrimSpace(q.Text)
|
||||
q.Text = strings.Join(strings.Fields(q.Text), " ")
|
||||
|
||||
return q
|
||||
}
|
||||
|
||||
func CORSMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
c.Writer.Header().
|
||||
Set("Access-Control-Allow-Headers", "X-Password, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")
|
||||
c.Writer.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS, GET, PUT, DELETE")
|
||||
|
||||
if c.Request.Method == "OPTIONS" {
|
||||
c.AbortWithStatus(204)
|
||||
return
|
||||
}
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
21
back/cmd/main.go
Normal file
21
back/cmd/main.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/backend/api"
|
||||
"github.com/anotherhadi/eleakxir/backend/server"
|
||||
)
|
||||
|
||||
func main() {
|
||||
server := server.NewServer()
|
||||
fmt.Println("Starting the server.")
|
||||
|
||||
api.Init(server)
|
||||
|
||||
err := server.Router.Run(":" + strconv.Itoa(server.Settings.Port))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
70
back/go.mod
Normal file
70
back/go.mod
Normal file
@@ -0,0 +1,70 @@
|
||||
module github.com/anotherhadi/eleakxir/backend
|
||||
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/anotherhadi/github-recon v1.5.6
|
||||
github.com/charmbracelet/log v0.4.2
|
||||
github.com/gin-gonic/gin v1.10.1
|
||||
github.com/marcboeker/go-duckdb v1.8.5
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/apache/arrow-go/v18 v18.1.0 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/bytedance/sonic v1.11.6 // indirect
|
||||
github.com/bytedance/sonic/loader v0.1.1 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
|
||||
github.com/charmbracelet/lipgloss v1.1.0 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.8.0 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/cloudwego/base64x v0.1.4 // indirect
|
||||
github.com/cloudwego/iasm v0.2.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-logfmt/logfmt v0.6.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.20.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/google/flatbuffers v25.1.24+incompatible // indirect
|
||||
github.com/google/go-github/v72 v72.0.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/joho/godotenv v1.5.1 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.9 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/saran13raj/go-pixels v0.0.0-20250629121333-58b240a3ae51 // indirect
|
||||
github.com/spf13/pflag v1.0.7 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
golang.org/x/arch v0.8.0 // indirect
|
||||
golang.org/x/crypto v0.38.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c // indirect
|
||||
golang.org/x/image v0.28.0 // indirect
|
||||
golang.org/x/mod v0.25.0 // indirect
|
||||
golang.org/x/net v0.40.0 // indirect
|
||||
golang.org/x/sync v0.15.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/text v0.26.0 // indirect
|
||||
golang.org/x/tools v0.33.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/protobuf v1.36.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
179
back/go.sum
Normal file
179
back/go.sum
Normal file
@@ -0,0 +1,179 @@
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/anotherhadi/github-recon v1.5.6 h1:IN3lQZRqqNbPpSyP5fvNoJrYODbM2tNwS5tiRgD+i1s=
|
||||
github.com/anotherhadi/github-recon v1.5.6/go.mod h1:E2tmCmjEZdJeBx8u1J8sSMtnmU8aDQ6IjCoq3ykoHtY=
|
||||
github.com/apache/arrow-go/v18 v18.1.0 h1:agLwJUiVuwXZdwPYVrlITfx7bndULJ/dggbnLFgDp/Y=
|
||||
github.com/apache/arrow-go/v18 v18.1.0/go.mod h1:tigU/sIgKNXaesf5d7Y95jBBKS5KsxTqYBKXFsvKzo0=
|
||||
github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE=
|
||||
github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0=
|
||||
github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
|
||||
github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
|
||||
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs=
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
|
||||
github.com/charmbracelet/log v0.4.2 h1:hYt8Qj6a8yLnvR+h7MwsJv/XvmBJXiueUcI3cIxsyig=
|
||||
github.com/charmbracelet/log v0.4.2/go.mod h1:qifHGX/tc7eluv2R6pWIpyHDDrrb/AG71Pf2ysQu5nw=
|
||||
github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE=
|
||||
github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=
|
||||
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
|
||||
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
|
||||
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
|
||||
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
|
||||
github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
|
||||
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
|
||||
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8=
|
||||
github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||
github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
|
||||
github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/flatbuffers v25.1.24+incompatible h1:4wPqL3K7GzBd1CwyhSd3usxLKOaJN/AC6puCca6Jm7o=
|
||||
github.com/google/flatbuffers v25.1.24+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/go-github/v72 v72.0.0 h1:FcIO37BLoVPBO9igQQ6tStsv2asG4IPcYFi655PPvBM=
|
||||
github.com/google/go-github/v72 v72.0.0/go.mod h1:WWtw8GMRiL62mvIquf1kO3onRHeWWKmK01qdCY8c5fg=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
|
||||
github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/marcboeker/go-duckdb v1.8.5 h1:tkYp+TANippy0DaIOP5OEfBEwbUINqiFqgwMQ44jME0=
|
||||
github.com/marcboeker/go-duckdb v1.8.5/go.mod h1:6mK7+WQE4P4u5AFLvVBmhFxY5fvhymFptghgJX6B+/8=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
|
||||
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/saran13raj/go-pixels v0.0.0-20250629121333-58b240a3ae51 h1:H/XUfYcLxI3CBmDlgBpnOeTntRgqWvIoUXnqhCF5a0s=
|
||||
github.com/saran13raj/go-pixels v0.0.0-20250629121333-58b240a3ae51/go.mod h1:sqhdZVLvqzTEBtmZBuTnFDUW0Lsryw2X2/wrLgqLEYg=
|
||||
github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M=
|
||||
github.com/spf13/pflag v1.0.7/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
|
||||
golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
|
||||
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
|
||||
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
|
||||
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c h1:KL/ZBHXgKGVmuZBZ01Lt57yE5ws8ZPSkkihmEyq7FXc=
|
||||
golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
|
||||
golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE=
|
||||
golang.org/x/image v0.28.0/go.mod h1:GUJYXtnGKEUgggyzh+Vxt+AviiCcyiwpsl8iQ8MvwGY=
|
||||
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
|
||||
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
|
||||
golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
|
||||
golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
|
||||
golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
|
||||
golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
|
||||
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
|
||||
golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
|
||||
golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
gonum.org/v1/gonum v0.15.1 h1:FNy7N6OUZVUaWG9pTiD+jlhdQ3lMP+/LcTpJ6+a8sQ0=
|
||||
gonum.org/v1/gonum v0.15.1/go.mod h1:eZTZuRFrzu5pcyjN5wJhcIhnUdNijYxX1T2IcrOGY0o=
|
||||
google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
|
||||
google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
|
||||
191
back/search/dataleak/dataleak.go
Normal file
191
back/search/dataleak/dataleak.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package dataleak
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/backend/server"
|
||||
"github.com/charmbracelet/log"
|
||||
)
|
||||
|
||||
type LeakResult struct {
|
||||
Duration time.Duration
|
||||
Rows []map[string]string
|
||||
Error string
|
||||
}
|
||||
|
||||
func Search(s *server.Server, queryText, column string, exactMatch bool) LeakResult {
|
||||
if len(*(s.Dataleaks)) == 0 {
|
||||
return LeakResult{
|
||||
Error: "No dataleak configured",
|
||||
}
|
||||
}
|
||||
now := time.Now()
|
||||
result := LeakResult{}
|
||||
|
||||
sqlQuery := buildSqlQuery(s, queryText, column, exactMatch)
|
||||
|
||||
if s.Settings.Debug {
|
||||
log.Info("New query:", "query", sqlQuery)
|
||||
}
|
||||
rows, err := s.Duckdb.Query(sqlQuery)
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
rawResult := make([][]byte, len(cols))
|
||||
dest := make([]any, len(cols))
|
||||
for i := range rawResult {
|
||||
dest[i] = &rawResult[i]
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(dest...)
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
rowMap := make(map[string]string)
|
||||
for i, colName := range cols {
|
||||
if rawResult[i] == nil || colName == "" {
|
||||
continue
|
||||
}
|
||||
if colName == "filename" {
|
||||
rowMap["source"] = server.FormatParquetName(string(rawResult[i]))
|
||||
continue
|
||||
}
|
||||
rowMap[colName] = string(rawResult[i])
|
||||
}
|
||||
result.Rows = append(result.Rows, rowMap)
|
||||
}
|
||||
|
||||
if err = rows.Err(); err != nil {
|
||||
result.Error = err.Error()
|
||||
return result
|
||||
}
|
||||
|
||||
result.Rows = removeDuplicateMaps(result.Rows)
|
||||
|
||||
result.Duration = time.Since(now)
|
||||
return result
|
||||
}
|
||||
|
||||
func removeDuplicateMaps(maps []map[string]string) []map[string]string {
|
||||
seen := make(map[string]struct{})
|
||||
result := []map[string]string{}
|
||||
|
||||
for _, m := range maps {
|
||||
// Create a unique key for the map by concatenating its key-value pairs
|
||||
var sb strings.Builder
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
slices.Sort(keys) // Sort keys to ensure consistent order
|
||||
for _, k := range keys {
|
||||
sb.WriteString(k)
|
||||
sb.WriteString("=")
|
||||
sb.WriteString(m[k])
|
||||
sb.WriteString(";")
|
||||
}
|
||||
key := sb.String()
|
||||
|
||||
if _, exists := seen[key]; !exists {
|
||||
seen[key] = struct{}{}
|
||||
result = append(result, m)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func buildSqlQuery(s *server.Server, queryText, column string, exactMatch bool) string {
|
||||
limit := strconv.Itoa(s.Settings.Limit)
|
||||
from := getFromClause(s)
|
||||
if column == "name" {
|
||||
column = "full_name"
|
||||
}
|
||||
columns := []string{column}
|
||||
if column == "all" || column == "" {
|
||||
columns = s.Settings.BaseColumns
|
||||
}
|
||||
columnsFiltered := []string{}
|
||||
allColumns := []string{}
|
||||
// TODO: Add columns that ends with _col aswell
|
||||
for _, dataleak := range *s.Dataleaks {
|
||||
for _, col := range dataleak.Columns {
|
||||
if !slices.Contains(allColumns, col) {
|
||||
allColumns = append(allColumns, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
if column == "full_text" {
|
||||
columnsFiltered = allColumns
|
||||
} else {
|
||||
for _, col := range columns {
|
||||
if slices.Contains(allColumns, col) {
|
||||
columnsFiltered = append(columnsFiltered, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(columnsFiltered) == 0 {
|
||||
return fmt.Sprintf("SELECT * FROM %s LIMIT %s", from, limit)
|
||||
}
|
||||
|
||||
where := getWhereClause(queryText, columnsFiltered, exactMatch)
|
||||
return fmt.Sprintf("SELECT * FROM %s WHERE %s LIMIT %s", from, where, limit)
|
||||
}
|
||||
|
||||
func getWhereClause(queryText string, columns []string, exactMatch bool) string {
|
||||
terms := strings.Fields(queryText)
|
||||
var andClauses []string
|
||||
|
||||
for _, term := range terms {
|
||||
var orClausesForTerm []string
|
||||
termEscaped := strings.ReplaceAll(term, "'", "''")
|
||||
|
||||
for _, col := range columns {
|
||||
if exactMatch {
|
||||
termEscapedILike := strings.ReplaceAll(termEscaped, "_", "\\_")
|
||||
termEscapedILike = strings.ReplaceAll(termEscapedILike, "%", "\\%")
|
||||
orClausesForTerm = append(orClausesForTerm, fmt.Sprintf("\"%s\" ILIKE '%s' ESCAPE '\\'", col, strings.ToLower(termEscapedILike)))
|
||||
} else {
|
||||
// Escape characters for ILIKE
|
||||
termEscapedILike := strings.ReplaceAll(termEscaped, "_", "\\_")
|
||||
termEscapedILike = strings.ReplaceAll(termEscapedILike, "%", "\\%")
|
||||
orClausesForTerm = append(orClausesForTerm, fmt.Sprintf("\"%s\" ILIKE '%%%s%%' ESCAPE '\\'", col, strings.ToLower(termEscapedILike)))
|
||||
}
|
||||
}
|
||||
andClauses = append(andClauses, "("+strings.Join(orClausesForTerm, " OR ")+")")
|
||||
}
|
||||
return strings.Join(andClauses, " AND ")
|
||||
}
|
||||
|
||||
func getFromClause(s *server.Server) string {
|
||||
parquets := []string{}
|
||||
for _, dataleak := range *s.Dataleaks {
|
||||
parquets = append(parquets, "'"+dataleak.Path+"'")
|
||||
}
|
||||
return fmt.Sprintf("read_parquet([%s], union_by_name=true, filename=true)", strings.Join(parquets, ", "))
|
||||
}
|
||||
|
||||
func castAllColumns(cols []string) []string {
|
||||
casted := make([]string, len(cols))
|
||||
for i, col := range cols {
|
||||
casted[i] = fmt.Sprintf("cast(%s as text)", col)
|
||||
}
|
||||
return casted
|
||||
}
|
||||
90
back/search/osint/github.go
Normal file
90
back/search/osint/github.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package osint
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/backend/server"
|
||||
|
||||
recon_email "github.com/anotherhadi/github-recon/github-recon/email"
|
||||
recon_username "github.com/anotherhadi/github-recon/github-recon/username"
|
||||
github_recon_settings "github.com/anotherhadi/github-recon/settings"
|
||||
)
|
||||
|
||||
type GithubResult struct {
|
||||
Duration time.Duration
|
||||
Error string
|
||||
|
||||
UsernameResult *recon_username.UsernameResult
|
||||
EmailResult *recon_email.EmailResult
|
||||
}
|
||||
|
||||
func Search(s *server.Server, queryText, column string) *GithubResult {
|
||||
if !s.Settings.GithubRecon {
|
||||
return nil
|
||||
}
|
||||
gr := GithubResult{}
|
||||
now := time.Now()
|
||||
settings := github_recon_settings.GetDefaultSettings()
|
||||
settings.Token = s.Settings.GithubToken
|
||||
settings.DeepScan = s.Settings.GithubDeepMode
|
||||
if settings.Token != "null" && strings.TrimSpace(settings.Token) != "" {
|
||||
settings.Client = settings.Client.WithAuthToken(settings.Token)
|
||||
}
|
||||
settings.Silent = true
|
||||
|
||||
queryText = strings.TrimSpace(queryText)
|
||||
|
||||
if column == "email" || strings.HasSuffix(column, "_email") ||
|
||||
column == "username" || strings.HasSuffix(column, "_username") ||
|
||||
column == "" || column == "all" {
|
||||
if isValidEmail(queryText) {
|
||||
settings.Target = queryText
|
||||
settings.TargetType = github_recon_settings.TargetEmail
|
||||
result := recon_email.Email(settings)
|
||||
gr.EmailResult = &result
|
||||
} else if isValidUsername(queryText) {
|
||||
settings.Target = queryText
|
||||
settings.TargetType = github_recon_settings.TargetUsername
|
||||
result, err := recon_username.Username(settings)
|
||||
if err != nil {
|
||||
gr.Error = err.Error()
|
||||
}
|
||||
if result.User.Username == "" {
|
||||
gr.UsernameResult = nil
|
||||
} else {
|
||||
gr.UsernameResult = &result
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
|
||||
gr.Duration = time.Since(now)
|
||||
return &gr
|
||||
}
|
||||
|
||||
func isValidEmail(email string) bool {
|
||||
if !strings.Contains(email, "@") || !strings.Contains(email, ".") {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(email, "@") || strings.HasSuffix(email, "@") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(email, " ") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func isValidUsername(username string) bool {
|
||||
if len(username) < 1 || len(username) > 39 {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(username, " ") {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
68
back/search/search.go
Normal file
68
back/search/search.go
Normal file
@@ -0,0 +1,68 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/backend/search/dataleak"
|
||||
"github.com/anotherhadi/eleakxir/backend/search/osint"
|
||||
"github.com/anotherhadi/eleakxir/backend/server"
|
||||
)
|
||||
|
||||
type Query struct {
|
||||
Text string
|
||||
Column string // The column to search in (e.g., "email", "password", etc.
|
||||
ExactMatch bool // Whether to search for an exact match
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Id string
|
||||
Date time.Time
|
||||
Status string // "pending", "completed"
|
||||
Query Query
|
||||
|
||||
LeakResult dataleak.LeakResult
|
||||
GithubResult osint.GithubResult
|
||||
}
|
||||
|
||||
func Search(s *server.Server, q Query, r *Result, mu *sync.RWMutex) {
|
||||
var wg sync.WaitGroup
|
||||
|
||||
mu.Lock()
|
||||
r.Date = time.Now()
|
||||
r.Status = "pending"
|
||||
r.Query = q
|
||||
mu.Unlock()
|
||||
|
||||
wg.Add(2)
|
||||
|
||||
go func() {
|
||||
leakResult := dataleak.Search(s, q.Text, q.Column, q.ExactMatch)
|
||||
mu.Lock()
|
||||
r.LeakResult = leakResult
|
||||
mu.Unlock()
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
go func() {
|
||||
githubResult := osint.Search(s, q.Text, q.Column)
|
||||
mu.Lock()
|
||||
r.GithubResult = *githubResult
|
||||
mu.Unlock()
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
|
||||
mu.Lock()
|
||||
r.Status = "completed"
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
func EncodeQueryID(q Query, dataleaksCount uint64) string {
|
||||
raw, _ := json.Marshal(q)
|
||||
return fmt.Sprintf("%d:%s", dataleaksCount, base64.URLEncoding.EncodeToString(raw))
|
||||
}
|
||||
111
back/server/dataleak.go
Normal file
111
back/server/dataleak.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
)
|
||||
|
||||
type Dataleak struct {
|
||||
Path string
|
||||
Name string
|
||||
Columns []string
|
||||
Length uint64
|
||||
Size uint64
|
||||
}
|
||||
|
||||
const CACHE_FILENAME = "dataleaks_cache.json"
|
||||
|
||||
// TODO: check os.FileInfo.ModTime() to see if the file has changed since last cache update
|
||||
|
||||
func Cache(s *Server) error {
|
||||
if len(s.Settings.Folders) == 0 {
|
||||
return nil
|
||||
}
|
||||
if s.Settings.CacheFolder == "" {
|
||||
s.Settings.CacheFolder = s.Settings.Folders[0]
|
||||
}
|
||||
if err := createDirectoryIfNotExists(s.Settings.CacheFolder); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cacheFile := filepath.Join(s.Settings.CacheFolder, CACHE_FILENAME)
|
||||
dataleaks := []Dataleak{}
|
||||
|
||||
data, err := os.ReadFile(cacheFile)
|
||||
if err == nil {
|
||||
if err := json.Unmarshal(data, &dataleaks); err != nil {
|
||||
log.Warn("Failed to unmarshal dataleaks cache", "error", err)
|
||||
}
|
||||
} else {
|
||||
log.Warn("Failed to read dataleaks cache file", "error", err)
|
||||
}
|
||||
|
||||
// Filter out non-existent files
|
||||
filteredDataleaks := []Dataleak{}
|
||||
writeOutput := false
|
||||
for _, d := range dataleaks {
|
||||
if _, err := os.Stat(d.Path); err == nil {
|
||||
filteredDataleaks = append(filteredDataleaks, d)
|
||||
} else if os.IsNotExist(err) {
|
||||
log.Info("Removing non-existent file from cache", "path", d.Path)
|
||||
writeOutput = true
|
||||
} else {
|
||||
log.Error("Error checking file existence", "path", d.Path, "error", err)
|
||||
}
|
||||
}
|
||||
dataleaks = filteredDataleaks
|
||||
|
||||
// Create a map for quick lookups
|
||||
dataleakMap := make(map[string]struct{}, len(dataleaks))
|
||||
for _, d := range dataleaks {
|
||||
dataleakMap[d.Path] = struct{}{}
|
||||
}
|
||||
|
||||
// Add new files
|
||||
parquetFiles := getAllParquetFiles(s.Settings.Folders)
|
||||
for _, p := range parquetFiles {
|
||||
if _, found := dataleakMap[p]; found {
|
||||
continue
|
||||
}
|
||||
|
||||
writeOutput = true
|
||||
dataleaks = append(dataleaks, getDataleak(*s, p))
|
||||
}
|
||||
|
||||
if writeOutput {
|
||||
data, err := json.MarshalIndent(dataleaks, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error marshalling cache: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(cacheFile, data, 0644); err != nil {
|
||||
return fmt.Errorf("error writing cache: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
s.Dataleaks = &dataleaks
|
||||
totalDataleaks := uint64(len(dataleaks))
|
||||
totalRows := uint64(0)
|
||||
totalSize := uint64(0)
|
||||
for _, d := range dataleaks {
|
||||
totalRows += d.Length
|
||||
totalSize += d.Size
|
||||
}
|
||||
s.TotalDataleaks = &totalDataleaks
|
||||
s.TotalSize = &totalSize
|
||||
s.TotalRows = &totalRows
|
||||
return nil
|
||||
}
|
||||
|
||||
func getDataleak(s Server, path string) Dataleak {
|
||||
return Dataleak{
|
||||
Path: path,
|
||||
Name: FormatParquetName(path),
|
||||
Columns: getParquetColumns(s, path),
|
||||
Length: getParquetLength(s, path),
|
||||
Size: getFileSize(path),
|
||||
}
|
||||
}
|
||||
61
back/server/server.go
Normal file
61
back/server/server.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/gin-gonic/gin"
|
||||
_ "github.com/marcboeker/go-duckdb"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
Settings ServerSettings
|
||||
|
||||
Dataleaks *[]Dataleak
|
||||
|
||||
TotalRows *uint64
|
||||
TotalDataleaks *uint64
|
||||
TotalSize *uint64 // MB
|
||||
|
||||
Router *gin.Engine
|
||||
Duckdb *sql.DB
|
||||
Mu *sync.RWMutex
|
||||
}
|
||||
|
||||
func NewServer() *Server {
|
||||
zero := uint64(0)
|
||||
emptyDataleak := []Dataleak{}
|
||||
s := &Server{
|
||||
Settings: LoadServerSettings(),
|
||||
Mu: &sync.RWMutex{},
|
||||
TotalDataleaks: &zero,
|
||||
TotalRows: &zero,
|
||||
TotalSize: &zero,
|
||||
Dataleaks: &emptyDataleak,
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
s.Duckdb, err = sql.Open("duckdb", "")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = Cache(s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
time.Sleep(s.Settings.ReloadDataleaksInterval)
|
||||
err := Cache(s)
|
||||
if err != nil {
|
||||
log.Error(err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return s
|
||||
}
|
||||
129
back/server/settings.go
Normal file
129
back/server/settings.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
github_recon_settings "github.com/anotherhadi/github-recon/settings"
|
||||
)
|
||||
|
||||
type ServerSettings struct {
|
||||
Port int `json:"-"` // Port to run the server on
|
||||
Debug bool
|
||||
Password string `json:"-"` // Do not expose the password in JSON
|
||||
MinimumQueryLength int
|
||||
MaxCacheDuration time.Duration // Delete a search from the cache after this duration
|
||||
|
||||
// Dataleaks
|
||||
Folders []string // Folders to search in for parquets, recursive
|
||||
CacheFolder string
|
||||
BaseColumns []string // Use these columns when column="all"
|
||||
Limit int // Limit number of rows returned
|
||||
ReloadDataleaksInterval time.Duration // Reload dataleaks files from disk every X
|
||||
|
||||
// OSINT Tools
|
||||
GithubRecon bool // Activate github-recon OSINT tool
|
||||
GithubToken string `json:"-"` // Github token for github-recon
|
||||
GithubTokenLoaded bool
|
||||
GithubDeepMode bool // Deep mode for github-recon
|
||||
}
|
||||
|
||||
func LoadServerSettings() ServerSettings {
|
||||
ss := ServerSettings{
|
||||
Port: getEnvPortOrDefault("PORT", 9198),
|
||||
Debug: getEnvBoolOrDefault("DEBUG", false),
|
||||
Password: getEnvStringOrDefault("PASSWORD", ""),
|
||||
MinimumQueryLength: getEnvIntOrDefault("MINIMUM_QUERY_LENGTH", 3),
|
||||
MaxCacheDuration: getEnvDurationOrDefault("MAX_CACHE_DURATION", 24*time.Hour),
|
||||
|
||||
// Dataleaks
|
||||
Folders: getEnvStringListOrDefault("DATALEAKS_FOLDERS", []string{}),
|
||||
CacheFolder: getEnvStringOrDefault("DATALEAKS_CACHE_FOLDER", ""),
|
||||
BaseColumns: getEnvStringListOrDefault("BASE_COLUMNS", []string{"email", "username", "password", "full_name", "phone", "url"}),
|
||||
Limit: getEnvIntOrDefault("LIMIT", 100),
|
||||
ReloadDataleaksInterval: getEnvDurationOrDefault("RELOAD_DATALEAKS_INTERVAL", 20*time.Minute),
|
||||
|
||||
// OSINT Tools
|
||||
GithubRecon: getEnvBoolOrDefault("GITHUB_RECON", true),
|
||||
GithubToken: getEnvStringOrDefault("GITHUB_TOKEN", "null"),
|
||||
GithubDeepMode: getEnvBoolOrDefault("GITHUB_DEEP_MODE", false),
|
||||
}
|
||||
|
||||
if ss.GithubToken == "null" || strings.TrimSpace(ss.GithubToken) == "" {
|
||||
ss.GithubToken = github_recon_settings.GetToken()
|
||||
}
|
||||
|
||||
if ss.GithubToken != "null" && strings.TrimSpace(ss.GithubToken) != "" {
|
||||
ss.GithubTokenLoaded = true
|
||||
}
|
||||
|
||||
return ss
|
||||
}
|
||||
|
||||
func getEnvStringOrDefault(envKey, defaultValue string) string {
|
||||
value := strings.TrimSpace(os.Getenv(envKey))
|
||||
if value == "" {
|
||||
return defaultValue
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func getEnvBoolOrDefault(envKey string, defaultValue bool) bool {
|
||||
value := strings.TrimSpace(os.Getenv(envKey))
|
||||
if value == "" {
|
||||
return defaultValue
|
||||
}
|
||||
value = strings.ToLower(value)
|
||||
if value == "true" || value == "1" {
|
||||
return true
|
||||
} else if value == "false" || value == "0" {
|
||||
return false
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func getEnvDurationOrDefault(envKey string, defaultValue time.Duration) time.Duration {
|
||||
v := getEnvStringOrDefault(envKey, "")
|
||||
if v == "" {
|
||||
return defaultValue
|
||||
}
|
||||
t, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
return defaultValue
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func getEnvStringListOrDefault(envKey string, defaultValue []string) []string {
|
||||
value := strings.TrimSpace(os.Getenv(envKey))
|
||||
if value == "" {
|
||||
return defaultValue
|
||||
}
|
||||
l := strings.Split(value, ",")
|
||||
for i := range l {
|
||||
l[i] = strings.TrimSpace(l[i])
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func getEnvIntOrDefault(envKey string, defaultValue int) int {
|
||||
value := strings.TrimSpace(os.Getenv(envKey))
|
||||
if value == "" {
|
||||
return defaultValue
|
||||
}
|
||||
i, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return defaultValue
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func getEnvPortOrDefault(envKey string, defaultValue int) int {
|
||||
p := getEnvIntOrDefault(envKey, defaultValue)
|
||||
if p <= 0 || p >= 65534 {
|
||||
return defaultValue
|
||||
}
|
||||
return p
|
||||
}
|
||||
131
back/server/utils.go
Normal file
131
back/server/utils.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func getParquetColumns(s Server, path string) []string {
|
||||
query := fmt.Sprintf("DESCRIBE SELECT * FROM '%s';", path)
|
||||
|
||||
rows, err := s.Duckdb.Query(query)
|
||||
if err != nil {
|
||||
return []string{}
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var columns []string
|
||||
|
||||
for rows.Next() {
|
||||
var columnName string
|
||||
var columnType string
|
||||
var nullable string
|
||||
|
||||
var key sql.NullString
|
||||
var defaultValue sql.NullString
|
||||
var extra sql.NullString
|
||||
|
||||
if err := rows.Scan(&columnName, &columnType, &nullable, &key, &defaultValue, &extra); err != nil {
|
||||
return []string{}
|
||||
}
|
||||
columns = append(columns, columnName)
|
||||
}
|
||||
|
||||
if err = rows.Err(); err != nil {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
if len(columns) == 0 {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
return columns
|
||||
}
|
||||
|
||||
func getParquetLength(s Server, path string) uint64 {
|
||||
query := fmt.Sprintf("SELECT COUNT(*) FROM '%s';", path)
|
||||
|
||||
row := s.Duckdb.QueryRow(query)
|
||||
|
||||
var count uint64
|
||||
if err := row.Scan(&count); err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
// Walk through the given folder and its subfolders to find all parquet files
|
||||
// Return a list of path
|
||||
func getAllParquetFiles(folders []string) []string {
|
||||
var paths []string
|
||||
for _, baseDir := range folders {
|
||||
_ = filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil || info.IsDir() || !strings.HasSuffix(info.Name(), ".parquet") {
|
||||
return err
|
||||
}
|
||||
paths = append(paths, path)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return paths
|
||||
}
|
||||
|
||||
func getFileSize(path string) uint64 {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return uint64(info.Size() / (1024 * 1024)) // MB
|
||||
}
|
||||
|
||||
func FormatParquetName(path string) string {
|
||||
_, file := filepath.Split(path)
|
||||
fileName := strings.TrimSuffix(file, ".parquet")
|
||||
|
||||
parts := strings.Split(fileName, "-")
|
||||
sourceName := parts[0]
|
||||
var blocks []string
|
||||
|
||||
for _, part := range parts[1:] {
|
||||
if strings.HasPrefix(part, "date_") {
|
||||
dateStr := strings.TrimPrefix(part, "date_")
|
||||
dateStr = strings.ReplaceAll(dateStr, "_", "/")
|
||||
blocks = append(blocks, fmt.Sprintf("date: %s", dateStr))
|
||||
} else if strings.HasPrefix(part, "source_") {
|
||||
sourceStr := strings.TrimPrefix(part, "source_")
|
||||
blocks = append(blocks, fmt.Sprintf("source: %s", sourceStr))
|
||||
} else if strings.HasPrefix(part, "notes_") {
|
||||
noteStr := strings.TrimPrefix(part, "notes_")
|
||||
noteStr = strings.ReplaceAll(noteStr, "_", " ")
|
||||
blocks = append(blocks, noteStr)
|
||||
}
|
||||
}
|
||||
|
||||
sourceName = strings.ReplaceAll(sourceName, "_", " ")
|
||||
sourceWords := strings.Fields(sourceName)
|
||||
for i, word := range sourceWords {
|
||||
if len(word) > 0 {
|
||||
sourceWords[i] = strings.ToUpper(string(word[0])) + word[1:]
|
||||
}
|
||||
}
|
||||
formattedSourceName := strings.Join(sourceWords, " ")
|
||||
|
||||
if len(blocks) > 0 {
|
||||
return fmt.Sprintf("%s (%s)", formattedSourceName, strings.Join(blocks, ", "))
|
||||
}
|
||||
|
||||
return formattedSourceName
|
||||
}
|
||||
|
||||
func createDirectoryIfNotExists(path string) error {
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user