Remove "full_name" indication, keep only name & *_name
Signed-off-by: Hadi <112569860+anotherhadi@users.noreply.github.com>
This commit is contained in:
@@ -118,11 +118,6 @@ func removeDuplicateMaps(maps []map[string]string) []map[string]string {
|
||||
}
|
||||
|
||||
func buildSqlQuery(s *server.Server, queryText, column string, exactMatch bool) string {
|
||||
// Normalize "name" -> "full_name"
|
||||
if strings.EqualFold(column, "name") {
|
||||
column = "full_name"
|
||||
}
|
||||
|
||||
// Step 1: Determine candidate columns to search
|
||||
var candidateColumns []string
|
||||
if column == "all" || column == "" {
|
||||
|
||||
@@ -42,7 +42,7 @@ func LoadServerSettings() ServerSettings {
|
||||
// Dataleaks
|
||||
Folders: getEnvStringListOrDefault("DATALEAKS_FOLDERS", []string{}),
|
||||
CacheFolder: getEnvStringOrDefault("DATALEAKS_CACHE_FOLDER", ""),
|
||||
BaseColumns: getEnvStringListOrDefault("BASE_COLUMNS", []string{"email", "username", "password", "full_name", "phone", "url"}),
|
||||
BaseColumns: getEnvStringListOrDefault("BASE_COLUMNS", []string{"email", "username", "password", "name", "phone", "url"}),
|
||||
Limit: getEnvIntOrDefault("LIMIT", 200),
|
||||
ReloadDataleaksInterval: getEnvDurationOrDefault("RELOAD_DATALEAKS_INTERVAL", 20*time.Minute),
|
||||
|
||||
|
||||
@@ -100,7 +100,7 @@ combo_french-notes_crypto.parquet
|
||||
| age |
|
||||
| first_name |
|
||||
| last_name |
|
||||
| full_name |
|
||||
| middle_name |
|
||||
| address |
|
||||
| city |
|
||||
| country |
|
||||
@@ -116,17 +116,12 @@ combo_french-notes_crypto.parquet
|
||||
|
||||
- **Phone**: keep only `[^0-9]`
|
||||
|
||||
- **Names**:
|
||||
- Keep `first_name` / `last_name` if present.
|
||||
- Generate `full_name = CONCAT(first_name, ' ', last_name)`.
|
||||
- If only `name` exists, rename it to `full_name`.
|
||||
|
||||
- **Passwords**:
|
||||
- Hashes → `password_hash`.
|
||||
- Plaintext → `password`.
|
||||
- Never mix hashes and plaintext in the same column.
|
||||
|
||||
- **NULLs**: always use SQL `NULL` (never `""` or `"NULL"`).
|
||||
- **NULLs**: always use SQL `NULL` (never `""` or `"NULL"`, ...).
|
||||
|
||||
## Deduplication
|
||||
|
||||
|
||||
@@ -1,75 +1,9 @@
|
||||
package parquet
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/anotherhadi/eleakxir/leak-utils/settings"
|
||||
)
|
||||
|
||||
// If there is no full_name but there is last_name and first_name, create full_name
|
||||
// If there is no full_name, no last_name or no first_name, but there is name, rename name to full_name
|
||||
func addFullname(operations []ColumnOperation) []ColumnOperation {
|
||||
hasFullName := false
|
||||
hasFirstName := false
|
||||
hasLastName := false
|
||||
hasName := false
|
||||
for _, op := range operations {
|
||||
if op.Action != "drop" {
|
||||
if op.NewName == "full_name" {
|
||||
hasFullName = true
|
||||
} else if op.NewName == "first_name" {
|
||||
hasFirstName = true
|
||||
} else if op.NewName == "last_name" {
|
||||
hasLastName = true
|
||||
} else if op.NewName == "name" {
|
||||
hasName = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if hasFullName {
|
||||
return operations
|
||||
}
|
||||
if hasFirstName && hasLastName {
|
||||
operations = append(operations, ColumnOperation{
|
||||
OriginalName: "first_name || ' ' || last_name",
|
||||
NewName: "full_name",
|
||||
Action: "rename",
|
||||
})
|
||||
fmt.Println(settings.Muted.Render("\nAdding new column 'full_name' as concatenation of 'first_name' and 'last_name'."))
|
||||
return operations
|
||||
}
|
||||
if hasName {
|
||||
for i, op := range operations {
|
||||
if op.NewName == "name" && op.Action != "drop" {
|
||||
operations[i].NewName = "full_name"
|
||||
fmt.Println(settings.Muted.Render("\nRenaming column 'name' to 'full_name'."))
|
||||
return operations
|
||||
}
|
||||
}
|
||||
}
|
||||
if hasFirstName {
|
||||
operations = append(operations, ColumnOperation{
|
||||
OriginalName: "first_name",
|
||||
NewName: "full_name",
|
||||
Action: "rename",
|
||||
})
|
||||
fmt.Println(settings.Muted.Render("\nAdding new column 'full_name' from 'first_name'."))
|
||||
return operations
|
||||
}
|
||||
if hasLastName {
|
||||
operations = append(operations, ColumnOperation{
|
||||
OriginalName: "last_name",
|
||||
NewName: "full_name",
|
||||
Action: "rename",
|
||||
})
|
||||
fmt.Println(settings.Muted.Render("\nAdding new column 'full_name' from 'last_name'."))
|
||||
return operations
|
||||
}
|
||||
|
||||
return operations
|
||||
}
|
||||
|
||||
// formatColumnName formats a column name to be SQL-compliant.
|
||||
func formatColumnName(columnName string) string {
|
||||
columnName = strings.TrimSpace(columnName)
|
||||
|
||||
@@ -182,7 +182,6 @@ func configureColumns(input Parquet, skipLineFormating bool) []ColumnOperation {
|
||||
if !skipLineFormating {
|
||||
operations = formatColumns(operations)
|
||||
}
|
||||
operations = addFullname(operations)
|
||||
|
||||
return operations
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user