format french chars

Signed-off-by: Hadi <112569860+anotherhadi@users.noreply.github.com>

back/api/api.go

@@ -94,6 +94,7 @@ func routes(s *server.Server, cache *map[string]*search.Result, searchQueue chan
 			c.JSON(http.StatusBadRequest, gin.H{"Error": "query too short"})
 			return
 		}
+		query.Folders = s.Settings.Folders
 		id := search.EncodeQueryID(query, *s.TotalDataleaks)
 		s.Mu.RLock()
 		_, exists := (*cache)[id]

back/search/dataleak/dataleak.go

@@ -2,6 +2,7 @@ package dataleak
 
 import (
 	"fmt"
+	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
@@ -19,7 +20,7 @@ type LeakResult struct {
 	LimitHit bool // Whether the search hit the limit
 }
 
-func Search(s *server.Server, queryText, column string, exactMatch bool) LeakResult {
+func Search(s *server.Server, queryText, column string, exactMatch bool, includeFolders []bool) LeakResult {
 	if len(*(s.Dataleaks)) == 0 {
 		return LeakResult{
 			Inactive: true,
@@ -28,7 +29,11 @@ func Search(s *server.Server, queryText, column string, exactMatch bool) LeakRes
 	now := time.Now()
 	result := LeakResult{}
 
-	sqlQuery := buildSqlQuery(s, queryText, column, exactMatch)
+	sqlQuery := buildSqlQuery(s, queryText, column, exactMatch, includeFolders)
+	if strings.HasPrefix(sqlQuery, "error:") {
+		result.Error = strings.TrimPrefix(sqlQuery, "error: ")
+		return result
+	}
 
 	if s.Settings.Debug {
 		log.Info("New query:", "query", sqlQuery)
@@ -117,7 +122,21 @@ func removeDuplicateMaps(maps []map[string]string) []map[string]string {
 	return result
 }
 
-func buildSqlQuery(s *server.Server, queryText, column string, exactMatch bool) string {
+func buildSqlQuery(s *server.Server, queryText, column string, exactMatch bool, includeFolders []bool) string {
+	folders := s.Settings.Folders
+	includedFolders := []string{}
+	for i, f := range folders {
+		if i >= len(includeFolders) {
+			break
+		}
+		if includeFolders[i] {
+			includedFolders = append(includedFolders, f)
+		}
+	}
+	if len(includedFolders) == 0 {
+		return "error: no folders included"
+	}
+
 	// Step 1: Determine candidate columns to search
 	var candidateColumns []string
 	if column == "all" || column == "" {
@@ -132,6 +151,9 @@ func buildSqlQuery(s *server.Server, queryText, column string, exactMatch bool)
 	allColumns := make([]string, 0)
 	seen := make(map[string]struct{})
 	for _, dataleak := range *s.Dataleaks {
+		if !isPathInFolders(dataleak.Path, includedFolders) {
+			continue
+		}
 		for _, col := range dataleak.Columns {
 			if _, ok := seen[col]; !ok {
 				seen[col] = struct{}{}
@@ -162,7 +184,7 @@ func buildSqlQuery(s *server.Server, queryText, column string, exactMatch bool)
 	}
 
 	limit := strconv.Itoa(s.Settings.Limit)
-	from := getFromClause(s)
+	from := getFromClause(s, includedFolders)
 
 	if len(columnsFiltered) == 0 {
 		return fmt.Sprintf("SELECT * FROM %s LIMIT %s", from, limit)
@@ -210,9 +232,12 @@ func getWhereClause(queryText string, columns []string, exactMatch bool) string
 	return strings.Join(andClauses, " AND ")
 }
 
-func getFromClause(s *server.Server) string {
+func getFromClause(s *server.Server, includedFolders []string) string {
 	parquets := []string{}
 	for _, dataleak := range *s.Dataleaks {
+		if !isPathInFolders(dataleak.Path, includedFolders) {
+			continue
+		}
 		parquets = append(parquets, "'"+dataleak.Path+"'")
 	}
 	return fmt.Sprintf("read_parquet([%s], union_by_name=true, filename=true)", strings.Join(parquets, ", "))
@@ -272,3 +297,16 @@ func GetDataleakSample(s server.Server, path string) ([][]string, error) {
 
 	return rowsData, nil
 }
+
+func isPathInFolders(path string, folders []string) bool {
+	for _, folder := range folders {
+		rel, err := filepath.Rel(folder, path)
+		if err != nil {
+			continue
+		}
+		if !strings.HasPrefix(rel, "..") {
+			return true
+		}
+	}
+	return false
+}

back/search/search.go

@@ -17,6 +17,8 @@ type Query struct {
 	Text           string
 	Column         string // The column to search in (e.g., "email", "password", etc.
 	ExactMatch     bool   // Whether to search for an exact match
+	Folders        []string
+	IncludeFolders []bool
 
 	// Services
 	Datawells     bool // Whether to include datawells in the search
@@ -53,7 +55,7 @@ func Search(s *server.Server, q Query, r *Result, mu *sync.RWMutex) {
 			wg.Done()
 			return
 		}
-		leakResult := dataleak.Search(s, q.Text, q.Column, q.ExactMatch)
+		leakResult := dataleak.Search(s, q.Text, q.Column, q.ExactMatch, q.IncludeFolders)
 		mu.Lock()
 		r.LeakResult = leakResult
 		r.ResultsCount += len(leakResult.Rows)

flake.lock

@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1757487488,
-        "narHash": "sha256-zwE/e7CuPJUWKdvvTCB7iunV4E/+G0lKfv4kk/5Izdg=",
+        "lastModified": 1767379071,
+        "narHash": "sha256-EgE0pxsrW9jp9YFMkHL9JMXxcqi/OoumPJYwf+Okucw=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "ab0f3607a6c7486ea22229b92ed2d355f1482ee0",
+        "rev": "fb7944c166a3b630f177938e478f0378e64ce108",
         "type": "github"
       },
       "original": {

front/src/lib/components/index/search/datawells.svelte

@@ -24,7 +24,7 @@
 
   let copyText = $state("Copy to clipboard");
 
-  async function copyDataleaksInformation(withSample: boolean) {
+  async function copyDataleaksInformation(withColumns: boolean, withSample: boolean) {
     if (!filteredDataleaks || filteredDataleaks.length === 0) {
       copyText = "No dataleaks to copy";
       return;
@@ -60,6 +60,8 @@
             console.error("Failed to fetch sample for", dataleak.Name, err);
             fullText += "Sample: [Failed to fetch]\n";
           });
+      } else if (withColumns) {
+        fullText += `Columns: ${dataleak.Columns.join(", ")}\n`;
       }
       fullText += "\n";
     }
@@ -150,12 +152,17 @@
       >
         <button
           class="btn btn-xs"
-          onclick={() => copyDataleaksInformation(false)}>{copyText}</button
+          onclick={() => copyDataleaksInformation(false, false)}>{copyText}</button
         >
         <button
           class="btn btn-xs"
-          onclick={() => copyDataleaksInformation(true)}
-          >{copyText} (w/ sample)</button
+          onclick={() => copyDataleaksInformation(true, false)}
+          >{copyText} (w/ columns)</button
+        >
+        <button
+          class="btn btn-xs"
+          onclick={() => copyDataleaksInformation(false, true)}
+          >{copyText} (w/ samples)</button
         >
       </div>
     </div>

front/src/lib/components/index/search/searchbar.svelte

@@ -12,6 +12,8 @@
     initialDatawells = true,
     initialGithubRecon = true,
     initialGravatarRecon = true,
+    folders = [],
+    initialIncludeFolders = [],
   }: {
     initialQuery?: string;
     initialFilter?: string;
@@ -19,6 +21,8 @@
     initialDatawells?: boolean;
     initialGithubRecon?: boolean;
     initialGravatarRecon?: boolean;
+    folders?: string[];
+    initialIncludeFolders?: boolean[];
   } = $props();
 
   let filters = [
@@ -38,6 +42,16 @@
   let datawells = $state<boolean>(initialDatawells);
   let githubRecon = $state<boolean>(initialGithubRecon);
   let gravatarRecon = $state<boolean>(initialGravatarRecon);
+  let includeFolders = $state<boolean[]>([]);
+
+  $effect(() => {
+    if (folders.length > 0 && includeFolders.length !== folders.length) {
+      includeFolders =
+        initialIncludeFolders.length === folders.length
+          ? [...initialIncludeFolders]
+          : new Array(folders.length).fill(true);
+    }
+  });
 
   function NewSearch() {
     axios
@@ -50,6 +64,7 @@
           Datawells: datawells,
           GithubRecon: githubRecon,
           GravatarRecon: gravatarRecon,
+          IncludeFolders: includeFolders,
         },
         {
           headers: {
@@ -104,6 +119,23 @@
             <input type="checkbox" bind:checked={datawells} class="checkbox" />
             Datawells lookup
           </label>
+          {#each folders as folder, i}
+            <label class="label">
+              {#if includeFolders[i] !== undefined}
+                <input
+                  type="checkbox"
+                  bind:checked={includeFolders[i]}
+                  class="checkbox checkbox-xs ml-5"
+                  disabled={!datawells}
+                />
+              {/if}
+              {
+              folder.split("/")
+                .filter((part) => part.length > 0)
+                .pop()
+              }
+            </label>
+          {/each}
         </li>
         <li>
           <label class="label">
@@ -142,14 +174,10 @@
         class="grow input-xl"
         type="text"
         bind:value={query}
-        placeholder={
-        (
-        activeFilter === "all"
+        placeholder={(activeFilter === "all"
           ? "Search..."
-          : `Search in ${activeFilter.replace("_", " ")}...`
-        )+
-        (activeFilter === "phone" && " (e.g. 612233445)" || "")
-        }
+          : `Search in ${activeFilter.replace("_", " ")}...`) +
+          ((activeFilter === "phone" && " (e.g. 612233445)") || "")}
         required
       />
 

front/src/lib/types.ts

@@ -2,6 +2,8 @@ export type Query = {
   Text: string;
   Column: string;
   ExactMatch: boolean;
+  Folders: string[];
+  IncludeFolders: boolean[];
 
   // Services
   Datawells: boolean;

front/src/routes/search/[id]/index.svelte

@@ -116,6 +116,8 @@
         initialDatawells={result.Query.Datawells}
         initialGithubRecon={result.Query.GithubRecon}
         initialGravatarRecon={result.Query.GravatarRecon}
+        folders={result.Query.Folders}
+        initialIncludeFolders={result.Query.IncludeFolders}
       />
     </header>
 

front/src/routes/search/index.svelte

@@ -55,7 +55,7 @@
 <main>
   <header class="flex gap-5 flex-col">
     <h1 class="h1"><span class="text-2xl align-middle">🔍</span> Search</h1>
-    <Searchbar />
+    <Searchbar folders={serverInfo?.Settings.Folders} />
   </header>
 
   <div class="my-10"></div>

leak-utils/misc/csv.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"slices"
 	"strings"
 
 	"github.com/anotherhadi/eleakxir/leak-utils/settings"
@@ -97,10 +96,12 @@ func csvHasHeader(inputFile, delimiter string) (hasHeader bool, err error) {
 	}
 	knownHeaders := []string{"email", "password", "username", "phone", "lastname", "firstname", "mail", "addresse", "nom", "id"}
 	for _, knownHeader := range knownHeaders {
-		if slices.Contains(firstRow, knownHeader) {
+		for _, col := range firstRow {
+			if strings.HasSuffix(col, knownHeader) {
 				return true, nil
 			}
 		}
+	}
 	return false, nil
 }
 

leak-utils/parquet/format.go

@@ -14,6 +14,11 @@ func formatColumnName(columnName string) string {
 	columnName = strings.ReplaceAll(columnName, " ", "_")
 	columnName = strings.ReplaceAll(columnName, "-", "_")
 	columnName = strings.ReplaceAll(columnName, ".", "_")
+	columnName = strings.ReplaceAll(columnName, "é", "e")
+	columnName = strings.ReplaceAll(columnName, "è", "e")
+	columnName = strings.ReplaceAll(columnName, "à", "a")
+	columnName = strings.ReplaceAll(columnName, "ù", "u")
+	columnName = strings.ReplaceAll(columnName, "ç", "c")
 	// Only keep a-z, 0-9 and _
 	var formatted strings.Builder
 	for _, r := range columnName {

leak-utils/parquet/suggestions.go

@@ -34,6 +34,7 @@ var (
 		"birth_date",
 		"url",
 		"ip",
+		"job",
 	}
 
 	suggestions = map[string]string{
@@ -62,6 +63,7 @@ var (
 		"email":           "email",
 		"zip":             "postal_code",
 		"postalcode":      "postal_code",
+		"postcode":        "postal_code",
 		"zipcode":         "postal_code",
 		"postal":          "postal_code",
 		"codepostal":      "postal_code",
@@ -86,6 +88,8 @@ var (
 		"numeromobile":    "phone",
 		"mobilephone":     "phone",
 		"mobile":          "phone",
+		"website":         "url",
+		"jobtitle":        "job",
 	}
 )