anotherhadi/eleakxir
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

jsonToParquet: add --delete-temp-file option

Browse Source 
Signed-off-by: Hadi <112569860+anotherhadi@users.noreply.github.com>
This commit is contained in:
Hadi
2025-10-01 12:07:00 +02:00
parent 81ff9804bb
commit e12e3e6e1b
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
leak-utils/misc/json.go
View File

@@ -110,10 +110,12 @@ func flattenJSONFile(inputFile string, outputFile string) error {
return nil
}
func JsonToParquet(lu settings.LeakUtils, inputFile string, outputFile string) error {
func JsonToParquet(lu settings.LeakUtils, inputFile string, outputFile string, deleteTempFile bool) error {
tmpFile := filepath.Join("/tmp", "leak-utils.flat.json")
err := flattenJSONFile(inputFile, tmpFile)
defer os.Remove(tmpFile)
if deleteTempFile {
defer os.Remove(tmpFile)
}
query := fmt.Sprintf(`COPY (FROM read_json('%s', union_by_name=true, ignore_errors=true)) TO '%s' (FORMAT 'parquet', COMPRESSION '%s', ROW_GROUP_SIZE 200000);`, tmpFile, outputFile, lu.Compression)