Created
March 12, 2024 00:27
-
-
Save peaeater/1b8c6e402d6b8cb71bd1b2527099e067 to your computer and use it in GitHub Desktop.
Split csv files with or without header rows, multiline columns with newlines ok.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<# | |
Split csv files with or without header rows, multiline columns with newlines ok. | |
#> | |
param ( | |
[string]$in, | |
[string]$outdir = [System.IO.Path]::GetDirectoryName($in), | |
[int]$count = 1000, | |
[string]$delimiter = ',', | |
[string[]]$header = @() | |
) | |
function createOutPath($in, $outdir, $fileIndex) { | |
$name = [System.IO.Path]::GetFileNameWithoutExtension($in) | |
$ext = [System.IO.Path]::GetExtension($in) | |
# append file index to original file name in outdir | |
return join-path $outdir "$name-$fileIndex$ext" | |
} | |
# create out dir if it doesn't exist | |
if (!(test-path $outdir)) { | |
mkdir $outdir | Out-Null | |
} | |
$fileIndex = 1 | |
$rowIndex = 0 | |
$file = @() | |
if ($null -ne $header -and $header.count -ne 0) { | |
# import with supplied header (file does not have its own header) | |
Import-Csv -Path $in -Delimiter $delimiter -Header $header | ForEach-Object { | |
$file += $_ | |
$rowIndex++ | |
$out = createOutPath $in $outdir $fileIndex | |
if ($rowIndex -eq $count) { | |
$file | Export-Csv -Path $out -Encoding utf8NoBOM -NoTypeInformation -UseQuotes AsNeeded | |
$fileIndex++ | |
$rowIndex = 0 | |
$file = @() | |
write-host $out | |
} | |
} | |
} | |
else { | |
# import and assume first row is header | |
Import-Csv -Path $in -Delimiter $delimiter | ForEach-Object { | |
$file += $_ | |
$rowIndex++ | |
$out = createOutPath $in $outdir $fileIndex | |
if ($rowIndex -eq $count) { | |
$file | Export-Csv -Path $out -Encoding utf8NoBOM -NoTypeInformation -UseQuotes AsNeeded | |
$fileIndex++ | |
$rowIndex = 0 | |
$file = @() | |
write-host $out | |
} | |
} | |
} | |
if ($file.Count -gt 0) { | |
$out = createOutPath $in $outdir $fileIndex | |
$file | Export-Csv -Path $out -Encoding utf8NoBOM -NoTypeInformation -UseQuotes AsNeeded | |
write-host $out | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment