Created
April 18, 2020 16:08
-
-
Save milinmestry/cf7ca21e9dffeb2354e6020218f30bdf to your computer and use it in GitHub Desktop.
PHP Swoole Writing large dataset as json into file
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
// functions.php | |
function memoryStats($real = false) { | |
return memory_get_usage($real); | |
} | |
function memoryStatsHuman($real = false) { | |
return (memoryStats($real) / 1024) . ' MB'; | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
require_once './functions.php'; | |
require_once './vendor/autoload.php'; | |
$allocatedMemory = memoryStatsHuman(true); | |
echo 'Allocated memory: ', $allocatedMemory, PHP_EOL; | |
$startTime = microtime(true); | |
echo 'Before loop memory usage: ', memoryStatsHuman(false), PHP_EOL; | |
$filename = __DIR__ . '/json-files/records-250000.json'; | |
// use the factory to create a Faker\Generator instance | |
function generatorData() { | |
$faker = Faker\Factory::create(); | |
$numRows = 250000; | |
for ($i = 0; $i < $numRows; $i++) { | |
yield [ | |
'name' => $faker->name, | |
'email' => $faker->email, | |
'spent' => $faker->randomFloat(15), | |
'start_date' => $faker->date(), | |
'end_date' => $faker->date(), | |
'cpa' => $faker->randomNumber(7), | |
'cpm' => $faker->randomFloat(10), | |
'created_at' => date('Y-m-d H:i:s'), | |
]; | |
// echo $i,PHP_EOL; | |
} | |
} | |
// Swoole\Runtime::enableCoroutine(); | |
// $data = new SplFixedArray($numRows); | |
// $rawData = [ | |
// 'records' => $data, // This fails memory exhausted when working with normal array | |
// 'recordCount' => $data>getSize(), | |
// ]; | |
$fp = null; | |
$fp = fopen($filename, 'ab+'); | |
// You need to know how you need to create JSON file and create accordingly | |
go(function() use ($fp) { | |
Swoole\Coroutine\System::fwrite($fp, '{' . PHP_EOL . '"records":[' . PHP_EOL); | |
foreach (generatorData() as $key => $data) { | |
// print_r($data); | |
$json = $jsonStr = ''; | |
$json = json_encode($data); | |
$jsonStr = (0 !== $key) ? ',' . PHP_EOL . $json : $json; | |
$bytesWrote = Swoole\Coroutine\System::fwrite($fp, $jsonStr); | |
// echo "bytes wrote {$bytesWrote} \n"; | |
} | |
// metadata | |
$metadata = [ | |
'recordCount' => $key + 1 // array starts with 0, so +1 added | |
]; | |
$metaStr = '],' . PHP_EOL . '"metadata":' . json_encode($metadata); | |
Swoole\Coroutine\System::fwrite($fp, $metaStr); | |
Swoole\Coroutine\System::fwrite($fp, PHP_EOL . '}' . PHP_EOL); | |
fclose($fp); | |
$memoryUsed = memoryStatsHuman(false); | |
$endTime = microtime(true); | |
$timeTaken = $endTime - $startTime; | |
echo 'Actual memory usage: ', $memoryUsed, PHP_EOL; | |
echo 'Total time taken: ', $timeTaken, ' seconds', PHP_EOL; | |
}); | |
/** | |
Allocated memory: 2048 MB | |
Before loop memory usage: 1261.5859375 MB | |
Actual memory usage: 2946.8125 MB | |
Total time taken: 1587224980.5645 seconds | |
*/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment