-
-
Save whoami15/814979d7a403e3eb35e327eb85ae68a1 to your computer and use it in GitHub Desktop.
A simple Laravel job that can help you export data through CSV from a database with millions of rows.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
namespace App\Jobs; | |
use App\Models\GeneralExport; | |
use Storage; | |
class CreateGeneralExportFileJob implements ShouldQueue | |
{ | |
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; | |
public $timeout = 1200; | |
/** | |
* Create a new job instance. | |
* | |
* @return void | |
*/ | |
public function __construct( | |
private GeneralExport $export, | |
private string $exportFileName, | |
private int $page = 1, | |
) | |
{ | |
// | |
} | |
/** | |
* Execute the job. | |
* | |
* @return void | |
*/ | |
public function handle() | |
{ | |
$members = $this->getMembers(); | |
$columns = [ | |
'Member ID', | |
'Full Name', | |
'Phone Number', | |
'Gender', | |
'Date of Birth', | |
'Email', | |
]; | |
$filesystemAdapter = Storage::disk('public'); | |
if($this->export->status === 'pending') { | |
$fileName = 'general_exports/' . Carbon::now()->timestamp . '-' . $this->exportFileName . '-' . $this->export->user_id . '.csv'; | |
// add the headers only on the first run of this job... on subsequent runs, only append the data | |
$filesystemAdapter->append($fileName, implode(',', $columns) . PHP_EOL); | |
} else { | |
$fileName = $this->exportFileName; | |
} | |
if($this->export->status !== 'processing') { | |
$this->export->update([ | |
'status' => 'processing', | |
'status_message' => "Job {$this->page} in export processing started" | |
]); | |
} elseif($this->export->status === 'processing') { | |
$this->export->update([ | |
'status_message' => "Job {$this->page} in export processing started" | |
]); | |
} | |
$fileResource = fopen($filesystemAdapter->path($fileName), 'a+'); | |
foreach ($members as $member) { | |
fwrite($fileResource, implode(',', [ | |
$member->id, | |
$member->user->first_name . ' ' . $member->user->last_name, | |
$member->user->phone, | |
$member->gender, | |
$member->dob, | |
$member->user->email, | |
]) . PHP_EOL); | |
} | |
fclose($fileResource); | |
$nextPageUrl = $members->nextPageUrl(); | |
$nextPage = null; | |
if(!is_null($nextPageUrl)) { | |
$nextPage = explode('=', $nextPageUrl, 2)[1]; | |
} | |
if(is_null($nextPage)) { | |
// we are done processing | |
$this->export->update([ | |
'status' => 'processed', | |
'status_message' => 'Export file processed successfully and ready for download', | |
'file' => $fileName, | |
]); | |
return; | |
} | |
// refresh to get current state of export before using it for next job | |
$this->export->refresh(); | |
dispatch(new static($this->export, $fileName, $nextPage)); | |
} | |
public function getMembers() { | |
return Member::paginate(10000, ['*'], 'page', $this->page) | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
namespace App\Models; | |
use Illuminate\Database\Eloquent\Factories\HasFactory; | |
use Illuminate\Database\Eloquent\Model; | |
class GeneralExport extends Model | |
{ | |
use HasFactory; | |
protected $fillable = ['user_id', 'status', 'file', 'status_message']; | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment