Last active
January 6, 2024 21:50
-
-
Save connor11528/a36676d3239884bd3193f1defef8ca57 to your computer and use it in GitHub Desktop.
Artisan command to scrape jobs from Greenhouse.io
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
namespace App\Console\Commands; | |
use App\Company; | |
use App\JobListing; | |
use Illuminate\Console\Command; | |
use Goutte\Client; | |
class ScrapeGreenhouse extends Command | |
{ | |
/** | |
* The name and signature of the console command. | |
* | |
* @var string | |
*/ | |
protected $signature = 'scrape:greenhouse'; | |
/** | |
* The console command description. | |
* | |
* @var string | |
*/ | |
protected $description = 'Scrape information from greenhouse dot com'; | |
/** | |
* Create a new command instance. | |
* | |
* @return void | |
*/ | |
public function __construct() | |
{ | |
parent::__construct(); | |
} | |
/** | |
* Execute the console command. | |
* | |
* @return mixed | |
*/ | |
public function handle() | |
{ | |
$endpoints = [ | |
[ | |
'name' => 'Stitch Labs', | |
'endpoint' => 'stitchlabs' | |
], | |
[ | |
'name' => 'Lob', | |
'endpoint' => 'lob' | |
], | |
[ | |
'name' => 'DoorDash', | |
'endpoint' => 'doordash' | |
], | |
[ | |
'name' => 'Instacart', | |
'endpoint' => 'instacart' | |
], | |
[ | |
'name' => 'Sift Science', | |
'endpoint' => 'siftscience' | |
], | |
]; | |
foreach ($endpoints as $endpoint) { | |
/** @var Company $company */ | |
$company = Company::where('name', $endpoint['name'])->first(); | |
$endpoint['company_id'] = $company->id; | |
$this->scrape($endpoint); | |
} | |
} | |
/** | |
* @param $endpoint array | |
* @return bool | |
*/ | |
public function scrape($endpoint) | |
{ | |
$crawler = new Client(); | |
$result = $crawler->request('GET', 'https://boards.greenhouse.io/' . $endpoint['endpoint']); | |
$result->filter('.opening')->each(function ($node) use ($endpoint) { | |
$jobListing = new JobListing; | |
$jobListing->title = $node->filter('a')->text(); | |
$jobListing->url = $node->filter('a')->attr('href'); | |
$jobListing->city = $node->filter('.location')->text(); | |
$jobListing->company_id = $endpoint['company_id']; | |
$jobListing->active = true; | |
$jobListing->save(); | |
$this->info('Added job listing: ' . $jobListing->title . ' for ' . $endpoint['name'] . "\n"); | |
}); | |
return true; | |
} | |
} |
Run with php artisan scrape:greenhouse
. The Goutte issue above had to do with my composer dependencies and composer.lock file. Code should be all good
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Currently having issue with deploy to prod:
Class 'Weidner\Goutte\GoutteServiceProvider' not found
dweidner/laravel-goutte#6
dweidner/laravel-goutte#12
Works locally tho...