Skip to content

Instantly share code, notes, and snippets.

@santaklouse

santaklouse/AddressParser.php

Created September 21, 2018 15:01
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save santaklouse/844e0ca0b090f1cfb41c75324685714a to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save santaklouse/844e0ca0b090f1cfb41c75324685714a to your computer and use it in GitHub Desktop.
Download ZIP
Example of XML parsing on PHP (Laravel)
Raw
AddressParser.php
<?php
namespace App\Parsers;
class AddressParser
{
private static $uniqueInstance = null;
protected function __construct()
{
}
final private function __clone()
{
}
public static function getInstance()
{
if (self::$uniqueInstance === null) {
self::$uniqueInstance = new self;
}
return self::$uniqueInstance;
}
private $_region_signs = [
'ОБЛ.',
'ОБЛАСТЬ',
];
private $_district_signs = [
'РАЙОН',
'Р-Н'
];
private $_city_signs = [
'М.',
'МІСТО',
'С.',
'Х.',
'СЕЛО',
'СЕЛИЩЕ',
'СЕЛ.',
'С-ЩЕ',
'ПРОВ.',
'СМТ',
'CMT.'
];
private $_street_signs = [
'ПЛОЩА',
'ПРОСПЕКТ',
'ПРОСП.',
'ПР-Т',
'БУЛЬВАР',
'БУЛ.',
'ШОСЕ',
'ВУЛИЦЯ',
'ВУЛ',
'ПРОВ',
'ПРОМЗОНА',
'ШАХТА',
'В\'ЇЗД',
'ПРОЇЗД',
'НАБЕРЕЖНА',
'ПЛ.',
'ПР.'
];
private $_probabilities = [
'region' => [3, 4, 5],
'district' => [4, 3],
'city' => [1, 2, 3, 4, 5],
'street' => [1, 2, 3]
];
public $notFullyFixed = [];
private function detector($name, $text) {
foreach($this->{'_'.$name.'_signs'} as $variant) {
if (mb_strpos(mb_strtoupper($text), $variant) !== false) {
return true;
}
}
}
public static function parseErrors()
{
return self::getInstance()->notFullyFixed;
}
private function _detectAddressPart($parts, $reversed = false)
{
//format <name> => <index in parts>
$result = ['postcode' => 0];
foreach($this->_probabilities as $name => $indexes) {
foreach($indexes as $index)
{
if (!array_key_exists($index, $parts)) {
continue;
}
if ($this->detector($name, $parts[$index]) && !in_array($index, $result)) {
$result[$name] = $index;
}
}
}
if (count($result) !== (count($this->_probabilities) + 1))
{
$this->notFullyFixed = [
'parts' => array_diff(
array_keys($this->_probabilities),
array_keys($result)
),
'orig_str' => join(';', $parts)
];
}
foreach($result as $name => &$index) {
if ($name == 'postcode') {
$index = $parts[$index];
continue;
}
$tmp = str_replace(
$this->{'_'.$name.'_signs'},
'',
$parts[$index]
);
$tmp = str_replace(array('.', ';', '-'), '', $tmp);
$tmp = preg_replace('/\s+/', ' ', $tmp);
$index = trim($tmp);
if ($name == 'region' && strpos($index, ' ')) {
$tmp = explode(' ', $index);
if (!is_numeric($tmp[0])) {
continue;
}
$result['postcode'] = $tmp[0];
$index = $tmp[1];
}
}
return $result;
}
public function main($address)
{
$this->notFullyFixed = [];
$parts = explode(',', $address);
array_walk($parts, function(&$part) {
$part = mb_strtoupper(trim($part));
});
$reversed = false;
if (end($parts) !== 'УКРАЇНА') {
$index = reset($parts);
$parts = array_reverse(array_splice($parts, 1));
array_unshift($parts, $index);
$reversed = true;
}
if (end($parts) == 'УКРАЇНА') {
array_pop($parts);
}
return $this->_detectAddressPart($parts, $reversed);
}
public static function parse($address)
{
/*
* Example of address formats
*
* '01196, м.Київ, Печерський район, ПЛОЩА ЛЕСІ УКРАЇНКИ, будинок 1, кімната 310',
'09600, Київська обл., Рокитнянський район, селище міського типу Рокитне, ВУЛИЦЯ ВОКЗАЛЬНА, будинок 3',
'61002, ВУЛ.ПЕТРОВСЬКОГО, 7, М. ХАРКІВ, КИЇВСЬКИЙ РАЙОН, УКРАЇНА',
'71500, ПРОМЗОНА А/С 33, М. ЕНЕРГОДАР, ЗАПОРІЗЬКА ОБЛАСТЬ, УКРАЇНА',
'42000, Сумська обл., місто Ромни, ВУЛИЦЯ КОРЖІВСЬКА, будинок 100',
'13643, , С.НЕМИРИНЦІ, РУЖИНСЬКИЙ РАЙОН, ЖИТОМИРСЬКА ОБЛАСТЬ, УКРАЇНА',
'68600, ВУЛ.МІЧУРІНА, 9, М.ІЗМАЇЛ, ОДЕСЬКА ОБЛАСТЬ, УКРАЇНА',
'82100, ВУЛ. ТУРАША, 28, М. ДРОГОБИЧ, ДРОГОБИЦЬКИЙ РАЙОН, ЛЬВІВСЬКА ОБЛАСТЬ, УКРАЇНА',
'84100, Донецька обл., місто Слов&apos;янськ, ВУЛИЦЯ КАРЛА МАРКСА, будинок 59',
'15300, ПРОВ.ІНДУСТРІАЛЬНИЙ, 1, М.КОРЮКІВКА, КОРЮКІВСЬКИЙ РАЙОН, ЧЕРНІГІВСЬКА ОБЛАСТЬ, УКРАЇНА'
*/
return self::getInstance()->main($address);
}
}
Raw
App\Console\Commands\XmlParser.php
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Parsers\SimpleXmlParser;
use App\Parsers\AddressParser;
use Modules\Index\Entities\Companies;
class XmlParser extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'xml:parse {path}';
/**
* The console command description.
*
* @var string
*/
protected $description = '';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return mixed
*/
public function handle()
{
$this->info($this->xml2DB($this->argument('path')));
}
private function progress_bar($done, $total, $info="", $width = 50) {
$perc = round(($done * 100) / $total);
$bar = round(($width * $perc) / 100);
return sprintf("%s%%[%s>%s]%s\r", $perc, str_repeat("=", $bar), str_repeat(" ", $width-$bar), $info);
}
private function filesize($url)
{
if (!filter_var($url, FILTER_VALIDATE_URL)) {
return filesize($url);
}
$x = array_change_key_case(get_headers($url, 1),CASE_LOWER);
if (strcasecmp($x[0], 'HTTP/1.1 200 OK') != 0) {
$fileSize = $x['content-length'][1];
} else {
$fileSize = $x['content-length'];
}
return $fileSize;
}
private function human_filesize($bytes, $decimals = 2) {
$sz = 'BKMGTP';
$factor = floor((strlen($bytes) - 1) / 3);
return sprintf("%.{$decimals}f", $bytes / pow(1024, $factor)) . @$sz[$factor];
}
public static function parseActivity($code)
{
if (!$code) {
return null;
}
$code = explode(' ', $code);
return [
'code' => reset($code),
'name' => implode(' ', array_slice($code, 1, count($code)))
];
}
public static function parseAddress($address)
{
if (!$address) {
return null;
}
return AddressParser::parse($address);
}
private function xml2DB($url)
{
ini_set('memory_limit', '100M');
ob_implicit_flush(true);
$start = microtime(true);
try {
$streamer = new SimpleXmlParser(
$url,
16384,
null,
floor($this->filesize($url))
);
} catch (\Exception $e) {
die($e->getMessage());
}
$i = 0;
$all = 0;
$duplicates = [];
$streamer->retrieve(100, function($items) use ($streamer, &$i, &$all, &$duplicates, $start)
{
foreach ($items as &$item)
{
$item['address'] = trim($item['address']);
$item['address_parsed'] = $item['address'] ? AddressParser::parse($item['address']) : null;
$item['activity_type'] = self::parseActivity(trim($item['activity_type']));
if (is_array($item['address']) && $item['address'])
{
$item['address']['errors'] = AddressParser::parseErrors();
}
}
Companies::addChunk($items);
$i += count($items);
$totalBytes = $streamer->getTotalBytes();
$readBytes = $streamer->getReadBytes();
print $this->progress_bar(
$readBytes,
$totalBytes,
$this->human_filesize($readBytes) .'/'. $this->human_filesize($totalBytes)
. '(parsed: ' . $i . ' speed:' . intval($i / (microtime(true) - $start)) .' items/sec)',
80
);
});
$time_elapsed_secs = microtime(true) - $start;
$time_elapsed_secs = $time_elapsed_secs > 120
? float($time_elapsed_secs / 60) . "min"
: $time_elapsed_secs."sec";
print PHP_EOL . "Duplicates: $all" . PHP_EOL;
print PHP_EOL . "Parsing XML completed. Total time: $time_elapsed_secs.Items extracted: $i";
}
}
Raw
App\Parsers\SimpleXmlParser.php
<?php
namespace App\Parsers;
use Prewk\XmlStreamer;
class SimpleXmlParser extends XmlStreamer
{
protected $pdo;
protected $sql = array();
protected $values = array();
private $_ccb;
private $_chunkItems = [];
private $_chunkItemsSize = 50;
private $_db2XmlFormatMap = [
'edrpou' => 'EDRPOU',
'name' => 'NAME',
'short_name' => 'SHORT_NAME',
'address' => 'ADDRESS',
'owner' => 'BOSS',
'activity_type' => 'KVED',
'status' => 'STAN'
];
public function CP1250ToUtf8($text) {
// map based on:
// http://konfiguracja.c0.pl/iso02vscp1250en.html
// http://konfiguracja.c0.pl/webpl/index_en.html#examp
// http://www.htmlentities.com/html/entities/
$map = array(
chr(0x8A) => chr(0xA9),
chr(0x8C) => chr(0xA6),
chr(0x8D) => chr(0xAB),
chr(0x8E) => chr(0xAE),
chr(0x8F) => chr(0xAC),
chr(0x9C) => chr(0xB6),
chr(0x9D) => chr(0xBB),
chr(0xA1) => chr(0xB7),
chr(0xA5) => chr(0xA1),
chr(0xBC) => chr(0xA5),
chr(0x9F) => chr(0xBC),
chr(0xB9) => chr(0xB1),
chr(0x9A) => chr(0xB9),
chr(0xBE) => chr(0xB5),
chr(0x9E) => chr(0xBE),
chr(0x80) => '&euro;',
chr(0x82) => '&sbquo;',
chr(0x84) => '&bdquo;',
chr(0x85) => '&hellip;',
chr(0x86) => '&dagger;',
chr(0x87) => '&Dagger;',
chr(0x89) => '&permil;',
chr(0x8B) => '&lsaquo;',
chr(0x91) => '&lsquo;',
chr(0x92) => '&rsquo;',
chr(0x93) => '&ldquo;',
chr(0x94) => '&rdquo;',
chr(0x95) => '&bull;',
chr(0x96) => '&ndash;',
chr(0x97) => '&mdash;',
chr(0x99) => '&trade;',
chr(0x9B) => '&rsquo;',
chr(0xA6) => '&brvbar;',
chr(0xA9) => '&copy;',
chr(0xAB) => '&laquo;',
chr(0xAE) => '&reg;',
chr(0xB1) => '&plusmn;',
chr(0xB5) => '&micro;',
chr(0xB6) => '&para;',
chr(0xB7) => '&middot;',
chr(0xBB) => '&raquo;',
);
return html_entity_decode(
mb_convert_encoding(strtr($text, $map), 'UTF-8', ['CP1251', 'CP1250']),
ENT_QUOTES,
'utf-8'
);
}
public function processNode($xmlString, $elementName, $nodeIndex)
{
$xmlString = iconv('CP1251', 'UTF-8', $xmlString);
$xmlString = preg_replace('/&(?!#?[a-z0-9]+;)/', '&amp;', $xmlString);
$xml = simplexml_load_string($xmlString);
$result = [];
foreach($xml->children() as $name => $content) {
foreach ($this->_db2XmlFormatMap as $db_key => $xml_key) {
$result[$db_key] = $this->CP1250ToUtf8(
mb_convert_encoding((string)$xml->{$xml_key}, 'CP1251', 'UTF-8')
);
}
}
$this->_chunkItems[] = $result;
if (count($this->_chunkItems) >= $this->_chunkItemsSize) {
if ($this->_ccb) {
call_user_func($this->_ccb, $this->_chunkItems);
}
$this->_chunkItems = [];
}
return true;
}
public function retrieve($chunk_size, $ccb)
{
$this->_chunkItemsSize = $chunk_size;
$this->_ccb = $ccb;
$this->parse();
}
}
Raw
Modules\Index\Entities\Companies.php
<?php
namespace Modules\Index\Entities;
use Generator;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Facades\DB;
use League\Csv\Reader;
use Modules\Index\Entities\Companies\ActivityTypes;
use Modules\Index\Entities\Address\Cities;
use Modules\Index\Entities\Address\Districts;
use Modules\Index\Entities\Address\Regions;
use Modules\Index\Entities\Address\Streets;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Arr;
class Companies extends Model
{
public $connection = 'mysql';
protected $table = 'companies';
public function activity_types()
{
return $this->hasOne(ActivityTypes::class, 'activity_type_id');
}
public function current()
{
try {
return app()->current_session;
}
catch (\ReflectionException $e) {
return false;
}
}
private static function _addAddresses($addresses)
{
/*
* create regions
*/
$regions = Regions::createIfNotExists($addresses);
/*
* create districts
*/
$districts = Districts::createIfNotExists($addresses, $regions);
/*
* create cities
*/
$cities = Cities::createIfNotExists($addresses, $regions, $districts);
/*
* create streets
*/
$streets = Streets::createIfNotExists(
$addresses,
$cities,
$regions,
$districts
);
return [
$regions,
$districts,
$cities,
$streets
];
}
public static function addChunk($items = array())
{
$activity_types = collect($items)->pluck('activity_type')->reject(function ($item) {
return is_null($item);
})->unique('code')->sortBy('code');
$types = ActivityTypes::createIfNotExists($activity_types);
list($regions, $districts, $cities, $streets) = self::_addAddresses(collect($items)->pluck('address_parsed')->filter());
//get activity types
//fetch --||-- from bd
//add missing types to DB
//we have DB ids of types
//get edrpou's
//fetch --||-- from DB
//find duplicates
//chose which records need to add or update
$companies_chunk = collect($items)->unique('edrpou')->sortBy('edrpou');
$companies = Companies::whereIn('edrpou', $companies_chunk->pluck('edrpou'))->get();
/*
* Companies table columns list:
*
edrpou
name
owner_name
activity_type_id
status
address
address_postcode
address_region_id
address_city_id
address_district_id
address_street_id
*/
$originalDataToDbMap = [
'edrpou' => '',
'name' => '',
'short_name' => '',
'owner_name' => 'owner',
'status' => '',
'address' => '',
];
$toCreate = $companies_chunk->map(function($item) use (
$originalDataToDbMap,
$types,
$regions,
$districts,
$cities,
$streets
) {
$result = [];
foreach ($originalDataToDbMap as $dbColName => $xmlColName)
{
$result[$dbColName] = $item[$xmlColName ?: $dbColName];
}
$activity_type_id = Arr::get($item, 'activity_type.code', null);
if ($activity_type_id) {
$activity_type_id = $types->firstWhere(
'code',
$activity_type_id
)->id;
}
$region_id = $regions->where(
'name',
Arr::get($item, 'address_parsed.region', null)
)->first();
$region_id = $region_id ? $region_id->id : null;
$district = Arr::get($item, 'address_parsed.district', null);
$district_id = $districts
->where('name', $district)
;
$tmp = $district_id->where('region_id', $region_id)->first();
if ($tmp) {
$district_id = $tmp;
} else {
$district_id = $district_id->first();
}
$district_id = $district_id ? $district_id->id : null;
$city = Arr::get($item, 'address_parsed.city', null);
$city_id = $cities
->where('name', $city)
->where('region_id', $region_id)
->first()
;
$city_id = $city_id ? $city_id->id : null;
$street = Arr::get($item, 'address_parsed.street', null);
$street_id = $streets
->where('name', $street)
->where('city_id', $city_id)
->first()
;
$street_id = $street_id ? $street_id->id : null;
$postcode = Arr::get($item, 'address_parsed.postcode', null);
return array_merge($result, [
'activity_type_id' => $activity_type_id,
'address_postcode' => $postcode,
'address_region_id' => $region_id,
'address_district_id' => $district_id ?: null,
'address_city_id' => $city_id,
'address_street_id' => $street_id
]);
});
if ($toCreate->count()) {
Companies::insert($toCreate->toArray());
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment