Skip to content

Instantly share code, notes, and snippets.

@LogIN-
Created October 4, 2014 11:19
Show Gist options
  • Select an option

  • Save LogIN-/0f5fc6f222adbb376967 to your computer and use it in GitHub Desktop.

Select an option

Save LogIN-/0f5fc6f222adbb376967 to your computer and use it in GitHub Desktop.
PHP detect UTF8 && arabic encoding
<?php
function detect_utf_encoding($data) {
$UTF32_BIG_ENDIAN_BOM = chr(0x00) . chr(0x00) . chr(0xFE) . chr(0xFF);
$UTF32_LITTLE_ENDIAN_BOM = chr(0xFF) . chr(0xFE) . chr(0x00) . chr(0x00);
$UTF16_BIG_ENDIAN_BOM = chr(0xFE) . chr(0xFF);
$UTF16_LITTLE_ENDIAN_BOM = chr(0xFF) . chr(0xFE);
$UTF8_BOM = chr(0xEF) . chr(0xBB) . chr(0xBF);
$first2 = substr($data, 0, 2);
$first3 = substr($data, 0, 3);
$first4 = substr($data, 0, 3);
if ($first3 == $UTF8_BOM){
return 'UTF-8';
}else if($first4 == $UTF32_BIG_ENDIAN_BOM){
return 'UTF-32BE';
}else if($first4 == $UTF32_LITTLE_ENDIAN_BOM){
return 'UTF-32LE';
}else if($first2 == $UTF16_BIG_ENDIAN_BOM){
return 'UTF-16BE';
}else if($first2 == $UTF16_LITTLE_ENDIAN_BOM){
return 'UTF-16LE';
}else{
return false;
}
};
function is_arabic($str) {
if(mb_detect_encoding($str) !== 'UTF-8') {
$str = mb_convert_encoding($str,mb_detect_encoding($str),'UTF-8');
}
/*
$str = str_split($str); <- this function is not mb safe, it splits by bytes, not characters. we cannot use it
$str = preg_split('//u',$str); <- this function woulrd probably work fine but there was a bug reported in some php version so it pslits by bytes and not chars as well
*/
preg_match_all('/.|\n/u', $str, $matches);
$chars = $matches[0];
$arabic_count = 0;
$latin_count = 0;
$total_count = 0;
foreach($chars as $char) {
//$pos = ord($char); we cant use that, its not binary safe
$pos = uniord($char);
if($pos >= 1536 && $pos <= 1791) {
$arabic_count++;
} else if($pos > 123 && $pos < 123) {
$latin_count++;
}
$total_count++;
}
if(($arabic_count/$total_count) > 0.6) {
// 60% arabic chars, its probably arabic
return true;
}
return false;
}
?>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment