bitPickups/friendica_2020-09-1_sharedHosting
1
0
Fork 0
Code Pull requests Activity

Add pear/Text_LanguageDetect to Composer/vendor

Browse source 
- Updated mobiledetectlib to 2.8.26
- Updated PEAR to 1.10.5
- Updated Archive_Tar to 1.4.3
- Updated XML_Util to 1.4.3
This commit is contained in:
Hypolite Petovan 2017-11-05 08:34:02 -05:00
commit 7d876aab98
109 changed files with 6517 additions and 243 deletions
Download patch file Download diff file Expand all files Collapse all files

1678
vendor/pear/text_languagedetect/Text/LanguageDetect.php vendored Normal file
View file

@ -0,0 +1,1678 @@
<?php
/**
* Part of Text_LanguageDetect
*
* PHP version 5
*
* @category Text
* @package Text_LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @copyright 2005-2006 Nicholas Pisarro
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @link http://pear.php.net/package/Text_LanguageDetect/
*/
require_once 'Text/LanguageDetect/Exception.php';
require_once 'Text/LanguageDetect/Parser.php';
require_once 'Text/LanguageDetect/ISO639.php';
/**
* Detects the language of a given piece of text.
*
* Attempts to detect the language of a sample of text by correlating ranked
* 3-gram frequencies to a table of 3-gram frequencies of known languages.
*
* Implements a version of a technique originally proposed by Cavnar & Trenkle
* (1994): "N-Gram-Based Text Categorization"
*
* Requires the language model database (lang.dat) that should have
* accompanied this class definition in order to be instantiated.
*
* Example usage:
*
* <code>
* require_once 'Text/LanguageDetect.php';
*
* $l = new Text_LanguageDetect;
*
* $stdin = fopen('php://stdin', 'r');
*
* echo "Supported languages:\n";
*
* try {
* $langs = $l->getLanguages();
* } catch (Text_LanguageDetect_Exception $e) {
* die($e->getMessage());
* }
*
* sort($langs);
* echo join(', ', $langs);
*
* while ($line = fgets($stdin)) {
* print_r($l->detect($line, 4));
* }
* </code>
*
* @category Text
* @package Text_LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @copyright 2005 Nicholas Pisarro
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @version Release: @package_version@
* @link http://pear.php.net/package/Text_LanguageDetect/
*/
class Text_LanguageDetect
{
/**
* The filename that stores the trigram data for the detector
*
* If this value starts with a slash (/) or a dot (.) the value of
* $this->_data_dir will be ignored
*
* @var string
*/
protected $_db_filename = 'lang.dat';
/**
* The filename that stores the unicode block definitions
*
* If this value starts with a slash (/) or a dot (.) the value of
* $this->_data_dir will be ignored
*
* @var string
*/
protected $_unicode_db_filename = 'unicode_blocks.dat';
/**
* The data directory
*
* Should be set by PEAR installer
*
* @var string
*/
protected $_data_dir = '@data_dir@';
/**
* The trigram data for comparison
*
* Will be loaded on start from $this->_db_filename
*
* @var array
*/
protected $_lang_db = array();
/**
* Stores the map of the trigram data to unicode characters
*
* @var array
*/
protected $_unicode_map;
/**
* The size of the trigram data arrays
*
* @var int
*/
protected $_threshold = 300;
/**
* The maximum possible score.
*
* Needed for score normalization. Different depending on the
* perl compatibility setting
*
* @var int
* @see setPerlCompatible()
*/
protected $_max_score = 0;
/**
* Whether or not to simulate perl's Language::Guess exactly
*
* @var bool
* @see setPerlCompatible()
*/
protected $_perl_compatible = false;
/**
* Whether to use the unicode block detection to speed up processing
*
* @var bool
*/
protected $_use_unicode_narrowing = true;
/**
* Stores the result of the clustering operation
*
* @var array
* @see clusterLanguages()
*/
protected $_clusters;
/**
* Which type of "language names" are accepted and returned:
*
* 0 - language name ("english")
* 2 - 2-letter ISO 639-1 code ("en")
* 3 - 3-letter ISO 639-2 code ("eng")
*/
protected $_name_mode = 0;
/**
* Constructor
*
* Will attempt to load the language database. If it fails, you will get
* an exception.
*/
public function __construct()
{
$data = $this->_readdb($this->_db_filename);
$this->_checkTrigram($data['trigram']);
$this->_lang_db = $data['trigram'];
if (isset($data['trigram-unicodemap'])) {
$this->_unicode_map = $data['trigram-unicodemap'];
}
// Not yet implemented:
if (isset($data['trigram-clusters'])) {
$this->_clusters = $data['trigram-clusters'];
}
}
/**
* Returns the path to the location of the database
*
* @param string $fname File name to load
*
* @return string expected path to the language model database
*/
protected function _get_data_loc($fname)
{
if ($fname{0} == '/' || $fname{0} == '.') {
// if filename starts with a slash, assume it's an absolute pathname
// and skip whatever is in $this->_data_dir
return $fname;
} elseif ($this->_data_dir != '@' . 'data_dir' . '@') {
// if the data dir was set by the PEAR installer, use that
return $this->_data_dir . '/Text_LanguageDetect/' . $fname;
} else {
// assume this was just unpacked somewhere
// try the local working directory if otherwise
return __DIR__ . '/../data/' . $fname;
}
}
/**
* Loads the language trigram database from filename
*
* Trigram datbase should be a serialize()'d array
*
* @param string $fname the filename where the data is stored
*
* @return array the language model data
* @throws Text_LanguageDetect_Exception
*/
protected function _readdb($fname)
{
// finds the correct data dir
$fname = $this->_get_data_loc($fname);
// input check
if (!file_exists($fname)) {
throw new Text_LanguageDetect_Exception(
'Language database does not exist: ' . $fname,
Text_LanguageDetect_Exception::DB_NOT_FOUND
);
} elseif (!is_readable($fname)) {
throw new Text_LanguageDetect_Exception(
'Language database is not readable: ' . $fname,
Text_LanguageDetect_Exception::DB_NOT_READABLE
);
}
return unserialize(file_get_contents($fname));
}
/**
* Checks if this object is ready to detect languages
*
* @param array $trigram Trigram data from database
*
* @return void
*/
protected function _checkTrigram($trigram)
{
if (!is_array($trigram)) {
if (ini_get('magic_quotes_runtime')) {
throw new Text_LanguageDetect_Exception(
'Error loading database. Try turning magic_quotes_runtime off.',
Text_LanguageDetect_Exception::MAGIC_QUOTES
);
}
throw new Text_LanguageDetect_Exception(
'Language database is not an array.',
Text_LanguageDetect_Exception::DB_NOT_ARRAY
);
} elseif (empty($trigram)) {
throw new Text_LanguageDetect_Exception(
'Language database has no elements.',
Text_LanguageDetect_Exception::DB_EMPTY
);
}
}
/**
* Omits languages
*
* Pass this function the name of or an array of names of
* languages that you don't want considered
*
* If you're only expecting a limited set of languages, this can greatly
* speed up processing
*
* @param mixed $omit_list language name or array of names to omit
* @param bool $include_only if true will include (rather than
* exclude) only those in the list
*
* @return int number of languages successfully deleted
* @throws Text_LanguageDetect_Exception
*/
public function omitLanguages($omit_list, $include_only = false)
{
$deleted = 0;
$omit_list = $this->_convertFromNameMode($omit_list);
if (!$include_only) {
// deleting the given languages
if (!is_array($omit_list)) {
$omit_list = strtolower($omit_list); // case desensitize
if (isset($this->_lang_db[$omit_list])) {
unset($this->_lang_db[$omit_list]);
$deleted++;
}
} else {
foreach ($omit_list as $omit_lang) {
if (isset($this->_lang_db[$omit_lang])) {
unset($this->_lang_db[$omit_lang]);
$deleted++;
}
}
}
} else {
// deleting all except the given languages
if (!is_array($omit_list)) {
$omit_list = array($omit_list);
}
// case desensitize
foreach ($omit_list as $key => $omit_lang) {
$omit_list[$key] = strtolower($omit_lang);
}
foreach (array_keys($this->_lang_db) as $lang) {
if (!in_array($lang, $omit_list)) {
unset($this->_lang_db[$lang]);
$deleted++;
}
}
}
// reset the cluster cache if the number of languages changes
// this will then have to be recalculated
if (isset($this->_clusters) && $deleted > 0) {
$this->_clusters = null;
}
return $deleted;
}
/**
* Returns the number of languages that this object can detect
*
* @return int the number of languages
* @throws Text_LanguageDetect_Exception
*/
public function getLanguageCount()
{
return count($this->_lang_db);
}
/**
* Checks if the language with the given name exists in the database
*
* @param mixed $lang Language name or array of language names
*
* @return bool true if language model exists
*/
public function languageExists($lang)
{
$lang = $this->_convertFromNameMode($lang);
if (is_string($lang)) {
return isset($this->_lang_db[strtolower($lang)]);
} elseif (is_array($lang)) {
foreach ($lang as $test_lang) {
if (!isset($this->_lang_db[strtolower($test_lang)])) {
return false;
}
}
return true;
} else {
throw new Text_LanguageDetect_Exception(
'Unsupported parameter type passed to languageExists()',
Text_LanguageDetect_Exception::PARAM_TYPE
);
}
}
/**
* Returns the list of detectable languages
*
* @return array the names of the languages known to this object<<<<<<<
* @throws Text_LanguageDetect_Exception
*/
public function getLanguages()
{
return $this->_convertToNameMode(
array_keys($this->_lang_db)
);
}
/**
* Make this object behave like Language::Guess
*
* @param bool $setting false to turn off perl compatibility
*
* @return void
*/
public function setPerlCompatible($setting = true)
{
if (is_bool($setting)) { // input check
$this->_perl_compatible = $setting;
if ($setting == true) {
$this->_max_score = $this->_threshold;
} else {
$this->_max_score = 0;
}
}
}
/**
* Sets the way how language names are accepted and returned.
*
* @param integer $name_mode One of the following modes:
* 0 - language name ("english")
* 2 - 2-letter ISO 639-1 code ("en")
* 3 - 3-letter ISO 639-2 code ("eng")
*
* @return void
*/
public function setNameMode($name_mode)
{
$this->_name_mode = $name_mode;
}
/**
* Whether to use unicode block ranges in detection
*
* Should speed up most detections if turned on (detault is on). In some
* circumstances it may be slower, such as for large text samples (> 10K)
* in languages that use latin scripts. In other cases it should speed up
* detection noticeably.
*
* @param bool $setting false to turn off
*
* @return void
*/
public function useUnicodeBlocks($setting = true)
{
if (is_bool($setting)) {
$this->_use_unicode_narrowing = $setting;
}
}
/**
* Converts a piece of text into trigrams
*
* @param string $text text to convert
*
* @return array array of trigram frequencies
* @deprecated Superceded by the Text_LanguageDetect_Parser class
*/
protected function _trigram($text)
{
$s = new Text_LanguageDetect_Parser($text);
$s->prepareTrigram();
$s->prepareUnicode(false);
$s->setPadStart(!$this->_perl_compatible);
$s->analyze();
return $s->getTrigramFreqs();
}
/**
* Converts a set of trigrams from frequencies to ranks
*
* Thresholds (cuts off) the list at $this->_threshold
*
* @param array $arr array of trigram
*
* @return array ranks of trigrams
*/
protected function _arr_rank($arr)
{
// sorts alphabetically first as a standard way of breaking rank ties
$this->_bub_sort($arr);
// below might also work, but seemed to introduce errors in testing
//ksort($arr);
//asort($arr);
$rank = array();
$i = 0;
foreach ($arr as $key => $value) {
$rank[$key] = $i++;
// cut off at a standard threshold
if ($i >= $this->_threshold) {
break;
}
}
return $rank;
}
/**
* Sorts an array by value breaking ties alphabetically
*
* @param array $arr the array to sort
*
* @return void
*/
protected function _bub_sort(&$arr)
{
// should do the same as this perl statement:
// sort { $trigrams{$b} == $trigrams{$a}
// ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} }
// needs to sort by both key and value at once
// using the key to break ties for the value
// converts array into an array of arrays of each key and value
// may be a better way of doing this
$combined = array();
foreach ($arr as $key => $value) {
$combined[] = array($key, $value);
}
usort($combined, array($this, '_sort_func'));
$replacement = array();
foreach ($combined as $key => $value) {
list($new_key, $new_value) = $value;
$replacement[$new_key] = $new_value;
}
$arr = $replacement;
}
/**
* Sort function used by bubble sort
*
* Callback function for usort().
*
* @param array $a first param passed by usort()
* @param array $b second param passed by usort()
*
* @return int 1 if $a is greater, -1 if not
* @see _bub_sort()
*/
protected function _sort_func($a, $b)
{
// each is actually a key/value pair, so that it can compare using both
list($a_key, $a_value) = $a;
list($b_key, $b_value) = $b;
if ($a_value == $b_value) {
// if the values are the same, break ties using the key
return strcmp($a_key, $b_key);
} else {
// if not, just sort normally
if ($a_value > $b_value) {
return -1;
} else {
return 1;
}
}
// 0 should not be possible because keys must be unique
}
/**
* Calculates a linear rank-order distance statistic between two sets of
* ranked trigrams
*
* Sums the differences in rank for each trigram. If the trigram does not
* appear in both, consider it a difference of $this->_threshold.
*
* This distance measure was proposed by Cavnar & Trenkle (1994). Despite
* its simplicity it has been shown to be highly accurate for language
* identification tasks.
*
* @param array $arr1 the reference set of trigram ranks
* @param array $arr2 the target set of trigram ranks
*
* @return int the sum of the differences between the ranks of
* the two trigram sets
*/
protected function _distance($arr1, $arr2)
{
$sumdist = 0;
foreach ($arr2 as $key => $value) {
if (isset($arr1[$key])) {
$distance = abs($value - $arr1[$key]);
} else {
// $this->_threshold sets the maximum possible distance value
// for any one pair of trigrams
$distance = $this->_threshold;
}
$sumdist += $distance;
}
return $sumdist;
// todo: there are other distance statistics to try, e.g. relative
// entropy, but they're probably more costly to compute
}
/**
* Normalizes the score returned by _distance()
*
* Different if perl compatible or not
*
* @param int $score the score from _distance()
* @param int $base_count the number of trigrams being considered
*
* @return float the normalized score
* @see _distance()
*/
protected function _normalize_score($score, $base_count = null)
{
if ($base_count === null) {
$base_count = $this->_threshold;
}
if (!$this->_perl_compatible) {
return 1 - ($score / $base_count / $this->_threshold);
} else {
return floor($score / $base_count);
}
}
/**
* Detects the closeness of a sample of text to the known languages
*
* Calculates the statistical difference between the text and
* the trigrams for each language, normalizes the score then
* returns results for all languages in sorted order
*
* If perl compatible, the score is 300-0, 0 being most similar.
* Otherwise, it's 0-1 with 1 being most similar.
*
* The $sample text should be at least a few sentences in length;
* should be ascii-7 or utf8 encoded, if another and the mbstring extension
* is present it will try to detect and convert. However, experience has
* shown that mb_detect_encoding() *does not work very well* with at least
* some types of encoding.
*
* @param string $sample a sample of text to compare.
* @param int $limit if specified, return an array of the most likely
* $limit languages and their scores.
*
* @return mixed sorted array of language scores, blank array if no
* useable text was found
* @see _distance()
* @throws Text_LanguageDetect_Exception
*/
public function detect($sample, $limit = 0)
{
// input check
if (!Text_LanguageDetect_Parser::validateString($sample)) {
return array();
}
// check char encoding
// (only if mbstring extension is compiled and PHP > 4.0.6)
if (function_exists('mb_detect_encoding')
&& function_exists('mb_convert_encoding')
) {
// mb_detect_encoding isn't very reliable, to say the least
// detection should still work with a sufficient sample
// of ascii characters
$encoding = mb_detect_encoding($sample);
// mb_detect_encoding() will return FALSE if detection fails
// don't attempt conversion if that's the case
if ($encoding != 'ASCII' && $encoding != 'UTF-8'
&& $encoding !== false
) {
// verify the encoding exists in mb_list_encodings
if (in_array($encoding, mb_list_encodings())) {
$sample = mb_convert_encoding($sample, 'UTF-8', $encoding);
}
}
}
$sample_obj = new Text_LanguageDetect_Parser($sample);
$sample_obj->prepareTrigram();
if ($this->_use_unicode_narrowing) {
$sample_obj->prepareUnicode();
}
$sample_obj->setPadStart(!$this->_perl_compatible);
$sample_obj->analyze();
$trigram_freqs = $sample_obj->getTrigramRanks();
$trigram_count = count($trigram_freqs);
if ($trigram_count == 0) {
return array();
}
$scores = array();
// use unicode block detection to narrow down the possibilities
if ($this->_use_unicode_narrowing) {
$blocks = $sample_obj->getUnicodeBlocks();
if (is_array($blocks)) {
$present_blocks = array_keys($blocks);
} else {
throw new Text_LanguageDetect_Exception(
'Error during block detection',
Text_LanguageDetect_Exception::BLOCK_DETECTION
);
}
$possible_langs = array();
foreach ($present_blocks as $blockname) {
if (isset($this->_unicode_map[$blockname])) {
$possible_langs = array_merge(
$possible_langs,
array_keys($this->_unicode_map[$blockname])
);
// todo: faster way to do this?
}
}
// could also try an intersect operation rather than a union
// in other words, choose languages whose trigrams contain
// ALL of the unicode blocks found in this sample
// would improve speed but would be completely thrown off by an
// unexpected character, like an umlaut appearing in english text
$possible_langs = array_intersect(
array_keys($this->_lang_db),
array_unique($possible_langs)
);
// needs to intersect it with the keys of _lang_db in case
// languages have been omitted
} else {
// or just try 'em all
$possible_langs = array_keys($this->_lang_db);
}
foreach ($possible_langs as $lang) {
$scores[$lang] = $this->_normalize_score(
$this->_distance($this->_lang_db[$lang], $trigram_freqs),
$trigram_count
);
}
unset($sample_obj);
if ($this->_perl_compatible) {
asort($scores);
} else {
arsort($scores);
}
// todo: drop languages with a score of $this->_max_score?
// limit the number of returned scores
if ($limit && is_numeric($limit)) {
$limited_scores = array();
$i = 0;
foreach ($scores as $key => $value) {
if ($i++ >= $limit) {
break;
}
$limited_scores[$key] = $value;
}
return $this->_convertToNameMode($limited_scores, true);
} else {
return $this->_convertToNameMode($scores, true);
}
}
/**
* Returns only the most similar language to the text sample
*
* Calls $this->detect() and returns only the top result
*
* @param string $sample text to detect the language of
*
* @return string the name of the most likely language
* or null if no language is similar
* @see detect()
* @throws Text_LanguageDetect_Exception
*/
public function detectSimple($sample)
{
$scores = $this->detect($sample, 1);
// if top language has the maximum possible score,
// then the top score will have been picked at random
if (!is_array($scores) || empty($scores)
|| current($scores) == $this->_max_score
) {
return null;
} else {
return key($scores);
}
}
/**
* Returns an array containing the most similar language and a confidence
* rating
*
* Confidence is a simple measure calculated from the similarity score
* minus the similarity score from the next most similar language
* divided by the highest possible score. Languages that have closely
* related cousins (e.g. Norwegian and Danish) should generally have lower
* confidence scores.
*
* The similarity score answers the question "How likely is the text the
* returned language regardless of the other languages considered?" The
* confidence score is one way of answering the question "how likely is the
* text the detected language relative to the rest of the language model
* set?"
*
* To see how similar languages are a priori, see languageSimilarity()
*
* @param string $sample text for which language will be detected
*
* @return array most similar language, score and confidence rating
* or null if no language is similar
* @see detect()
* @throws Text_LanguageDetect_Exception
*/
public function detectConfidence($sample)
{
$scores = $this->detect($sample, 2);
// if most similar language has the max score, it
// will have been picked at random
if (!is_array($scores) || empty($scores)
|| current($scores) == $this->_max_score
) {
return null;
}
$arr['language'] = key($scores);
$arr['similarity'] = current($scores);
if (next($scores) !== false) { // if false then no next element
// the goal is to return a higher value if the distance between
// the similarity of the first score and the second score is high
if ($this->_perl_compatible) {
$arr['confidence'] = (current($scores) - $arr['similarity'])
/ $this->_max_score;
} else {
$arr['confidence'] = $arr['similarity'] - current($scores);
}
} else {
$arr['confidence'] = null;
}
return $arr;
}
/**
* Returns the distribution of unicode blocks in a given utf8 string
*
* For the block name of a single char, use unicodeBlockName()
*
* @param string $str input string. Must be ascii or utf8
* @param bool $skip_symbols if true, skip ascii digits, symbols and
* non-printing characters. Includes spaces,
* newlines and common punctutation characters.
*
* @return array
* @throws Text_LanguageDetect_Exception
*/
public function detectUnicodeBlocks($str, $skip_symbols)
{
$skip_symbols = (bool)$skip_symbols;
$str = (string)$str;
$sample_obj = new Text_LanguageDetect_Parser($str);
$sample_obj->prepareUnicode();
$sample_obj->prepareTrigram(false);
$sample_obj->setUnicodeSkipSymbols($skip_symbols);
$sample_obj->analyze();
$blocks = $sample_obj->getUnicodeBlocks();
unset($sample_obj);
return $blocks;
}
/**
* Returns the block name for a given unicode value
*
* If passed a string, will assume it is being passed a UTF8-formatted
* character and will automatically convert. Otherwise it will assume it
* is being passed a numeric unicode value.
*
* Make sure input is of the correct type!
*
* @param mixed $unicode unicode value or utf8 char
*
* @return mixed the block name string or false if not found
* @throws Text_LanguageDetect_Exception
*/
public function unicodeBlockName($unicode)
{
if (is_string($unicode)) {
// assume it is being passed a utf8 char, so convert it
if (self::utf8strlen($unicode) > 1) {
throw new Text_LanguageDetect_Exception(
'Pass a single char only to this method',
Text_LanguageDetect_Exception::PARAM_TYPE
);
}
$unicode = $this->_utf8char2unicode($unicode);
} elseif (!is_int($unicode)) {
throw new Text_LanguageDetect_Exception(
'Input must be of type string or int.',
Text_LanguageDetect_Exception::PARAM_TYPE
);
}
$blocks = $this->_read_unicode_block_db();
$result = $this->_unicode_block_name($unicode, $blocks);
if ($result == -1) {
return false;
} else {
return $result[2];
}
}
/**
* Searches the unicode block database
*
* Returns the block name for a given unicode value. unicodeBlockName() is
* the public interface for this function, which does input checks which
* this function omits for speed.
*
* @param int $unicode the unicode value
* @param array $blocks the block database
* @param int $block_count the number of defined blocks in the database
*
* @return mixed Block name, -1 if it failed
* @see unicodeBlockName()
*/
protected function _unicode_block_name($unicode, $blocks, $block_count = -1)
{
// for a reference, see
// http://www.unicode.org/Public/UNIDATA/Blocks.txt
// assume that ascii characters are the most common
// so try it first for efficiency
if ($unicode <= $blocks[0][1]) {
return $blocks[0];
}
// the optional $block_count param is for efficiency
// so we this function doesn't have to run count() every time
if ($block_count != -1) {
$high = $block_count - 1;
} else {
$high = count($blocks) - 1;
}
$low = 1; // start with 1 because ascii was 0
// your average binary search algorithm
while ($low <= $high) {
$mid = floor(($low + $high) / 2);
if ($unicode < $blocks[$mid][0]) {
// if it's lower than the lower bound
$high = $mid - 1;
} elseif ($unicode > $blocks[$mid][1]) {
// if it's higher than the upper bound
$low = $mid + 1;
} else {
// found it
return $blocks[$mid];
}
}
// failed to find the block
return -1;
// todo: differentiate when it's out of range or when it falls
// into an unassigned range?
}
/**
* Brings up the unicode block database
*
* @return array the database of unicode block definitions
* @throws Text_LanguageDetect_Exception
*/
protected function _read_unicode_block_db()
{
// since the unicode definitions are always going to be the same,
// might as well share the memory for the db with all other instances
// of this class
static $data;
if (!isset($data)) {
$data = $this->_readdb($this->_unicode_db_filename);
}
return $data;
}
/**
* Calculate the similarities between the language models
*
* Use this function to see how similar languages are to each other.
*
* If passed 2 language names, will return just those languages compared.
* If passed 1 language name, will return that language compared to
* all others.
* If passed none, will return an array of every language model compared
* to every other one.
*
* @param string $lang1 the name of the first language to be compared
* @param string $lang2 the name of the second language to be compared
*
* @return array scores of every language compared
* or the score of just the provided languages
* or null if one of the supplied languages does not exist
* @throws Text_LanguageDetect_Exception
*/
public function languageSimilarity($lang1 = null, $lang2 = null)
{
$lang1 = $this->_convertFromNameMode($lang1);
$lang2 = $this->_convertFromNameMode($lang2);
if ($lang1 != null) {
$lang1 = strtolower($lang1);
// check if language model exists
if (!isset($this->_lang_db[$lang1])) {
return null;
}
if ($lang2 != null) {
if (!isset($this->_lang_db[$lang2])) {
// check if language model exists
return null;
}
$lang2 = strtolower($lang2);
// compare just these two languages
return $this->_normalize_score(
$this->_distance(
$this->_lang_db[$lang1],
$this->_lang_db[$lang2]
)
);
} else {
// compare just $lang1 to all languages
$return_arr = array();
foreach ($this->_lang_db as $key => $value) {
if ($key != $lang1) {
// don't compare a language to itself
$return_arr[$key] = $this->_normalize_score(
$this->_distance($this->_lang_db[$lang1], $value)
);
}
}
asort($return_arr);
return $return_arr;
}
} else {
// compare all languages to each other
$return_arr = array();
foreach (array_keys($this->_lang_db) as $lang1) {
foreach (array_keys($this->_lang_db) as $lang2) {
// skip comparing languages to themselves
if ($lang1 != $lang2) {
if (isset($return_arr[$lang2][$lang1])) {
// don't re-calculate what's already been done
$return_arr[$lang1][$lang2]
= $return_arr[$lang2][$lang1];
} else {
// calculate
$return_arr[$lang1][$lang2]
= $this->_normalize_score(
$this->_distance(
$this->_lang_db[$lang1],
$this->_lang_db[$lang2]
)
);
}
}
}
}
return $return_arr;
}
}
/**
* Cluster known languages according to languageSimilarity()
*
* WARNING: this method is EXPERIMENTAL. It is not recommended for common
* use, and it may disappear or its functionality may change in future
* releases without notice.
*
* Uses a nearest neighbor technique to generate the maximum possible
* number of dendograms from the similarity data.
*
* @return array language cluster data
* @throws Text_LanguageDetect_Exception
* @see languageSimilarity()
* @deprecated this function will eventually be removed and placed into
* the model generation class
*/
public function clusterLanguages()
{
// todo: set the maximum number of clusters
// return cached result, if any
if (isset($this->_clusters)) {
return $this->_clusters;
}
$langs = array_keys($this->_lang_db);
$arr = $this->languageSimilarity();
sort($langs);
foreach ($langs as $lang) {
if (!isset($this->_lang_db[$lang])) {
throw new Text_LanguageDetect_Exception(
"missing $lang!",
Text_LanguageDetect_Exception::UNKNOWN_LANGUAGE
);
}
}
// http://www.psychstat.missouristate.edu/multibook/mlt04m.html
foreach ($langs as $old_key => $lang1) {
$langs[$lang1] = $lang1;
unset($langs[$old_key]);
}
$result_data = $really_map = array();
$i = 0;
while (count($langs) > 2 && $i++ < 200) {
$highest_score = -1;
$highest_key1 = '';
$highest_key2 = '';
foreach ($langs as $lang1) {
foreach ($langs as $lang2) {
if ($lang1 != $lang2
&& $arr[$lang1][$lang2] > $highest_score
) {
$highest_score = $arr[$lang1][$lang2];
$highest_key1 = $lang1;
$highest_key2 = $lang2;
}
}
}
if (!$highest_key1) {
// should not ever happen
throw new Text_LanguageDetect_Exception(
"no highest key? (step: $i)",
Text_LanguageDetect_Exception::NO_HIGHEST_KEY
);
}
if ($highest_score == 0) {
// languages are perfectly dissimilar
break;
}
// $highest_key1 and $highest_key2 are most similar
$sum1 = array_sum($arr[$highest_key1]);
$sum2 = array_sum($arr[$highest_key2]);
// use the score for the one that is most similar to the rest of
// the field as the score for the group
// todo: could try averaging or "centroid" method instead
// seems like that might make more sense
// actually nearest neighbor may be better for binary searching
// for "Complete Linkage"/"furthest neighbor"
// sign should be <
// for "Single Linkage"/"nearest neighbor" method
// should should be >
// results seem to be pretty much the same with either method
// figure out which to delete and which to replace
if ($sum1 > $sum2) {
$replaceme = $highest_key1;
$deleteme = $highest_key2;
} else {
$replaceme = $highest_key2;
$deleteme = $highest_key1;
}
$newkey = $replaceme . ':' . $deleteme;
// $replaceme is most similar to remaining languages
// replace $replaceme with '$newkey', deleting $deleteme
// keep a record of which fork is really which language
$really_lang = $replaceme;
while (isset($really_map[$really_lang])) {
$really_lang = $really_map[$really_lang];
}
$really_map[$newkey] = $really_lang;
// replace the best fitting key, delete the other
foreach ($arr as $key1 => $arr2) {
foreach ($arr2 as $key2 => $value2) {
if ($key2 == $replaceme) {
$arr[$key1][$newkey] = $arr[$key1][$key2];
unset($arr[$key1][$key2]);
// replacing $arr[$key1][$key2] with $arr[$key1][$newkey]
}
if ($key1 == $replaceme) {
$arr[$newkey][$key2] = $arr[$key1][$key2];
unset($arr[$key1][$key2]);
// replacing $arr[$key1][$key2] with $arr[$newkey][$key2]
}
if ($key1 == $deleteme || $key2 == $deleteme) {
// deleting $arr[$key1][$key2]
unset($arr[$key1][$key2]);
}
}
}
unset($langs[$highest_key1]);
unset($langs[$highest_key2]);
$langs[$newkey] = $newkey;
// some of these may be overkill
$result_data[$newkey] = array(
'newkey' => $newkey,
'count' => $i,
'diff' => abs($sum1 - $sum2),
'score' => $highest_score,
'bestfit' => $replaceme,
'otherfit' => $deleteme,
'really' => $really_lang,
);
}
$return_val = array(
'open_forks' => $langs,
// the top level of clusters
// clusters that are mutually exclusive
// or specified by a specific maximum
'fork_data' => $result_data,
// data for each split
'name_map' => $really_map,
// which cluster is really which language
// using the nearest neighbor technique, the cluster
// inherits all of the properties of its most-similar member
// this keeps track
);
// saves the result in the object
$this->_clusters = $return_val;
return $return_val;
}
/**
* Perform an intelligent detection based on clusterLanguages()
*
* WARNING: this method is EXPERIMENTAL. It is not recommended for common
* use, and it may disappear or its functionality may change in future
* releases without notice.
*
* This compares the sample text to top the top level of clusters. If the
* sample is similar to the cluster it will drop down and compare it to the
* languages in the cluster, and so on until it hits a leaf node.
*
* this should find the language in considerably fewer compares
* (the equivalent of a binary search), however clusterLanguages() is costly
* and the loss of accuracy from this technique is significant.
*
* This method may need to be 'fuzzier' in order to become more accurate.
*
* This function could be more useful if the universe of possible languages
* was very large, however in such cases some method of Bayesian inference
* might be more helpful.
*
* @param string $str input string
*
* @return array language scores (only those compared)
* @throws Text_LanguageDetect_Exception
* @see clusterLanguages()
*/
public function clusteredSearch($str)
{
// input check
if (!Text_LanguageDetect_Parser::validateString($str)) {
return array();
}
// clusterLanguages() will return a cached result if possible
// so it's safe to call it every time
$result = $this->clusterLanguages();
$dendogram_start = $result['open_forks'];
$dendogram_data = $result['fork_data'];
$dendogram_alias = $result['name_map'];
$sample_obj = new Text_LanguageDetect_Parser($str);
$sample_obj->prepareTrigram();
$sample_obj->setPadStart(!$this->_perl_compatible);
$sample_obj->analyze();
$sample_result = $sample_obj->getTrigramRanks();
$sample_count = count($sample_result);
// input check
if ($sample_count == 0) {
return array();
}
$i = 0; // counts the number of steps
foreach ($dendogram_start as $lang) {
if (isset($dendogram_alias[$lang])) {
$lang_key = $dendogram_alias[$lang];
} else {
$lang_key = $lang;
}
$scores[$lang] = $this->_normalize_score(
$this->_distance($this->_lang_db[$lang_key], $sample_result),
$sample_count
);
$i++;
}
if ($this->_perl_compatible) {
asort($scores);
} else {
arsort($scores);
}
$top_score = current($scores);
$top_key = key($scores);
// of starting forks, $top_key is the most similar to the sample
$cur_key = $top_key;
while (isset($dendogram_data[$cur_key])) {
$lang1 = $dendogram_data[$cur_key]['bestfit'];
$lang2 = $dendogram_data[$cur_key]['otherfit'];
foreach (array($lang1, $lang2) as $lang) {
if (isset($dendogram_alias[$lang])) {
$lang_key = $dendogram_alias[$lang];
} else {
$lang_key = $lang;
}
$scores[$lang] = $this->_normalize_score(
$this->_distance($this->_lang_db[$lang_key], $sample_result),
$sample_count
);
//todo: does not need to do same comparison again
}
$i++;
if ($scores[$lang1] > $scores[$lang2]) {
$cur_key = $lang1;
$loser_key = $lang2;
} else {
$cur_key = $lang2;
$loser_key = $lang1;
}
$diff = $scores[$cur_key] - $scores[$loser_key];
// $cur_key ({$dendogram_alias[$cur_key]}) wins
// over $loser_key ({$dendogram_alias[$loser_key]})
// with a difference of $diff
}
// found result in $i compares
// rather than sorting the result, preserve it so that you can see
// which paths the algorithm decided to take along the tree
// but sometimes the last item is only the second highest
if (($this->_perl_compatible && (end($scores) > prev($scores)))
|| (!$this->_perl_compatible && (end($scores) < prev($scores)))
) {
$real_last_score = current($scores);
$real_last_key = key($scores);
// swaps the 2nd-to-last item for the last item
unset($scores[$real_last_key]);
$scores[$real_last_key] = $real_last_score;
}
if (!$this->_perl_compatible) {
$scores = array_reverse($scores, true);
// second param requires php > 4.0.3
}
return $scores;
}
/**
* UTF8-safe strlen()
*
* Returns the numbers of characters (not bytes) in a utf8 string
*
* @param string $str string to get the length of
*
* @return int number of chars
*/
public static function utf8strlen($str)
{
// utf8_decode() will convert unknown chars to '?', which is actually
// ideal for counting.
return strlen(utf8_decode($str));
// idea stolen from dokuwiki
}
/**
* Returns the unicode value of a utf8 char
*
* @param string $char a utf8 (possibly multi-byte) char
*
* @return int unicode value
* @link http://en.wikipedia.org/wiki/UTF-8
*/
protected function _utf8char2unicode($char)
{
// strlen() here will actually get the binary length of a single char
switch (strlen($char)) {
case 1:
// normal ASCII-7 byte
// 0xxxxxxx --> 0xxxxxxx
return ord($char{0});
case 2:
// 2 byte unicode
// 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx
$z = (ord($char{0}) & 0x000001F) << 6;
$x = (ord($char{1}) & 0x0000003F);
return ($z | $x);
case 3:
// 3 byte unicode
// 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx
$z = (ord($char{0}) & 0x0000000F) << 12;
$x1 = (ord($char{1}) & 0x0000003F) << 6;
$x2 = (ord($char{2}) & 0x0000003F);
return ($z | $x1 | $x2);
case 4:
// 4 byte unicode
// 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx -->
// 000zzzzz xxxxxxxx xxxxxxxx
$z1 = (ord($char{0}) & 0x00000007) << 18;
$z2 = (ord($char{1}) & 0x0000003F) << 12;
$x1 = (ord($char{2}) & 0x0000003F) << 6;
$x2 = (ord($char{3}) & 0x0000003F);
return ($z1 | $z2 | $x1 | $x2);
}
}
/**
* UTF8-safe fast character iterator
*
* Will get the next character starting from $counter, which will then be
* incremented. If a multi-byte char the bytes will be concatenated and
* $counter will be incremeted by the number of bytes in the char.
*
* @param string $str the string being iterated over
* @param int $counter the iterator, will increment by reference
* @param bool $special_convert whether to do special conversions
*
* @return char the next (possibly multi-byte) char from $counter
*/
protected static function _next_char($str, &$counter, $special_convert = false)
{
$char = $str{$counter++};
$ord = ord($char);
// for a description of the utf8 system see
// http://www.phpclasses.org/browse/file/5131.html
// normal ascii one byte char
if ($ord <= 127) {
// special conversions needed for this package
// (that only apply to regular ascii characters)
// lower case, and convert all non-alphanumeric characters
// other than "'" to space
if ($special_convert && $char != ' ' && $char != "'") {
if ($ord >= 65 && $ord <= 90) { // A-Z
$char = chr($ord + 32); // lower case
} elseif ($ord < 97 || $ord > 122) { // NOT a-z
$char = ' '; // convert to space
}
}
return $char;
} elseif ($ord >> 5 == 6) { // two-byte char
// multi-byte chars
$nextchar = $str{$counter++}; // get next byte
// lower-casing of non-ascii characters is still incomplete
if ($special_convert) {
// lower case latin accented characters
if ($ord == 195) {
$nextord = ord($nextchar);
$nextord_adj = $nextord + 64;
// for a reference, see
// http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html
// &Agrave; - &THORN; but not &times;
if ($nextord_adj >= 192
&& $nextord_adj <= 222
&& $nextord_adj != 215
) {
$nextchar = chr($nextord + 32);
}
} elseif ($ord == 208) {
// lower case cyrillic alphabet
$nextord = ord($nextchar);
// if A - Pe
if ($nextord >= 144 && $nextord <= 159) {
// lower case
$nextchar = chr($nextord + 32);
} elseif ($nextord >= 160 && $nextord <= 175) {
// if Er - Ya
// lower case
$char = chr(209); // == $ord++
$nextchar = chr($nextord - 32);
}
}
}
// tag on next byte
return $char . $nextchar;
} elseif ($ord >> 4 == 14) { // three-byte char
// tag on next 2 bytes
return $char . $str{$counter++} . $str{$counter++};
} elseif ($ord >> 3 == 30) { // four-byte char
// tag on next 3 bytes
return $char . $str{$counter++} . $str{$counter++} . $str{$counter++};
} else {
// error?
}
}
/**
* Converts an $language input parameter from the configured mode
* to the language name that is used internally.
*
* Works for strings and arrays.
*
* @param string|array $lang A language description ("english"/"en"/"eng")
* @param boolean $convertKey If $lang is an array, setting $key
* converts the keys to the language name.
*
* @return string|array Language name
*/
protected function _convertFromNameMode($lang, $convertKey = false)
{
if ($this->_name_mode == 0) {
return $lang;
}
if ($this->_name_mode == 2) {
$method = 'code2ToName';
} else {
$method = 'code3ToName';
}
if (is_string($lang)) {
return (string)Text_LanguageDetect_ISO639::$method($lang);
}
$newlang = array();
foreach ($lang as $key => $val) {
if ($convertKey) {
$newkey = (string)Text_LanguageDetect_ISO639::$method($key);
$newlang[$newkey] = $val;
} else {
$newlang[$key] = (string)Text_LanguageDetect_ISO639::$method($val);
}
}
return $newlang;
}
/**
* Converts an $language output parameter from the language name that is
* used internally to the configured mode.
*
* Works for strings and arrays.
*
* @param string|array $lang A language description ("english"/"en"/"eng")
* @param boolean $convertKey If $lang is an array, setting $key
* converts the keys to the language name.
*
* @return string|array Language name
*/
protected function _convertToNameMode($lang, $convertKey = false)
{
if ($this->_name_mode == 0) {
return $lang;
}
if ($this->_name_mode == 2) {
$method = 'nameToCode2';
} else {
$method = 'nameToCode3';
}
if (is_string($lang)) {
return Text_LanguageDetect_ISO639::$method($lang);
}
$newlang = array();
foreach ($lang as $key => $val) {
if ($convertKey) {
$newkey = Text_LanguageDetect_ISO639::$method($key);
$newlang[$newkey] = $val;
} else {
$newlang[$key] = Text_LanguageDetect_ISO639::$method($val);
}
}
return $newlang;
}
}
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
?>

81
vendor/pear/text_languagedetect/Text/LanguageDetect/Exception.php vendored Normal file
View file

@ -0,0 +1,81 @@
<?php
/**
* Part of Text_LanguageDetect
*
* PHP version 5
*
* @category Text
* @package Text_LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @link http://pear.php.net/package/Text_LanguageDetect/
*/
/**
* Part of the PEAR language detection package
*
* PHP version 5
*
* @category Text
* @package Text_LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @link http://pear.php.net/package/Text_LanguageDetect/
* @link http://langdetect.blogspot.com/
*/
class Text_LanguageDetect_Exception extends Exception
{
/**
* Database file could not be found
*/
const DB_NOT_FOUND = 10;
/**
* Database file found, but not readable
*/
const DB_NOT_READABLE = 11;
/**
* Database file is empty
*/
const DB_EMPTY = 12;
/**
* Database contents is not a PHP array
*/
const DB_NOT_ARRAY = 13;
/**
* Magic quotes are activated
*/
const MAGIC_QUOTES = 14;
/**
* Parameter of invalid type passed to method
*/
const PARAM_TYPE = 20;
/**
* Character in parameter is invalid
*/
const INVALID_CHAR = 21;
/**
* Language is not in the database
*/
const UNKNOWN_LANGUAGE = 30;
/**
* Error during block detection
*/
const BLOCK_DETECTION = 40;
/**
* Error while clustering languages
*/
const NO_HIGHEST_KEY = 50;
}

340
vendor/pear/text_languagedetect/Text/LanguageDetect/ISO639.php vendored Normal file
View file

@ -0,0 +1,340 @@
<?php
/**
* Part of Text_LanguageDetect
*
* PHP version 5
*
* @category Text
* @package Text_LanguageDetect
* @author Christian Weiske <cweiske@php.net>
* @copyright 2011 Christian Weiske <cweiske@php.net>
* @license http://www.debian.org/misc/bsd.license BSD
* @link http://pear.php.net/package/Text_LanguageDetect/
*/
/**
* Provides a mapping between the languages from lang.dat and the
* ISO 639-1 and ISO-639-2 codes.
*
* Note that this class contains only languages that exist in lang.dat.
*
* @category Text
* @package Text_LanguageDetect
* @author Christian Weiske <cweiske@php.net>
* @copyright 2011 Christian Weiske <cweiske@php.net>
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @link http://www.loc.gov/standards/iso639-2/php/code_list.php
*/
class Text_LanguageDetect_ISO639
{
/**
* Maps all language names from the language database to the
* ISO 639-1 2-letter language code.
*
* NULL indicates that there is no 2-letter code.
*
* @var array
*/
public static $nameToCode2 = array(
'albanian' => 'sq',
'arabic' => 'ar',
'azeri' => 'az',
'bengali' => 'bn',
'bulgarian' => 'bg',
'cebuano' => null,
'croatian' => 'hr',
'czech' => 'cs',
'danish' => 'da',
'dutch' => 'nl',
'english' => 'en',
'estonian' => 'et',
'farsi' => 'fa',
'finnish' => 'fi',
'french' => 'fr',
'german' => 'de',
'hausa' => 'ha',
'hawaiian' => null,
'hindi' => 'hi',
'hungarian' => 'hu',
'icelandic' => 'is',
'indonesian' => 'id',
'italian' => 'it',
'kazakh' => 'kk',
'kyrgyz' => 'ky',
'latin' => 'la',
'latvian' => 'lv',
'lithuanian' => 'lt',
'macedonian' => 'mk',
'mongolian' => 'mn',
'nepali' => 'ne',
'norwegian' => 'no',
'pashto' => 'ps',
'pidgin' => null,
'polish' => 'pl',
'portuguese' => 'pt',
'romanian' => 'ro',
'russian' => 'ru',
'serbian' => 'sr',
'slovak' => 'sk',
'slovene' => 'sl',
'somali' => 'so',
'spanish' => 'es',
'swahili' => 'sw',
'swedish' => 'sv',
'tagalog' => 'tl',
'turkish' => 'tr',
'ukrainian' => 'uk',
'urdu' => 'ur',
'uzbek' => 'uz',
'vietnamese' => 'vi',
'welsh' => 'cy',
);
/**
* Maps all language names from the language database to the
* ISO 639-2 3-letter language code.
*
* @var array
*/
public static $nameToCode3 = array(
'albanian' => 'sqi',
'arabic' => 'ara',
'azeri' => 'aze',
'bengali' => 'ben',
'bulgarian' => 'bul',
'cebuano' => 'ceb',
'croatian' => 'hrv',
'czech' => 'ces',
'danish' => 'dan',
'dutch' => 'nld',
'english' => 'eng',
'estonian' => 'est',
'farsi' => 'fas',
'finnish' => 'fin',
'french' => 'fra',
'german' => 'deu',
'hausa' => 'hau',
'hawaiian' => 'haw',
'hindi' => 'hin',
'hungarian' => 'hun',
'icelandic' => 'isl',
'indonesian' => 'ind',
'italian' => 'ita',
'kazakh' => 'kaz',
'kyrgyz' => 'kir',
'latin' => 'lat',
'latvian' => 'lav',
'lithuanian' => 'lit',
'macedonian' => 'mkd',
'mongolian' => 'mon',
'nepali' => 'nep',
'norwegian' => 'nor',
'pashto' => 'pus',
'pidgin' => 'crp',
'polish' => 'pol',
'portuguese' => 'por',
'romanian' => 'ron',
'russian' => 'rus',
'serbian' => 'srp',
'slovak' => 'slk',
'slovene' => 'slv',
'somali' => 'som',
'spanish' => 'spa',
'swahili' => 'swa',
'swedish' => 'swe',
'tagalog' => 'tgl',
'turkish' => 'tur',
'ukrainian' => 'ukr',
'urdu' => 'urd',
'uzbek' => 'uzb',
'vietnamese' => 'vie',
'welsh' => 'cym',
);
/**
* Maps ISO 639-1 2-letter language codes to the language names
* in the language database
*
* Not all languages have a 2 letter code, so some are missing
*
* @var array
*/
public static $code2ToName = array(
'ar' => 'arabic',
'az' => 'azeri',
'bg' => 'bulgarian',
'bn' => 'bengali',
'cs' => 'czech',
'cy' => 'welsh',
'da' => 'danish',
'de' => 'german',
'en' => 'english',
'es' => 'spanish',
'et' => 'estonian',
'fa' => 'farsi',
'fi' => 'finnish',
'fr' => 'french',
'ha' => 'hausa',
'hi' => 'hindi',
'hr' => 'croatian',
'hu' => 'hungarian',
'id' => 'indonesian',
'is' => 'icelandic',
'it' => 'italian',
'kk' => 'kazakh',
'ky' => 'kyrgyz',
'la' => 'latin',
'lt' => 'lithuanian',
'lv' => 'latvian',
'mk' => 'macedonian',
'mn' => 'mongolian',
'ne' => 'nepali',
'nl' => 'dutch',
'no' => 'norwegian',
'pl' => 'polish',
'ps' => 'pashto',
'pt' => 'portuguese',
'ro' => 'romanian',
'ru' => 'russian',
'sk' => 'slovak',
'sl' => 'slovene',
'so' => 'somali',
'sq' => 'albanian',
'sr' => 'serbian',
'sv' => 'swedish',
'sw' => 'swahili',
'tl' => 'tagalog',
'tr' => 'turkish',
'uk' => 'ukrainian',
'ur' => 'urdu',
'uz' => 'uzbek',
'vi' => 'vietnamese',
);
/**
* Maps ISO 639-2 3-letter language codes to the language names
* in the language database.
*
* @var array
*/
public static $code3ToName = array(
'ara' => 'arabic',
'aze' => 'azeri',
'ben' => 'bengali',
'bul' => 'bulgarian',
'ceb' => 'cebuano',
'ces' => 'czech',
'crp' => 'pidgin',
'cym' => 'welsh',
'dan' => 'danish',
'deu' => 'german',
'eng' => 'english',
'est' => 'estonian',
'fas' => 'farsi',
'fin' => 'finnish',
'fra' => 'french',
'hau' => 'hausa',
'haw' => 'hawaiian',
'hin' => 'hindi',
'hrv' => 'croatian',
'hun' => 'hungarian',
'ind' => 'indonesian',
'isl' => 'icelandic',
'ita' => 'italian',
'kaz' => 'kazakh',
'kir' => 'kyrgyz',
'lat' => 'latin',
'lav' => 'latvian',
'lit' => 'lithuanian',
'mkd' => 'macedonian',
'mon' => 'mongolian',
'nep' => 'nepali',
'nld' => 'dutch',
'nor' => 'norwegian',
'pol' => 'polish',
'por' => 'portuguese',
'pus' => 'pashto',
'rom' => 'romanian',
'rus' => 'russian',
'slk' => 'slovak',
'slv' => 'slovene',
'som' => 'somali',
'spa' => 'spanish',
'sqi' => 'albanian',
'srp' => 'serbian',
'swa' => 'swahili',
'swe' => 'swedish',
'tgl' => 'tagalog',
'tur' => 'turkish',
'ukr' => 'ukrainian',
'urd' => 'urdu',
'uzb' => 'uzbek',
'vie' => 'vietnamese',
);
/**
* Returns the 2-letter ISO 639-1 code for the given language name.
*
* @param string $lang English language name like "swedish"
*
* @return string Two-letter language code (e.g. "sv") or NULL if not found
*/
public static function nameToCode2($lang)
{
$lang = strtolower($lang);
if (!isset(self::$nameToCode2[$lang])) {
return null;
}
return self::$nameToCode2[$lang];
}
/**
* Returns the 3-letter ISO 639-2 code for the given language name.
*
* @param string $lang English language name like "swedish"
*
* @return string Three-letter language code (e.g. "swe") or NULL if not found
*/
public static function nameToCode3($lang)
{
$lang = strtolower($lang);
if (!isset(self::$nameToCode3[$lang])) {
return null;
}
return self::$nameToCode3[$lang];
}
/**
* Returns the language name for the given 2-letter ISO 639-1 code.
*
* @param string $code Two-letter language code (e.g. "sv")
*
* @return string English language name like "swedish"
*/
public static function code2ToName($code)
{
$lang = strtolower($code);
if (!isset(self::$code2ToName[$code])) {
return null;
}
return self::$code2ToName[$code];
}
/**
* Returns the language name for the given 3-letter ISO 639-2 code.
*
* @param string $code Three-letter language code (e.g. "swe")
*
* @return string English language name like "swedish"
*/
public static function code3ToName($code)
{
$lang = strtolower($code);
if (!isset(self::$code3ToName[$code])) {
return null;
}
return self::$code3ToName[$code];
}
}
?>

358
vendor/pear/text_languagedetect/Text/LanguageDetect/Parser.php vendored Normal file
View file

@ -0,0 +1,358 @@
<?php
/**
* Part of Text_LanguageDetect
*
* PHP version 5
*
* @category Text
* @package Text_LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @copyright 2006 Nicholas Pisarro
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @link http://pear.php.net/package/Text_LanguageDetect/
*/
/**
* This class represents a text sample to be parsed.
*
* This separates the analysis of a text sample from the primary LanguageDetect
* class. After a new profile has been built, the data can be retrieved using
* the accessor functions.
*
* This class is intended to be used by the Text_LanguageDetect class, not
* end-users.
*
* @category Text
* @package Text_LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @copyright 2006 Nicholas Pisarro
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @version Release: @package_version@
* @link http://pear.php.net/package/Text_LanguageDetect/
*/
class Text_LanguageDetect_Parser extends Text_LanguageDetect
{
/**
* The piece of text being parsed
*
* @var string
*/
protected $_string;
/**
* Stores the trigram frequencies of the sample
*
* @var string
*/
protected $_trigrams = array();
/**
* Stores the trigram ranks of the sample
*
* @var array
*/
protected $_trigram_ranks = array();
/**
* Stores the unicode blocks of the sample
*
* @var array
*/
protected $_unicode_blocks = array();
/**
* Whether the parser should compile the unicode ranges
*
* @var bool
*/
protected $_compile_unicode = false;
/**
* Whether the parser should compile trigrams
*
* @var bool
*/
protected $_compile_trigram = false;
/**
* Whether the trigram parser should pad the beginning of the string
*
* @var bool
*/
protected $_trigram_pad_start = false;
/**
* Whether the unicode parser should skip non-alphabetical ascii chars
*
* @var bool
*/
protected $_unicode_skip_symbols = true;
/**
* Constructor
*
* @param string $string string to be parsed
*/
public function __construct($string)
{
$this->_string = $string;
}
/**
* PHP 4 constructor for backwards compatibility.
*
* @param string $string string to be parsed
*
* @return void
*/
public function Text_LanguageDetect_Parser($string)
{
self::__construct($string);
}
/**
* Returns true if a string is suitable for parsing
*
* @param string $str input string to test
*
* @return bool true if acceptable, false if not
*/
public static function validateString($str)
{
if (!empty($str) && strlen($str) > 3 && preg_match('/\S/', $str)) {
return true;
} else {
return false;
}
}
/**
* Turn on/off trigram counting
*
* @param bool $bool true for on, false for off
*
* @return void
*/
public function prepareTrigram($bool = true)
{
$this->_compile_trigram = $bool;
}
/**
* Turn on/off unicode block counting
*
* @param bool $bool true for on, false for off
*
* @return void
*/
public function prepareUnicode($bool = true)
{
$this->_compile_unicode = $bool;
}
/**
* Turn on/off padding the beginning of the sample string
*
* @param bool $bool true for on, false for off
*
* @return void
*/
public function setPadStart($bool = true)
{
$this->_trigram_pad_start = $bool;
}
/**
* Should the unicode block counter skip non-alphabetical ascii chars?
*
* @param bool $bool true for on, false for off
*
* @return void
*/
public function setUnicodeSkipSymbols($bool = true)
{
$this->_unicode_skip_symbols = $bool;
}
/**
* Returns the trigram ranks for the text sample
*
* @return array Trigram ranks in the text sample
*/
public function getTrigramRanks()
{
return $this->_trigram_ranks;
}
/**
* Return the trigram freqency table
*
* Only used in testing to make sure the parser is working
*
* @return array Trigram freqencies in the text sample
*/
public function getTrigramFreqs()
{
return $this->_trigram;
}
/**
* Returns the array of unicode blocks
*
* @return array Unicode blocks in the text sample
*/
public function getUnicodeBlocks()
{
return $this->_unicode_blocks;
}
/**
* Executes the parsing operation
*
* Be sure to call the set*() functions to set options and the
* prepare*() functions first to tell it what kind of data to compute
*
* Afterwards the get*() functions can be used to access the compiled
* information.
*
* @return void
*/
public function analyze()
{
$len = strlen($this->_string);
$byte_counter = 0;
// unicode startup
if ($this->_compile_unicode) {
$blocks = $this->_read_unicode_block_db();
$block_count = count($blocks);
$skipped_count = 0;
$unicode_chars = array();
}
// trigram startup
if ($this->_compile_trigram) {
// initialize them as blank so the parser will skip the first two
// (since it skips trigrams with more than 2 contiguous spaces)
$a = ' ';
$b = ' ';
// kludge
// if it finds a valid trigram to start and the start pad option is
// off, then set a variable that will be used to reduce this
// trigram after parsing has finished
if (!$this->_trigram_pad_start) {
$a = $this->_next_char($this->_string, $byte_counter, true);
if ($a != ' ') {
$b = $this->_next_char($this->_string, $byte_counter, true);
$dropone = " $a$b";
}
$byte_counter = 0;
$a = ' ';
$b = ' ';
}
}
while ($byte_counter < $len) {
$char = $this->_next_char($this->_string, $byte_counter, true);
// language trigram detection
if ($this->_compile_trigram) {
if (!($b == ' ' && ($a == ' ' || $char == ' '))) {
if (!isset($this->_trigram[$a . $b . $char])) {
$this->_trigram[$a . $b . $char] = 1;
} else {
$this->_trigram[$a . $b . $char]++;
}
}
$a = $b;
$b = $char;
}
// unicode block detection
if ($this->_compile_unicode) {
if ($this->_unicode_skip_symbols
&& strlen($char) == 1
&& ($char < 'A' || $char > 'z'
|| ($char > 'Z' && $char < 'a'))
&& $char != "'"
) { // does not skip the apostrophe
// since it's included in the language
// models
$skipped_count++;
continue;
}
// build an array of all the characters
if (isset($unicode_chars[$char])) {
$unicode_chars[$char]++;
} else {
$unicode_chars[$char] = 1;
}
}
// todo: add byte detection here
}
// unicode cleanup
if ($this->_compile_unicode) {
foreach ($unicode_chars as $utf8_char => $count) {
$search_result = $this->_unicode_block_name(
$this->_utf8char2unicode($utf8_char), $blocks, $block_count
);
if ($search_result != -1) {
$block_name = $search_result[2];
} else {
$block_name = '[Malformatted]';
}
if (isset($this->_unicode_blocks[$block_name])) {
$this->_unicode_blocks[$block_name] += $count;
} else {
$this->_unicode_blocks[$block_name] = $count;
}
}
}
// trigram cleanup
if ($this->_compile_trigram) {
// pad the end
if ($b != ' ') {
if (!isset($this->_trigram["$a$b "])) {
$this->_trigram["$a$b "] = 1;
} else {
$this->_trigram["$a$b "]++;
}
}
// perl compatibility; Language::Guess does not pad the beginning
// kludge
if (isset($dropone)) {
if ($this->_trigram[$dropone] == 1) {
unset($this->_trigram[$dropone]);
} else {
$this->_trigram[$dropone]--;
}
}
if (!empty($this->_trigram)) {
$this->_trigram_ranks = $this->_arr_rank($this->_trigram);
} else {
$this->_trigram_ranks = array();
}
}
}
}
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
?>