Commit 8e4c5db7 authored by Klaus Weidenbach's avatar Klaus Weidenbach

Update Text_LanguageDetect.

Update from v0.3.0 (2012) to v1.0.0 (2017) which should remove some
warnings and improve PHP7 support.
Using composer to handle this PEAR library now.

Fix a problem in FeedutilsTest.
parent fe5f1e4d
......@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"content-hash": "4a78983d966b7641fd532909d625c21a",
"content-hash": "50f781f18deef8573188a713376a711d",
"packages": [
{
"name": "bshaffer/oauth2-server-php",
......@@ -224,6 +224,50 @@
],
"time": "2016-10-29T18:58:20+00:00"
},
{
"name": "pear/text_languagedetect",
"version": "v1.0.0",
"source": {
"type": "git",
"url": "https://github.com/pear/Text_LanguageDetect.git",
"reference": "bb9ff6f4970f686fac59081e916b456021fe7ba6"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pear/Text_LanguageDetect/zipball/bb9ff6f4970f686fac59081e916b456021fe7ba6",
"reference": "bb9ff6f4970f686fac59081e916b456021fe7ba6",
"shasum": ""
},
"require-dev": {
"phpunit/phpunit": "*"
},
"suggest": {
"ext-mbstring": "May require the mbstring PHP extension"
},
"type": "library",
"autoload": {
"psr-0": {
"Text": "./"
}
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
"./"
],
"license": [
"BSD-2-Clause"
],
"authors": [
{
"name": "Nicholas Pisarro",
"email": "taak@php.net",
"role": "Lead"
}
],
"description": "Identify human languages from text samples",
"homepage": "http://pear.php.net/package/Text_LanguageDetect",
"time": "2017-03-02T16:14:08+00:00"
},
{
"name": "psr/log",
"version": "1.0.2",
......@@ -276,12 +320,12 @@
"version": "3.2.2",
"source": {
"type": "git",
"url": "https://github.com/fruux/sabre-dav.git",
"url": "https://github.com/sabre-io/dav.git",
"reference": "e987775e619728f12205606c9cc3ee565ffb1516"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fruux/sabre-dav/zipball/e987775e619728f12205606c9cc3ee565ffb1516",
"url": "https://api.github.com/repos/sabre-io/dav/zipball/e987775e619728f12205606c9cc3ee565ffb1516",
"reference": "e987775e619728f12205606c9cc3ee565ffb1516",
"shasum": ""
},
......@@ -359,12 +403,12 @@
"version": "3.0.0",
"source": {
"type": "git",
"url": "https://github.com/fruux/sabre-event.git",
"url": "https://github.com/sabre-io/event.git",
"reference": "831d586f5a442dceacdcf5e9c4c36a4db99a3534"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fruux/sabre-event/zipball/831d586f5a442dceacdcf5e9c4c36a4db99a3534",
"url": "https://api.github.com/repos/sabre-io/event/zipball/831d586f5a442dceacdcf5e9c4c36a4db99a3534",
"reference": "831d586f5a442dceacdcf5e9c4c36a4db99a3534",
"shasum": ""
},
......@@ -416,12 +460,12 @@
"version": "4.2.3",
"source": {
"type": "git",
"url": "https://github.com/fruux/sabre-http.git",
"url": "https://github.com/sabre-io/http.git",
"reference": "0295f9a3ee39be97e0898592fc19e42421e0cd93"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fruux/sabre-http/zipball/0295f9a3ee39be97e0898592fc19e42421e0cd93",
"url": "https://api.github.com/repos/sabre-io/http/zipball/0295f9a3ee39be97e0898592fc19e42421e0cd93",
"reference": "0295f9a3ee39be97e0898592fc19e42421e0cd93",
"shasum": ""
},
......@@ -472,12 +516,12 @@
"version": "1.2.1",
"source": {
"type": "git",
"url": "https://github.com/fruux/sabre-uri.git",
"url": "https://github.com/sabre-io/uri.git",
"reference": "ada354d83579565949d80b2e15593c2371225e61"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fruux/sabre-uri/zipball/ada354d83579565949d80b2e15593c2371225e61",
"url": "https://api.github.com/repos/sabre-io/uri/zipball/ada354d83579565949d80b2e15593c2371225e61",
"reference": "ada354d83579565949d80b2e15593c2371225e61",
"shasum": ""
},
......@@ -523,12 +567,12 @@
"version": "4.1.2",
"source": {
"type": "git",
"url": "https://github.com/fruux/sabre-vobject.git",
"url": "https://github.com/sabre-io/vobject.git",
"reference": "d0fde2fafa2a3dad1f559c2d1c2591d4fd75ae3c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fruux/sabre-vobject/zipball/d0fde2fafa2a3dad1f559c2d1c2591d4fd75ae3c",
"url": "https://api.github.com/repos/sabre-io/vobject/zipball/d0fde2fafa2a3dad1f559c2d1c2591d4fd75ae3c",
"reference": "d0fde2fafa2a3dad1f559c2d1c2591d4fd75ae3c",
"shasum": ""
},
......@@ -620,12 +664,12 @@
"version": "1.5.0",
"source": {
"type": "git",
"url": "https://github.com/fruux/sabre-xml.git",
"url": "https://github.com/sabre-io/xml.git",
"reference": "59b20e5bbace9912607481634f97d05a776ffca7"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/fruux/sabre-xml/zipball/59b20e5bbace9912607481634f97d05a776ffca7",
"url": "https://api.github.com/repos/sabre-io/xml/zipball/59b20e5bbace9912607481634f97d05a776ffca7",
"reference": "59b20e5bbace9912607481634f97d05a776ffca7",
"shasum": ""
},
......
......@@ -28,7 +28,7 @@ function get_help_content($tocpath = false) {
}
if($path) {
$title = basename($path);
if(! $tocpath)
\App::$page['title'] = t('Help:') . ' ' . ucwords(str_replace('-',' ',notags($title)));
......@@ -38,10 +38,10 @@ function get_help_content($tocpath = false) {
// available and so default back to the English TOC at /doc/toc.{html,bb,md}
// TODO: This is incompatible with the hierarchical TOC construction
// defined in /Zotlabs/Widget/Helpindex.php.
if($tocpath !== false &&
load_doc_file('doc/' . $path . '.md') === '' &&
load_doc_file('doc/' . $path . '.bb') === '' &&
load_doc_file('doc/' . $path . '.html') === ''
if($tocpath !== false &&
load_doc_file('doc/' . $path . '.md') === '' &&
load_doc_file('doc/' . $path . '.bb') === '' &&
load_doc_file('doc/' . $path . '.html') === ''
) {
$path = $title;
}
......@@ -120,22 +120,28 @@ function preg_callback_help_include($matches) {
}
/**
* @brief
*
* @return boolean|array
*/
function determine_help_language() {
require_once('Text/LanguageDetect.php');
$lang_detect = new Text_LanguageDetect();
// Set this mode to recognize language by the short code like "en", "ru", etc.
$lang_detect->setNameMode(2);
// If the language was specified in the URL, override the language preference
// If the language was specified in the URL, override the language preference
// of the browser. Default to English if both of these are absent.
if($lang_detect->languageExists(argv(1))) {
$lang = argv(1);
$from_url = true;
} else {
$lang = \App::$language;
if(! isset($lang))
if(! isset($lang))
$lang = 'en';
$from_url = false;
}
return array('language' => $lang, 'from_url' => $from_url);
}
......@@ -145,14 +151,14 @@ function load_doc_file($s) {
$x = determine_help_language();
$lang = $x['language'];
$url_idx = ($x['from_url'] ? 1 : 0);
// The English translation is at the root of /doc/. Other languages are in
// The English translation is at the root of /doc/. Other languages are in
// subfolders named by the language code such as "de", "es", etc.
if($lang !== 'en') {
$path .= '/' . $lang;
$path .= '/' . $lang;
}
$b = basename($s);
for($i=1+$url_idx; $i<argc()-1; $i++) {
$path .= '/' . argv($i);
}
......
......@@ -17,7 +17,7 @@
*
* Get the language setting directly from system variables, bypassing get_config()
* as database may not yet be configured.
*
*
* If possible, we use the value from the browser.
*
* @return array with ordered list of preferred languages from browser
......@@ -28,7 +28,7 @@ function get_browser_language() {
if (x($_SERVER, 'HTTP_ACCEPT_LANGUAGE')) {
// break up string into pieces (languages and q factors)
preg_match_all('/([a-z]{1,8}(-[a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i',
preg_match_all('/([a-z]{1,8}(-[a-z]{1,8})?)\s*(;\s*q\s*=\s*(1|0\.[0-9]+))?/i',
$_SERVER['HTTP_ACCEPT_LANGUAGE'], $lang_parse);
if (count($lang_parse[1])) {
......@@ -40,7 +40,7 @@ function get_browser_language() {
if ($val === '') $langs[$lang] = 1;
}
// sort list based on value
// sort list based on value
arsort($langs, SORT_NUMERIC);
}
}
......@@ -52,7 +52,7 @@ function get_browser_language() {
* @brief Returns the best language for which also a translation exists.
*
* This function takes the results from get_browser_language() and compares it
* with the available translations and returns the best fitting language for
* with the available translations and returns the best fitting language for
* which there exists a translation.
*
* If there is no match fall back to config['system']['language']
......@@ -243,11 +243,9 @@ function string_plural_select_default($n) {
*
* @see http://pear.php.net/package/Text_LanguageDetect
* @param string $s A string to examine
* @return Language code in 2-letter ISO 639-1 (en, de, fr) format
* @return string Language code in 2-letter ISO 639-1 (en, de, fr) format
*/
function detect_language($s) {
require_once('Text/LanguageDetect.php');
$min_length = get_config('system', 'language_detect_min_length');
if ($min_length === false)
$min_length = LANGUAGE_DETECT_MIN_LENGTH;
......@@ -257,7 +255,7 @@ function detect_language($s) {
$min_confidence = LANGUAGE_DETECT_MIN_CONFIDENCE;
// embedded apps have long base64 strings which will trip up the detector.
$naked_body = preg_replace('/\[app\](.*?)\[\/app\]/','',$s);
$naked_body = preg_replace('/\[app\](.*?)\[\/app\]/', '', $s);
// strip off bbcode
$naked_body = preg_replace('/\[(.+?)\]/', '', $naked_body);
if (mb_strlen($naked_body) < intval($min_length)) {
......@@ -381,7 +379,7 @@ function lang_selector() {
$o = replace_macros($tpl, array(
'$title' => t('Select an alternate language'),
'$langs' => array($lang_options, $selected),
));
return $o;
......
This diff is collapsed.
<?php
/**
* example usage (CLI)
*
* @package Text_LanguageDetect
* @version CVS: $Id: example_clui.php 322305 2012-01-15 00:04:17Z clockwerx $
*/
require_once 'Text/LanguageDetect.php';
$l = new Text_LanguageDetect;
$stdin = fopen('php://stdin', 'r');
echo "Supported languages:\n";
$langs = $l->getLanguages();
sort($langs);
echo join(', ', $langs);
echo "\ntotal ", count($langs), "\n\n";
while ($line = fgets($stdin)) {
$result = $l->detect($line, 4);
print_r($result);
$blocks = $l->detectUnicodeBlocks($line, true);
print_r($blocks);
}
fclose($stdin);
unset($l);
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
?>
<?php
/**
* example usage (web)
*
* @package Text_LanguageDetect
* @version CVS: $Id: example_web.php 205493 2006-01-18 00:26:57Z taak $
*/
// browsers will encode multi-byte characters wrong unless they think the page is utf8-encoded
header('Content-type: text/html; charset=utf-8', true);
require_once 'Text/LanguageDetect.php';
$l = new Text_LanguageDetect;
if (isset($_REQUEST['q'])) {
$q = stripslashes($_REQUEST['q']);
}
?>
<html>
<head>
<title>Text_LanguageDetect demonstration</title>
</head>
<body>
<h2>Text_LanguageDetect</h2>
<?
echo "<small>Supported languages:\n";
$langs = $l->getLanguages();
sort($langs);
foreach ($langs as $lang) {
echo ucfirst($lang), ', ';
$i++;
}
echo "<br />total $i</small><br /><br />";
?>
<form method="post">
Enter text to identify language (at least a couple of sentences):<br />
<textarea name="q" wrap="virtual" cols="80" rows="8"><?= $q ?></textarea>
<br />
<input type="submit" value="Submit" />
</form>
<?
if (isset($q) && strlen($q)) {
$len = $l->utf8strlen($q);
if ($len < 20) { // this value picked somewhat arbitrarily
echo "Warning: string not very long ($len chars)<br />\n";
}
$result = $l->detectConfidence($q);
if ($result == null) {
echo "Text_LanguageDetect cannot identify this piece of text. <br /><br />\n";
} else {
echo "Text_LanguageDetect thinks this text is written in <b>{$result['language']}</b> ({$result['similarity']}, {$result['confidence']})<br /><br />\n";
}
$result = $l->detectUnicodeBlocks($q, false);
if (!empty($result)) {
arsort($result);
echo "Unicode blocks present: ", join(', ', array_keys($result)), "\n<br /><br />";
}
}
unset($l);
/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
?>
</body></html>
<?php
/**
* Demonstrates how to use ISO language codes.
*
* The "name mode" changes the way languages are accepted and returned.
*/
require_once 'Text/LanguageDetect.php';
$l = new Text_LanguageDetect();
//will output the ISO 639-1 two-letter language code
// "de"
$l->setNameMode(2);
echo $l->detectSimple('Das ist ein kleiner Text') . "\n";
//will output the ISO 639-2 three-letter language code
// "deu"
$l->setNameMode(3);
echo $l->detectSimple('Das ist ein kleiner Text') . "\n";
?>
\ No newline at end of file
This diff is collapsed.
<?php
set_include_path(
__DIR__ . '/../' . PATH_SEPARATOR . get_include_path()
);
require_once 'Text/LanguageDetect/ISO639.php';
class Text_LanguageDetect_ISO639Test extends PHPUnit_Framework_TestCase
{
public function testNameToCode2()
{
$this->assertEquals(
'de',
Text_LanguageDetect_ISO639::nameToCode2('german')
);
}
public function testNameToCode2Fail()
{
$this->assertNull(
Text_LanguageDetect_ISO639::nameToCode2('doesnotexist')
);
}
public function testNameToCode3()
{
$this->assertEquals(
'fra',
Text_LanguageDetect_ISO639::nameToCode3('french')
);
}
public function testNameToCode3Fail()
{
$this->assertNull(
Text_LanguageDetect_ISO639::nameToCode3('doesnotexist')
);
}
public function testCode2ToName()
{
$this->assertEquals(
'english',
Text_LanguageDetect_ISO639::code2ToName('en')
);
}
public function testCode2ToNameFail()
{
$this->assertNull(
Text_LanguageDetect_ISO639::code2ToName('nx')
);
}
public function testCode3ToName()
{
$this->assertEquals(
'romanian',
Text_LanguageDetect_ISO639::code3ToName('rom')
);
}
public function testCode3ToNameFail()
{
$this->assertNull(
Text_LanguageDetect_ISO639::code3ToName('nxx')
);
}
}
?>
\ No newline at end of file
......@@ -24,7 +24,9 @@ class FeedutilsTest extends UnitTestCase {
$b = ['attribs' => ['' => [
'rel' => 'rel_value',
'type' => 'type_value',
'href' => 'href_value'
'href' => 'href_value',
'length' => 'length_value',
'title' => 'title_value'
]]];
$blink1 = ['link1' => $b];
$bresult[] = $b['attribs'][''];
......@@ -37,6 +39,9 @@ class FeedutilsTest extends UnitTestCase {
//Illegal string offset 'attribs'
}*/
/**
* @uses ::xmlify
*/
public function test_atom_author() {
$this->assertEquals('', atom_author('', 'nick', 'name', 'uri', 72, 72, 'png', 'photourl'));
......@@ -47,7 +52,7 @@ class FeedutilsTest extends UnitTestCase {
<link rel="photo" type="png" media:width="72" media:height="72" href="http://photourl" />
<link rel="avatar" type="png" media:width="72" media:height="72" href="http://photourl" />
<poco:preferredUsername>nick</poco:preferredUsername>
<poco:displayName>name<poco:displayName>
<poco:displayName>name</poco:displayName>
</tag>';
$this->assertXmlStringEqualsXmlString($a, atom_author('tag', 'nick', 'name', 'uri', 72, 72, 'png', 'http://photourl'));
......
......@@ -52,7 +52,6 @@ class LanguageTest extends UnitTestCase {
// Can not unit test detect_language(), therefore test the used library
// only for now to find regressions on library updates.
require_once('library/langdet/Text/LanguageDetect.php');
$l = new Text_LanguageDetect;
// return 2-letter ISO 639-1 (en) language code
$l->setNameMode(2);
......@@ -71,8 +70,8 @@ class LanguageTest extends UnitTestCase {
],
'English' => [
'English is a West Germanic language that was first spoken in early medieval England and is now a global lingua franca.[4][5] Named after the Angles, one of the Germanic tribes that migrated to England, it ultimately derives its name from the Anglia (Angeln) peninsula in the Baltic Sea. It is closely related to the Frisian languages, but its vocabulary has been significantly influenced by other Germanic languages, particularly Norse (a North Germanic language), as well as by Latin and Romance languages, especially French.',
'da', // nearly impossible to find a wikipedia article which is detected as english
0.000367 // minimum confidence is checked in detect_language(), but that is not yet unit testable
'en',
0.078422
],
'German' => [
'Deutschland ist ein Bundesstaat in Mitteleuropa. Er besteht aus 16 Ländern und ist als freiheitlich-demokratischer und sozialer Rechtsstaat verfasst. Die Bundesrepublik Deutschland stellt die jüngste Ausprägung des deutschen Nationalstaates dar. Mit rund 82,8 Millionen Einwohnern (31. Dezember 2016) zählt Deutschland zu den dicht besiedelten Flächenstaaten.',
......
<?php
// include_paths.php @generated by Composer
$vendorDir = dirname(dirname(__FILE__));
$baseDir = dirname($vendorDir);
return array(
$vendorDir . '/pear/text_languagedetect',
);
*******************
Text_LanguageDetect
*******************
PHP library to identify human languages from text samples.
Returns confidence scores for each.
Installation
============
PEAR
----
::
$ pear install Text_LanguageDetect
Composer
--------
::
$ composer require pear/text_languagedetect
Usage
=====
Also see the examples in the ``docs/`` directory and
the `official documentation`__.
__ http://pear.php.net/package/Text_LanguageDetect/docs
Language detection
------------------
Simple language detection::
<?php
require_once 'Text/LanguageDetect.php';
$text = 'Was wäre, wenn ich Ihnen das jetzt sagen würde?';
$ld = new Text_LanguageDetect();
$language = $ld->detectSimple($text);
echo $language;
//output: german
Show the three most probable languages with their confidence score::
<?php
require_once 'Text/LanguageDetect.php';
$text = 'Was wäre, wenn ich Ihnen das jetzt sagen würde?';
$ld = new Text_LanguageDetect();
//3 most probable languages
$results = $ld->detect($text, 3);
foreach ($results as $language => $confidence) {
echo $language . ': ' . number_format($confidence, 2) . "\n";
}
//output:
//german: 0.35
//dutch: 0.25
//swedish: 0.20
?>
Language code
-------------
Instead of returning the full language name, ISO 639-2 two and three
letter codes can be returned::
<?php
require_once 'Text/LanguageDetect.php';
$ld = new Text_LanguageDetect();
//will output the ISO 639-1 two-letter language code
// "de"
$ld->setNameMode(2);
echo $ld->detectSimple('Das ist ein kleiner Text') . "\n";
//will output the ISO 639-2 three-letter language code
// "deu"
$ld->setNameMode(3);
echo $ld->detectSimple('Das ist ein kleiner Text') . "\n";
?>
Supported languages
===================
- albanian
- arabic
- azeri
- bengali
- bulgarian
- cebuano
- croatian
- czech
- danish
- dutch
- english
- estonian
- farsi
- finnish
- french
- german
- hausa
- hawaiian
- hindi
- hungarian
- icelandic
- indonesian
- italian
- kazakh
- kyrgyz
- latin
- latvian
- lithuanian
- macedonian
- mongolian
- nepali
- norwegian
- pashto
- pidgin
- polish
- portuguese
- romanian
- russian
- serbian
- slovak
- slovene
- somali
- spanish
- swahili
- swedish
- tagalog
- turkish
- ukrainian
- urdu
- uzbek
- vietnamese
- welsh
Links
=====
Homepage
http://pear.php.net/package/Text_LanguageDetect
Bug tracker
http://pear.php.net/bugs/search.php?cmd=display&package_name[]=Text_LanguageDetect
Documentation
http://pear.php.net/package/Text_LanguageDetect/docs
Unit test status
https://travis-ci.org/pear/Text_LanguageDetect
.. image:: https://travis-ci.org/pear/Text_LanguageDetect.svg?branch=master
:target: https://travis-ci.org/pear/Text_LanguageDetect
<?php
/**
* Part of Text_LanguageDetect
*
* PHP version 5
*
* @category Text
* @package Text_LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @link http://pear.php.net/package/Text_LanguageDetect/
*/
/**
* Part of the PEAR language detection package
*
* PHP version 5
*
* @category Text
* @package Text_LanguageDetect
* @author Nicholas Pisarro <infinityminusnine+pear@gmail.com>
* @license BSD http://www.opensource.org/licenses/bsd-license.php
* @link http://pear.php.net/package/Text_LanguageDetect/
* @link http://langdetect.blogspot.com/
*/
class Text_LanguageDetect_Exception extends Exception
{