Commit 6c79e0c0 authored by Klaus Weidenbach's avatar Klaus Weidenbach

🔨 Upgrade Markdownify library.

The current version 2.0.0 (alpha) throws deprecated warning with
PHP7.1 and PHPUnit.
Upgrade the HTML to Markdown converter for PHP to the current
Markdownify 2.2.1.
Used composer to manage this library.
parent 8e171606
......@@ -29,7 +29,8 @@
"ext-xml" : "*",
"ext-openssl" : "*",
"sabre/dav" : "~3.2",
"michelf/php-markdown" : "^1.7"
"michelf/php-markdown" : "^1.7",
"pixel418/markdownify": "^2.2"
},
"require-dev" : {
"php" : ">=5.6",
......
......@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"content-hash": "4b24468c1f8babe7c8804fba8ee602f7",
"content-hash": "c0cafbf9fd702be588f6b392b9742cb6",
"packages": [
{
"name": "michelf/php-markdown",
......@@ -57,6 +57,62 @@
],
"time": "2016-10-29T18:58:20+00:00"
},
{
"name": "pixel418/markdownify",
"version": "v2.2.1",
"source": {
"type": "git",
"url": "https://github.com/Elephant418/Markdownify.git",
"reference": "0160677f04c784550dd10fd72fdf3994967db848"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Elephant418/Markdownify/zipball/0160677f04c784550dd10fd72fdf3994967db848",
"reference": "0160677f04c784550dd10fd72fdf3994967db848",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "^4.8"
},
"type": "lib",
"autoload": {
"psr-4": {
"Markdownify\\": "src",
"Test\\Markdownify\\": "test"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"LGPL"
],
"authors": [
{
"name": "Peter Kruithof",
"email": "pkruithof@gmail.com",
"homepage": "http://pkruithof.tumblr.com/"
},
{
"name": "Milian Wolff",
"email": "mail@milianw.de",
"homepage": "http://milianw.de"
},
{
"name": "Thomas Zilliox",
"email": "hello@tzi.fr",
"homepage": "http://tzi.fr"
}
],
"description": "The HTML to Markdown converter for PHP ",
"homepage": "https://github.com/elephant418/Markdownify",
"keywords": [
"markdown",
"markdownify"
],
"time": "2016-09-21T13:01:43+00:00"
},
{
"name": "psr/log",
"version": "1.0.2",
......
......@@ -5,12 +5,12 @@
*/
use Michelf\MarkdownExtra;
use Markdownify\Converter;
require_once("include/oembed.php");
require_once("include/event.php");
require_once("include/html2bbcode.php");
require_once("include/bbcode.php");
require_once("library/markdownify/markdownify.php");
function get_bb_tag_pos($s, $name, $occurance = 1) {
......@@ -367,7 +367,6 @@ function bb2diaspora_itemwallwall(&$item,$uplink = false) {
function bb2diaspora_itembody($item, $force_update = false, $have_channel = false, $uplink = false) {
if(! get_iconfig($item,'diaspora','fields')) {
$force_update = true;
}
......@@ -454,7 +453,7 @@ function bb2diaspora_itembody($item, $force_update = false, $have_channel = fals
return html_entity_decode($body);
}
function bb2diaspora($Text,$preserve_nl = false, $fordiaspora = true) {
function bb2diaspora($Text, $preserve_nl = false, $fordiaspora = true) {
// Re-enabling the converter again.
// The bbcode parser now handles youtube-links (and the other stuff) correctly.
......@@ -496,11 +495,10 @@ function bb2diaspora($Text,$preserve_nl = false, $fordiaspora = true) {
$Text = str_replace(array('<','>','&'),array('&_lt_;','&_gt_;','&_amp_;'),$Text);
// Now convert HTML to Markdown
$md = new Markdownify(false, false, false);
$md = new Converter(Converter::LINK_AFTER_CONTENT, false, false);
$Text = $md->parseString($Text);
// It also adds backslashes to our attempt at getting around the html entity preservation for some weird reason.
$Text = str_replace(array('&\\_lt\\_;','&\\_gt\\_;','&\\_amp\\_;'),array('<','>','&'),$Text);
......@@ -522,7 +520,7 @@ function bb2diaspora($Text,$preserve_nl = false, $fordiaspora = true) {
$Text = trim($Text);
call_hooks('bb2diaspora',$Text);
call_hooks('bb2diaspora', $Text);
return $Text;
}
......
Markdownify
===========
* handle non-markdownifiable lists (i.e. `<ul><li id="foobar">asdf</li></ul>`)
* organize methods better (i.e. flushlinebreaks & setlinebreaks close to each other)
* take a look at function names etc.
* is the new (in rev. 93) lastclosedtag property needed?
* word wrapping (some work is done but it's still very buggy)
Markdownify Extra
=================
* handle table alignment with KEEP_HTML=false
* handle tables without headings when KEEP_HTML=false is set
* handle Markdown inside non-markdownable tags
Implementation Thoughts
=======================
* non-markdownifiable lists and markdown inside non-markdownable tags as well as the current
table implementation could be rewritten by using a rollback mechanism.
example:
<ul><li>asdf</li><li id="foobar">asdf</li></ul>
we come to `<ul>`, know that this might fail and create a snapshot of our current parser
we keep on parsing and when we reach `<li id="foobar">` we gotta rollback and keep this
list in HTML format.
<?php
error_reporting(E_ALL);
if (!empty($_POST['input'])) {
include 'markdownify_extra.php';
if (!isset($_POST['leap'])) {
$leap = MDFY_LINKS_EACH_PARAGRAPH;
} else {
$leap = $_POST['leap'];
}
if (!isset($_POST['keepHTML'])) {
$keephtml = MDFY_KEEPHTML;
} else {
$keephtml = $_POST['keepHTML'];
}
if (!empty($_POST['extra'])) {
$md = new Markdownify_Extra($leap, MDFY_BODYWIDTH, $keephtml);
} else {
$md = new Markdownify($leap, MDFY_BODYWIDTH, $keephtml);
}
if (ini_get('magic_quotes_gpc')) {
$_POST['input'] = stripslashes($_POST['input']);
}
$output = $md->parseString($_POST['input']);
} else {
$_POST['input'] = '';
}
?><!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en-US" lang="en-US">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<title>HTML to Markdown Converter</title>
</head>
<body>
<?php if (empty($_POST['input'])): ?>
<form action="<?php echo $_SERVER['PHP_SELF']; ?>" method="post">
<fieldset>
<legend>HTML Input</legend>
<textarea style="width:100%;" cols="85" rows="40" name="input"><?php echo htmlspecialchars($_POST['input'], ENT_NOQUOTES, 'UTF-8'); ?></textarea>
</fieldset>
<label for="extra">Markdownify Extra: <input name="extra" checked="checked" id="extra" type="checkbox" value="1" /></label>
<label for="leap">Links after each block elem: <input name="leap" id="leap" type="checkbox" value="1" /></label>
<label for="keepHTML">keep HTML: <input name="keepHTML" id="keepHTML" type="checkbox" value="1" checked="checked" /></label>
<input type="submit" name="submit" value="submit" />
</form>
<?php else: ?>
<h1 style="text-align:right;"><a href="<?php echo $_SERVER['PHP_SELF']; ?>">BACK</a></h1>
<pre><?php echo htmlspecialchars($output, ENT_NOQUOTES, 'UTF-8'); ?></pre>
<?php endif; ?>
</body>
</html>
\ No newline at end of file
<?php
/**
* Markdownify converts HTML Markup to [Markdown][1] (by [John Gruber][2]. It
* also supports [Markdown Extra][3] by [Michel Fortin][4] via Markdownify_Extra.
*
* It all started as `html2text.php` - a port of [Aaron Swartz'][5] [`html2text.py`][6] - but
* got a long way since. This is far more than a mere port now!
* Starting with version 2.0.0 this is a complete rewrite and cannot be
* compared to Aaron Swatz' `html2text.py` anylonger. I'm now using a HTML parser
* (see `parsehtml.php` which I also wrote) which makes most of the evil
* RegEx magic go away and additionally it gives a much cleaner class
* structure. Also notably is the fact that I now try to prevent regressions by
* utilizing testcases of Michel Fortin's [MDTest][7].
*
* [1]: http://daringfireball.com/projects/markdown
* [2]: http://daringfireball.com/
* [3]: http://www.michelf.com/projects/php-markdown/extra/
* [4]: http://www.michelf.com/
* [5]: http://www.aaronsw.com/
* [6]: http://www.aaronsw.com/2002/html2text/
* [7]: http://article.gmane.org/gmane.text.markdown.general/2540
*
* @version 2.0.0 alpha
* @author Milian Wolff (<mail@milianw.de>, <http://milianw.de>)
* @license LGPL, see LICENSE_LGPL.txt and the summary below
* @copyright (C) 2007 Milian Wolff
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* HTML Parser, see http://sf.net/projects/parseHTML
*/
require_once dirname(__FILE__).'/parsehtml/parsehtml.php';
/**
* default configuration
*/
define('MDFY_LINKS_EACH_PARAGRAPH', false);
define('MDFY_BODYWIDTH', false);
define('MDFY_KEEPHTML', true);
/**
* HTML to Markdown converter class
*/
class Markdownify {
/**
* html parser object
*
* @var parseHTML
*/
var $parser;
/**
* markdown output
*
* @var string
*/
var $output;
/**
* stack with tags which where not converted to html
*
* @var array<string>
*/
var $notConverted = array();
/**
* skip conversion to markdown
*
* @var bool
*/
var $skipConversion = false;
/* options */
/**
* keep html tags which cannot be converted to markdown
*
* @var bool
*/
var $keepHTML = false;
/**
* wrap output, set to 0 to skip wrapping
*
* @var int
*/
var $bodyWidth = 0;
/**
* minimum body width
*
* @var int
*/
var $minBodyWidth = 25;
/**
* display links after each paragraph
*
* @var bool
*/
var $linksAfterEachParagraph = false;
/**
* constructor, set options, setup parser
*
* @param bool $linksAfterEachParagraph wether or not to flush stacked links after each paragraph
* defaults to false
* @param int $bodyWidth wether or not to wrap the output to the given width
* defaults to false
* @param bool $keepHTML wether to keep non markdownable HTML or to discard it
* defaults to true (HTML will be kept)
* @return void
*/
function Markdownify($linksAfterEachParagraph = MDFY_LINKS_EACH_PARAGRAPH, $bodyWidth = MDFY_BODYWIDTH, $keepHTML = MDFY_KEEPHTML) {
$this->linksAfterEachParagraph = $linksAfterEachParagraph;
$this->keepHTML = $keepHTML;
if ($bodyWidth > $this->minBodyWidth) {
$this->bodyWidth = intval($bodyWidth);
} else {
$this->bodyWidth = false;
}
$this->parser = new parseHTML;
$this->parser->noTagsInCode = true;
# we don't have to do this every time
$search = array();
$replace = array();
foreach ($this->escapeInText as $s => $r) {
array_push($search, '#(?<!\\\)'.$s.'#U');
array_push($replace, $r);
}
$this->escapeInText = array(
'search' => $search,
'replace' => $replace
);
}
/**
* parse a HTML string
*
* @param string $html
* @return string markdown formatted
*/
function parseString($html) {
$this->parser->html = $html;
$this->parse();
return $this->output;
}
/**
* tags with elements which can be handled by markdown
*
* @var array<string>
*/
var $isMarkdownable = array(
'p' => array(),
'ul' => array(),
'ol' => array(),
'li' => array(),
'br' => array(),
'blockquote' => array(),
'code' => array(),
'pre' => array(),
'a' => array(
'href' => 'required',
'title' => 'optional',
),
'strong' => array(),
'b' => array(),
'em' => array(),
'i' => array(),
'img' => array(
'src' => 'required',
'alt' => 'optional',
'title' => 'optional',
),
'h1' => array(),
'h2' => array(),
'h3' => array(),
'h4' => array(),
'h5' => array(),
'h6' => array(),
'hr' => array(),
);
/**
* html tags to be ignored (contents will be parsed)
*
* @var array<string>
*/
var $ignore = array(
'html',
'body',
);
/**
* html tags to be dropped (contents will not be parsed!)
*
* @var array<string>
*/
var $drop = array(
'script',
'head',
'style',
'form',
'area',
'object',
'param',
'iframe',
);
/**
* Markdown indents which could be wrapped
* @note: use strings in regex format
*
* @var array<string>
*/
var $wrappableIndents = array(
'\* ', # ul
'\d. ', # ol
'\d\d. ', # ol
'> ', # blockquote
'', # p
);
/**
* list of chars which have to be escaped in normal text
* @note: use strings in regex format
*
* @var array
*
* TODO: what's with block chars / sequences at the beginning of a block?
*/
var $escapeInText = array(
'([-*_])([ ]{0,2}\1){2,}' => '\\\\$0|', # hr
'\*\*([^*\s]+)\*\*' => '\*\*$1\*\*', # strong
'\*([^*\s]+)\*' => '\*$1\*', # em
'__(?! |_)(.+)(?!<_| )__' => '\_\_$1\_\_', # em
'_(?! |_)(.+)(?!<_| )_' => '\_$1\_', # em
'`(.+)`' => '\`$1\`', # code
'\[(.+)\](\s*\()' => '\[$1\]$2', # links: [text] (url) => [text\] (url)
'\[(.+)\](\s*)\[(.*)\]' => '\[$1\]$2\[$3\]', # links: [text][id] => [text\][id\]
);
/**
* wether last processed node was a block tag or not
*
* @var bool
*/
var $lastWasBlockTag = false;
/**
* name of last closed tag
*
* @var string
*/
var $lastClosedTag = '';
/**
* iterate through the nodes and decide what we
* shall do with the current node
*
* @param void
* @return void
*/
function parse() {
$this->output = '';
# drop tags
$this->parser->html = preg_replace('#<('.implode('|', $this->drop).')[^>]*>.*</\\1>#sU', '', $this->parser->html);
while ($this->parser->nextNode()) {
switch ($this->parser->nodeType) {
case 'doctype':
break;
case 'pi':
case 'comment':
if ($this->keepHTML) {
$this->flushLinebreaks();
$this->out($this->parser->node);
$this->setLineBreaks(2);
}
# else drop
break;
case 'text':
$this->handleText();
break;
case 'tag':
if (in_array($this->parser->tagName, $this->ignore)) {
break;
}
if ($this->parser->isStartTag) {
$this->flushLinebreaks();
}
if ($this->skipConversion) {
$this->isMarkdownable(); # update notConverted
$this->handleTagToText();
continue;
}
if (!$this->parser->keepWhitespace && $this->parser->isBlockElement && $this->parser->isStartTag) {
$this->parser->html = ltrim($this->parser->html);
}
if ($this->isMarkdownable()) {
if ($this->parser->isBlockElement && $this->parser->isStartTag && !$this->lastWasBlockTag && !empty($this->output)) {
if (!empty($this->buffer)) {
$str =& $this->buffer[count($this->buffer) -1];
} else {
$str =& $this->output;
}
if (substr($str, -strlen($this->indent)-1) != "\n".$this->indent) {
$str .= "\n".$this->indent;
}
}
$func = 'handleTag_'.$this->parser->tagName;
$this->$func();
if ($this->linksAfterEachParagraph && $this->parser->isBlockElement && !$this->parser->isStartTag && empty($this->parser->openTags)) {
$this->flushStacked();
}
if (!$this->parser->isStartTag) {
$this->lastClosedTag = $this->parser->tagName;
}
} else {
$this->handleTagToText();
$this->lastClosedTag = '';
}
break;
default:
trigger_error('invalid node type', E_USER_ERROR);
break;
}
$this->lastWasBlockTag = $this->parser->nodeType == 'tag' && $this->parser->isStartTag && $this->parser->isBlockElement;
}
if (!empty($this->buffer)) {
trigger_error('buffer was not flushed, this is a bug. please report!', E_USER_WARNING);
while (!empty($this->buffer)) {
$this->out($this->unbuffer());
}
}
### cleanup
$this->output = rtrim(str_replace('&amp;', '&', str_replace('&lt;', '<', str_replace('&gt;', '>', $this->output))));
# end parsing, flush stacked tags
$this->flushStacked();
$this->stack = array();
}
/**
* check if current tag can be converted to Markdown
*
* @param void
* @return bool
*/
function isMarkdownable() {
if (!isset($this->isMarkdownable[$this->parser->tagName])) {
# simply not markdownable
return false;
}
if ($this->parser->isStartTag) {
$return = true;
if ($this->keepHTML) {
$diff = array_diff(array_keys($this->parser->tagAttributes), array_keys($this->isMarkdownable[$this->parser->tagName]));
if (!empty($diff)) {
# non markdownable attributes given
$return = false;
}
}
if ($return) {
foreach ($this->isMarkdownable[$this->parser->tagName] as $attr => $type) {
if ($type == 'required' && !isset($this->parser->tagAttributes[$attr])) {
# required markdown attribute not given
$return = false;
break;
}
}
}
if (!$return) {
array_push($this->notConverted, $this->parser->tagName.'::'.implode('/', $this->parser->openTags));
}
return $return;
} else {
if (!empty($this->notConverted) && end($this->notConverted) === $this->parser->tagName.'::'.implode('/', $this->parser->openTags)) {
array_pop($this->notConverted);
return false;
}
return true;
}
}
/**
* output all stacked tags
*
* @param void
* @return void
*/
function flushStacked() {
# links
foreach ($this->stack as $tag => $a) {
if (!empty($a)) {
call_user_func(array(&$this, 'flushStacked_'.$tag));
}
}
}
/**
* output link references (e.g. [1]: http://example.com "title");
*
* @param void
* @return void
*/
function flushStacked_a() {
$out = false;
foreach ($this->stack['a'] as $k => $tag) {
if (!isset($tag['unstacked'])) {
if (!$out) {
$out = true;
$this->out("\n\n", true);
} else {
$this->out("\n", true);
}
$this->out(' ['.$tag['linkID'].']: '.$tag['href'].(isset($tag['title']) ? ' "'.$tag['title'].'"' : ''), true);
$tag['unstacked'] = true;
$this->stack['a'][$k] = $tag;
}
}
}
/**
* flush enqued linebreaks
*
* @param void
* @return void
*/
function flushLinebreaks() {
if ($this->lineBreaks && !empty($this->output)) {
$this->out(str_repeat("\n".$this->indent, $this->lineBreaks), true);
}
$this->lineBreaks = 0;
}
/**
* handle non Markdownable tags
*
* @param void
* @return void
*/
function handleTagToText() {
if (!$this->keepHTML) {
if (!$this->parser->isStartTag && $this->parser->isBlockElement) {
$this->setLineBreaks(2);
}
} else {
# dont convert to markdown inside this tag
/** TODO: markdown extra **/
if (!$this->parser->isEmptyTag) {
if ($this->parser->isStartTag) {
if (!$this->skipConversion) {
$this->skipConversion = $this->parser->tagName.'::'.implode('/', $this->parser->openTags);
}
} else {
if ($this->skipConversion == $this->parser->tagName.'::'.implode('/', $this->parser->openTags)) {
$this->skipConversion = false;
}
}
}
if ($this->parser->isBlockElement) {
if ($this->parser->isStartTag) {
if (in_array($this->parent(), array('ins', 'del'))) {
# looks like ins or del are block elements now
$this->out("\n", true);
$this->indent(' ');
}
if ($this->parser->tagName != 'pre') {
$this->out($this->parser->node."\n".$this->indent);
if (!$this->parser->isEmptyTag) {
$this->indent(' ');
} else {
$this->setLineBreaks(1);
}
$this->parser->html = ltrim($this->parser->html);
} else {
# don't indent inside <pre> tags
$this->out($this->parser->node);
static $indent;
$indent = $this->indent;
$this->indent = '';
}
} else {
if (!$this->parser->keepWhitespace) {
$this->output = rtrim($this->output);
}
if ($this->parser->tagName != 'pre') {
$this->indent(' ');
$this->out("\n".$this->indent.$this->parser->node);
} else {
# reset indentation
$this->out($this->parser->node);
static $indent;
$this->indent = $indent;
}
if (in_array($this->parent(), array('ins', 'del'))) {
# ins or del was block element
$this->out("\n");
$this->indent(' ');
}
if ($this->parser->tagName == 'li') {
$this->setLineBreaks(1);
} else {
$this->setLineBreaks(2);
}
}
} else {
$this->out($this->parser->node);
}
if (in_array($this->parser->tagName, array('code', 'pre'))) {
if ($this->parser->isStartTag) {
$this->buffer();
} else {
# add stuff so cleanup just reverses this
$this->out(str_replace('&lt;', '&amp;lt;', str_replace('&gt;', '&amp;gt;', $this->unbuffer())));