friendica
/
friendica
mirror of https://git.friendi.ca/friendica/friendica.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #5765 from MrPetovan/bug/php7-remove-pear-text_highlighter 

[php7] Remove PHP code highlighting
pull/5769/head
Michael Vogel 2018-09-18 18:34:46 +00:00 committed by GitHub
parent 1fa37586ca 71575671a5
commit bd0515714c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 102 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
composer.json
View File

@ -18,13 +18,12 @@
"asika/simple-console": "^1.0",
"divineomega/password_exposed": "^2.4",
"ezyang/htmlpurifier": "~4.7.0",
"league/html-to-markdown": "~4.4.1",
"league/html-to-markdown": "~4.8.0",
"lightopenid/lightopenid": "dev-master",
"michelf/php-markdown": "^1.7",
"mobiledetect/mobiledetectlib": "2.8.*",
"paragonie/random_compat": "^2.0",
"pear/Text_LanguageDetect": "1.*",
"pear/Text_Highlighter": "dev-master",
"seld/cli-prompt": "^1.0",
"smarty/smarty": "^3.1",
"fxp/composer-asset-plugin": "~1.3",

237
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"content-hash": "d62c3e3d6971ee63a862a22ff3cd3768",
"content-hash": "5f6a43237dc52758484cd21cd76e8ce6",
"packages": [
{
"name": "asika/simple-console",
@ -483,16 +483,16 @@
},
{
"name": "league/html-to-markdown",
"version": "4.4.1",
"version": "4.8.0",
"source": {
"type": "git",
"url": "https://github.com/thephpleague/html-to-markdown.git",
"reference": "82ea375b5b2b1da1da222644c0565c695bf88186"
"reference": "f9a879a068c68ff47b722de63f58bec79e448f9d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thephpleague/html-to-markdown/zipball/82ea375b5b2b1da1da222644c0565c695bf88186",
"reference": "82ea375b5b2b1da1da222644c0565c695bf88186",
"url": "https://api.github.com/repos/thephpleague/html-to-markdown/zipball/f9a879a068c68ff47b722de63f58bec79e448f9d",
"reference": "f9a879a068c68ff47b722de63f58bec79e448f9d",
"shasum": ""
},
"require": {
@ -511,7 +511,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "4.5-dev"
"dev-master": "4.9-dev"
}
},
"autoload": {
@ -524,17 +524,17 @@
"MIT"
],
"authors": [
{
"name": "Colin O'Dell",
"email": "colinodell@gmail.com",
"homepage": "http://www.colinodell.com",
"role": "Lead Developer"
},
{
"name": "Nick Cernis",
"email": "nick@cern.is",
"homepage": "http://modernnerd.net",
"role": "Original Author"
},
{
"name": "Colin O'Dell",
"email": "colinodell@gmail.com",
"homepage": "https://www.colinodell.com",
"role": "Lead Developer"
}
],
"description": "An HTML-to-markdown conversion helper for PHP",
@ -543,7 +543,7 @@
"html",
"markdown"
],
"time": "2017-03-16T00:45:59+00:00"
"time": "2018-09-18T12:18:08+00:00"
},
{
"name": "lightopenid/lightopenid",
@ -1669,204 +1669,6 @@
],
"time": "2018-02-15T05:50:20+00:00"
},
{
"name": "pear/console_getopt",
"version": "v1.4.1",
"source": {
"type": "git",
"url": "https://github.com/pear/Console_Getopt.git",
"reference": "82f05cd1aa3edf34e19aa7c8ca312ce13a6a577f"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pear/Console_Getopt/zipball/82f05cd1aa3edf34e19aa7c8ca312ce13a6a577f",
"reference": "82f05cd1aa3edf34e19aa7c8ca312ce13a6a577f",
"shasum": ""
},
"type": "library",
"autoload": {
"psr-0": {
"Console": "./"
}
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
"./"
],
"license": [
"BSD-2-Clause"
],
"authors": [
{
"name": "Greg Beaver",
"email": "cellog@php.net",
"role": "Helper"
},
{
"name": "Andrei Zmievski",
"email": "andrei@php.net",
"role": "Lead"
},
{
"name": "Stig Bakken",
"email": "stig@php.net",
"role": "Developer"
}
],
"description": "More info available on: http://pear.php.net/package/Console_Getopt",
"time": "2015-07-20T20:28:12+00:00"
},
{
"name": "pear/pear-core-minimal",
"version": "v1.10.3",
"source": {
"type": "git",
"url": "https://github.com/pear/pear-core-minimal.git",
"reference": "070f0b600b2caca2501e2c9b7e553016e4b0d115"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pear/pear-core-minimal/zipball/070f0b600b2caca2501e2c9b7e553016e4b0d115",
"reference": "070f0b600b2caca2501e2c9b7e553016e4b0d115",
"shasum": ""
},
"require": {
"pear/console_getopt": "~1.4",
"pear/pear_exception": "~1.0"
},
"replace": {
"rsky/pear-core-min": "self.version"
},
"type": "library",
"autoload": {
"psr-0": {
"": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
"src/"
],
"license": [
"BSD-3-Clause"
],
"authors": [
{
"name": "Christian Weiske",
"email": "cweiske@php.net",
"role": "Lead"
}
],
"description": "Minimal set of PEAR core files to be used as composer dependency",
"time": "2017-02-28T16:46:11+00:00"
},
{
"name": "pear/pear_exception",
"version": "v1.0.0",
"source": {
"type": "git",
"url": "https://github.com/pear/PEAR_Exception.git",
"reference": "8c18719fdae000b690e3912be401c76e406dd13b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pear/PEAR_Exception/zipball/8c18719fdae000b690e3912be401c76e406dd13b",
"reference": "8c18719fdae000b690e3912be401c76e406dd13b",
"shasum": ""
},
"require": {
"php": ">=4.4.0"
},
"require-dev": {
"phpunit/phpunit": "*"
},
"type": "class",
"extra": {
"branch-alias": {
"dev-master": "1.0.x-dev"
}
},
"autoload": {
"psr-0": {
"PEAR": ""
}
},
"notification-url": "https://packagist.org/downloads/",
"include-path": [
"."
],
"license": [
"BSD-2-Clause"
],
"authors": [
{
"name": "Helgi Thormar",
"email": "dufuz@php.net"
},
{
"name": "Greg Beaver",
"email": "cellog@php.net"
}
],
"description": "The PEAR Exception base class.",
"homepage": "https://github.com/pear/PEAR_Exception",
"keywords": [
"exception"
],
"time": "2015-02-10T20:07:52+00:00"
},
{
"name": "pear/text_highlighter",
"version": "dev-master",
"source": {
"type": "git",
"url": "https://github.com/pear/Text_Highlighter.git",
"reference": "2ccac2d9eaf55dc08bbbdb7136c93fb399d0f855"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/pear/Text_Highlighter/zipball/2ccac2d9eaf55dc08bbbdb7136c93fb399d0f855",
"reference": "2ccac2d9eaf55dc08bbbdb7136c93fb399d0f855",
"shasum": ""
},
"require": {
"pear/pear-core-minimal": "~1.10.0",
"php": ">=5.4.0"
},
"require-dev": {
"phpunit/phpunit": "@stable"
},
"type": "library",
"autoload": {
"psr-0": {
"Text": "./"
}
},
"include-path": [
"./"
],
"license": [
"PHP-3.01"
],
"authors": [
{
"email": "ssttoo@gmail.com",
"name": "Stoyan Stefanov",
"role": "Lead"
},
{
"email": "demenev@gmail.com",
"name": "Andrey Demenev",
"role": "Lead"
}
],
"description": "More info available on: http://pear.php.net/package/Text_Highlighter",
"support": {
"issues": "http://pear.php.net/bugs/search.php?cmd=display&package_name[]=Text_Highlighter",
"source": "https://github.com/pear/Text_Highlighter"
},
"time": "2018-01-27T08:24:15+00:00"
},
{
"name": "pear/text_languagedetect",
"version": "v1.0.0",
@ -3145,7 +2947,7 @@
}
],
"description": "Provides the functionality to compare PHP values for equality",
"homepage": "https://github.com/sebastianbergmann/comparator",
"homepage": "http://www.github.com/sebastianbergmann/comparator",
"keywords": [
"comparator",
"compare",
@ -3247,7 +3049,7 @@
}
],
"description": "Provides functionality to handle HHVM/PHP environments",
"homepage": "https://github.com/sebastianbergmann/environment",
"homepage": "http://www.github.com/sebastianbergmann/environment",
"keywords": [
"Xdebug",
"environment",
@ -3315,7 +3117,7 @@
}
],
"description": "Provides the functionality to export PHP variables for visualization",
"homepage": "https://github.com/sebastianbergmann/exporter",
"homepage": "http://www.github.com/sebastianbergmann/exporter",
"keywords": [
"export",
"exporter"
@ -3367,7 +3169,7 @@
}
],
"description": "Snapshotting of global state",
"homepage": "https://github.com/sebastianbergmann/global-state",
"homepage": "http://www.github.com/sebastianbergmann/global-state",
"keywords": [
"global state"
],
@ -3469,7 +3271,7 @@
}
],
"description": "Provides functionality to recursively process PHP variables",
"homepage": "https://github.com/sebastianbergmann/recursion-context",
"homepage": "http://www.github.com/sebastianbergmann/recursion-context",
"time": "2016-11-19T07:33:16+00:00"
},
{
@ -3669,8 +3471,7 @@
"aliases": [],
"minimum-stability": "stable",
"stability-flags": {
"lightopenid/lightopenid": 20,
"pear/text_highlighter": 20
"lightopenid/lightopenid": 20
},
"prefer-stable": false,
"prefer-lowest": false,

55
include/text.php
View File

@ -1911,58 +1911,3 @@ function format_network_name($network, $url = 0) {
return $network_name;
}
}
/**
* @brief Syntax based code highlighting for popular languages.
* @param string $s Code block
* @param string $lang Programming language
* @return string Formated html
*/
function text_highlight($s, $lang) {
if ($lang === 'js') {
$lang = 'javascript';
}
if ($lang === 'bash') {
$lang = 'sh';
}
// @TODO: Replace Text_Highlighter_Renderer_Html by scrivo/highlight.php
// Autoload the library to make constants available
class_exists('Text_Highlighter_Renderer_Html');
$options = [
'numbers' => HL_NUMBERS_LI,
'tabsize' => 4,
];
$tag_added = false;
$s = trim(html_entity_decode($s, ENT_COMPAT));
$s = str_replace(' ', "\t", $s);
/*
* The highlighter library insists on an opening php tag for php code blocks. If
* it isn't present, nothing is highlighted. So we're going to see if it's present.
* If not, we'll add it, and then quietly remove it after we get the processed output back.
*/
if ($lang === 'php' && strpos($s, '<?php') !== 0) {
$s = '<?php' . "\n" . $s;
$tag_added = true;
}
$renderer = new Text_Highlighter_Renderer_Html($options);
$factory = new Text_Highlighter();
$hl = $factory->factory($lang);
$hl->setRenderer($renderer);
$o = $hl->highlight($s);
$o = str_replace("\n", '', $o);
if ($tag_added) {
$b = substr($o, 0, strpos($o, '<li>'));
$e = substr($o, strpos($o, '</li>'));
$o = $b . $e;
}
return '<code>' . $o . '</code>';
}

47
mod/babel.php
View File

@ -6,9 +6,11 @@
use Friendica\Content\Text;
use Friendica\Core\L10n;
function visible_lf($s)
function visible_whitespace($s)
{
return str_replace("\n", '<br />', $s);
$s = str_replace(' ', '&nbsp;', $s);
return str_replace(["\r\n", "\n", "\r"], '<br />', $s);
}
function babel_content()
@ -20,19 +22,19 @@ function babel_content()
$bbcode = trim($_REQUEST['text']);
$results[] = [
'title' => L10n::t('Source input'),
'content' => visible_lf($bbcode)
'content' => visible_whitespace($bbcode)
];
$plain = Text\BBCode::toPlaintext($bbcode, false);
$results[] = [
'title' => L10n::t('BBCode::toPlaintext'),
'content' => visible_lf($plain)
'content' => visible_whitespace($plain)
];
$html = Text\BBCode::convert($bbcode);
$results[] = [
'title' => L10n::t("BBCode::convert \x28raw HTML\x29"),
'content' => htmlspecialchars($html)
'title' => L10n::t('BBCode::convert (raw HTML)'),
'content' => visible_whitespace(htmlspecialchars($html))
];
$results[] = [
@ -43,13 +45,13 @@ function babel_content()
$bbcode2 = Text\HTML::toBBCode($html);
$results[] = [
'title' => L10n::t('BBCode::convert => HTML::toBBCode'),
'content' => visible_lf($bbcode2)
'content' => visible_whitespace($bbcode2)
];
$markdown = Text\BBCode::toMarkdown($bbcode);
$results[] = [
'title' => L10n::t('BBCode::toMarkdown'),
'content' => visible_lf($markdown)
'content' => visible_whitespace($markdown)
];
$html2 = Text\Markdown::convert($markdown);
@ -61,22 +63,33 @@ function babel_content()
$bbcode3 = Text\Markdown::toBBCode($markdown);
$results[] = [
'title' => L10n::t('BBCode::toMarkdown => Markdown::toBBCode'),
'content' => visible_lf($bbcode3)
'content' => visible_whitespace($bbcode3)
];
$bbcode4 = Text\HTML::toBBCode($html2);
$results[] = [
'title' => L10n::t('BBCode::toMarkdown => Markdown::convert => HTML::toBBCode'),
'content' => visible_lf($bbcode4)
'content' => visible_whitespace($bbcode4)
];
break;
case 'markdown':
$markdown = trim($_REQUEST['text']);
$results[] = [
'title' => L10n::t('Source input \x28Diaspora format\x29'),
'title' => L10n::t('Source input (Diaspora format)'),
'content' => '<pre>' . $markdown . '</pre>'
];
$html = Text\Markdown::convert($markdown);
$results[] = [
'title' => L10n::t('Markdown::convert (raw HTML)'),
'content' => htmlspecialchars($html)
];
$results[] = [
'title' => L10n::t('Markdown::convert'),
'content' => $html
];
$bbcode = Text\Markdown::toBBCode($markdown);
$results[] = [
'title' => L10n::t('Markdown::toBBCode'),
@ -86,7 +99,7 @@ function babel_content()
case 'html' :
$html = trim($_REQUEST['text']);
$results[] = [
'title' => L10n::t("Raw HTML input"),
'title' => L10n::t('Raw HTML input'),
'content' => htmlspecialchars($html)
];
@ -98,7 +111,13 @@ function babel_content()
$bbcode = Text\HTML::toBBCode($html);
$results[] = [
'title' => L10n::t('HTML::toBBCode'),
'content' => visible_lf($bbcode)
'content' => visible_whitespace($bbcode)
];
$markdown = Text\HTML::toMarkdown($html);
$results[] = [
'title' => L10n::t('HTML::toMarkdown'),
'content' => visible_whitespace($markdown)
];
$text = Text\HTML::toPlaintext($html);
@ -111,7 +130,7 @@ function babel_content()
$tpl = get_markup_template('babel.tpl');
$o = replace_macros($tpl, [
'$text' => ['text', L10n::t('Source text'), defaults($_REQUEST, 'text', ''), ''],
'$text' => ['text', L10n::t('Source text'), htmlentities(defaults($_REQUEST, 'text', '')), ''],
'$type_bbcode' => ['type', L10n::t('BBCode'), 'bbcode', '', defaults($_REQUEST, 'type', 'bbcode') == 'bbcode'],
'$type_markdown' => ['type', L10n::t('Markdown'), 'markdown', '', defaults($_REQUEST, 'type', 'bbcode') == 'markdown'],
'$type_html' => ['type', L10n::t('HTML'), 'html', '', defaults($_REQUEST, 'type', 'bbcode') == 'html'],

83
src/Content/Text/BBCode.php
View File

@ -25,7 +25,6 @@ use Friendica\Util\Map;
use Friendica\Util\Network;
use Friendica\Util\ParseUrl;
use Friendica\Util\Proxy as ProxyUtils;
use League\HTMLToMarkdown\HtmlConverter;
class BBCode extends BaseObject
{
@ -348,7 +347,7 @@ class BBCode extends BaseObject
*/
public static function toPlaintext($text, $keep_urls = true)
{
$naked_text = preg_replace('/\[(.+?)\]/','', $text);
$naked_text = preg_replace('/\[(.+?)\]\s*/','', $text);
if (!$keep_urls) {
$naked_text = preg_replace('#https?\://[^\s<]+[^\s\.\)]#i', '', $naked_text);
}
@ -1164,21 +1163,6 @@ class BBCode extends BaseObject
return $return;
}
private static function textHighlightCallback($match)
{
// Fallback in case the language doesn't exist
$return = '[code]' . $match[2] . '[/code]';
if (in_array(strtolower($match[1]),
['php', 'css', 'mysql', 'sql', 'abap', 'diff', 'html', 'perl', 'ruby',
'vbscript', 'avrc', 'dtd', 'java', 'xml', 'cpp', 'python', 'javascript', 'js', 'sh', 'bash'])
) {
$return = text_highlight($match[2], strtolower($match[1]));
}
return $return;
}
/**
* @brief Converts a BBCode message to HTML message
*
@ -1227,6 +1211,22 @@ class BBCode extends BaseObject
return $return;
};
// Extracting multi-line code blocks before the whitespace processing
$codeblocks = [];
$text = preg_replace_callback("#\[code(?:=([^\]]*))?\](.*?)\[\/code\]#is",
function ($matches) use (&$codeblocks) {
$return = $matches[0];
if (strpos($matches[2], "\n") !== false) {
$return = '#codeblock-' . count($codeblocks) . '#';
$codeblocks[] = '<pre><code class="language-' . trim($matches[1]) . '">' . trim($matches[2], "\n\r") . '</code></pre>';
}
return $return;
},
$text
);
// Hide all [noparse] contained bbtags by spacefying them
// POSSIBLE BUG --> Will the 'preg' functions crash if there's an embedded image?
@ -1273,11 +1273,6 @@ class BBCode extends BaseObject
$text = preg_replace("/\[share(.*?)avatar\s?=\s?'.*?'\s?(.*?)\]\s?(.*?)\s?\[\/share\]\s?/ism", "\n[share$1$2]$3[/share]", $text);
}
// Check for [code] text here, before the linefeeds are messed with.
// The highlighter will unescape and re-escape the content.
if (strpos($text, '[code=') !== false) {
$text = preg_replace_callback("/\[code=(.*?)\](.*?)\[\/code\]/ism", 'self::textHighlightCallback', $text);
}
// Convert new line chars to html <br /> tags
// nlbr seems to be hopelessly messed up
@ -1771,6 +1766,18 @@ class BBCode extends BaseObject
$text = self::interpolateSavedImagesIntoItemBody($text, $saved_image);
}
// Restore code blocks
$text = preg_replace_callback('/#codeblock-([0-9]+)#/iU',
function ($matches) use ($codeblocks) {
$return = $matches[0];
if (isset($codeblocks[intval($matches[1])])) {
$return = $codeblocks[$matches[1]];
}
return $return;
},
$text
);
// Clean up the HTML by loading and saving the HTML with the DOM.
// Bad structured html can break a whole page.
// For performance reasons do it only with ativated item cache or at export.
@ -1905,23 +1912,6 @@ class BBCode extends BaseObject
// Converting images with size parameters to simple images. Markdown doesn't know it.
$text = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '[img]$3[/img]', $text);
// Extracting multi-line code blocks before the whitespace processing/code highlighter in self::convert()
$codeblocks = [];
$text = preg_replace_callback("#\[code(?:=([^\]]*))?\](.*?)\[\/code\]#is",
function ($matches) use (&$codeblocks) {
$return = $matches[0];
if (strpos($matches[2], "\n") !== false) {
$return = '#codeblock-' . count($codeblocks) . '#';
$prefix = '````' . $matches[1] . PHP_EOL;
$codeblocks[] = $prefix . trim($matches[2]) . PHP_EOL . '````';
}
return $return;
},
$text
);
// Convert it to HTML - don't try oembed
if ($for_diaspora) {
$text = self::convert($text, false, 3);
@ -1951,8 +1941,7 @@ class BBCode extends BaseObject
$stamp1 = microtime(true);
// Now convert HTML to Markdown
$converter = new HtmlConverter();
$text = $converter->convert($text);
$text = HTML::toMarkdown($text);
// unmask the special chars back to HTML
$text = str_replace(['&\_lt\_;', '&\_gt\_;', '&\_amp\_;'], ['&lt;', '&gt;', '&amp;'], $text);
@ -1975,18 +1964,6 @@ class BBCode extends BaseObject
);
}
// Restore code blocks
$text = preg_replace_callback('/#codeblock-([0-9]+)#/iU',
function ($matches) use ($codeblocks) {
$return = '';
if (isset($codeblocks[intval($matches[1])])) {
$return = $codeblocks[$matches[1]];
}
return $return;
},
$text
);
Addon::callHooks('bb2diaspora', $text);
return $text;

20
src/Content/Text/HTML.php
View File

@ -11,6 +11,7 @@ use DOMXPath;
use Friendica\Core\Addon;
use Friendica\Util\Network;
use Friendica\Util\XML;
use League\HTMLToMarkdown\HtmlConverter;
class HTML
{
@ -122,7 +123,7 @@ class HTML
// Removing code blocks before the whitespace removal processing below
$codeblocks = [];
$message = preg_replace_callback(
'#<pre><code(?: class="([^"]*)")?>(.*)</code></pre>#iUs',
'#<pre><code(?: class="language-([^"]*)")?>(.*)</code></pre>#iUs',
function ($matches) use (&$codeblocks) {
$return = '[codeblock-' . count($codeblocks) . ']';
@ -131,7 +132,7 @@ class HTML
$prefix = '[code=' . $matches[1] . ']';
}
$codeblocks[] = $prefix . trim($matches[2]) . '[/code]';
$codeblocks[] = $prefix . PHP_EOL . trim($matches[2]) . PHP_EOL . '[/code]';
return $return;
},
$message
@ -672,4 +673,19 @@ class HTML
return trim($message);
}
/**
* Converts provided HTML code to Markdown. The hardwrap parameter maximizes
* compatibility with Diaspora in spite of the Markdown standards.
*
* @param string $html
* @return string
*/
public static function toMarkdown($html)
{
$converter = new HtmlConverter(['hard_break' => true]);
$markdown = $converter->convert($html);
return $markdown;
}
}

1
src/Content/Text/Markdown.php
View File

@ -32,6 +32,7 @@ class Markdown extends BaseObject
$MarkdownParser = new MarkdownExtra();
$MarkdownParser->hard_wrap = $hardwrap;
$MarkdownParser->code_class_prefix = 'language-';
$html = $MarkdownParser->transform($text);
self::getApp()->save_timestamp($stamp1, "parser");