From f8f7cc9292e2f01637f0fec991229e8ffe16b83b Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 18 Nov 2014 23:55:45 +0100 Subject: [PATCH 01/11] API: Possibilty of throttling when posting limit is reached --- include/api.php | 61 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/include/api.php b/include/api.php index 374a718479..ffa5d0e9df 100644 --- a/include/api.php +++ b/include/api.php @@ -169,7 +169,7 @@ $json = json_encode($rr); if ($_GET['callback']) $json = $_GET['callback']."(".$json.")"; - return $json; + return $json; break; case "rss": header ("Content-Type: application/rss+xml"); @@ -681,6 +681,7 @@ logger('api_statuses_update: no user'); return false; } + $user_info = api_get_user($a); // convert $_POST array items to the form we use for web posts. @@ -725,6 +726,64 @@ if($parent) $_REQUEST['type'] = 'net-comment'; else { + // Check for throttling (maximum posts per day, week and month) + $throttle_day = get_config('system','throttle_limit_day'); + if ($throttle_day > 0) { + $datefrom = date("Y-m-d H:i:s", time() - 24*60*60); + + $r = q("SELECT COUNT(*) AS `posts_day` FROM `item` WHERE `uid`=%d AND `wall` + AND `created` > '%s' AND `id` = `parent`", + intval(api_user()), dbesc($datefrom)); + + if ($r) + $posts_day = $r[0]["posts_day"]; + else + $posts_day = 0; + + if ($posts_day > $throttle_day) { + logger('Daily posting limit reached for user '.api_user(), LOGGER_DEBUG); + die(api_error($a, $type, sprintf(t("Daily posting limit of %d posts reached. The post was rejected."), $throttle_day))); + } + } + + $throttle_week = get_config('system','throttle_limit_week'); + if ($throttle_week > 0) { + $datefrom = date("Y-m-d H:i:s", time() - 24*60*60*7); + + $r = q("SELECT COUNT(*) AS `posts_week` FROM `item` WHERE `uid`=%d AND `wall` + AND `created` > '%s' AND `id` = `parent`", + intval(api_user()), dbesc($datefrom)); + + if ($r) + $posts_week = $r[0]["posts_week"]; + else + $posts_week = 0; + + if ($posts_week > $throttle_week) { + logger('Weekly posting limit reached for user '.api_user(), LOGGER_DEBUG); + die(api_error($a, $type, sprintf(t("Weekly posting limit of %d posts reached. The post was rejected."), $throttle_week))); + } + } + + $throttle_month = get_config('system','throttle_limit_month'); + if ($throttle_month > 0) { + $datefrom = date("Y-m-d H:i:s", time() - 24*60*60*30); + + $r = q("SELECT COUNT(*) AS `posts_month` FROM `item` WHERE `uid`=%d AND `wall` + AND `created` > '%s' AND `id` = `parent`", + intval(api_user()), dbesc($datefrom)); + + if ($r) + $posts_month = $r[0]["posts_month"]; + else + $posts_month = 0; + + if ($posts_month > $throttle_month) { + logger('Monthly posting limit reached for user '.api_user(), LOGGER_DEBUG); + die(api_error($a, $type, sprintf(t("Monthly posting limit of %d posts reached. The post was rejected."), $throttle_month))); + } + } + $_REQUEST['type'] = 'wall'; if(x($_FILES,'media')) { // upload the image if we have one From 7b26aa2b84d5679168cfa0b61d3329a173377cce Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 1 Dec 2014 22:45:13 +0100 Subject: [PATCH 02/11] When exporting shared posts to diaspora now removed hashtags will be added again. --- include/bb2diaspora.php | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/include/bb2diaspora.php b/include/bb2diaspora.php index 39742291fd..d836f325e8 100644 --- a/include/bb2diaspora.php +++ b/include/bb2diaspora.php @@ -56,6 +56,8 @@ function diaspora2bb($s) { function bb2diaspora($Text,$preserve_nl = false, $fordiaspora = true) { + $OriginalText = $Text; + // Since Diaspora is creating a summary for links, this function removes them before posting if ($fordiaspora) $Text = bb_remove_share_information($Text); @@ -73,9 +75,20 @@ function bb2diaspora($Text,$preserve_nl = false, $fordiaspora = true) { $Text = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '[img]$3[/img]', $Text); // Convert it to HTML - don't try oembed - if ($fordiaspora) + if ($fordiaspora) { $Text = bbcode($Text, $preserve_nl, false, 3); - else { + + // Add all tags that maybe were removed + if (preg_match_all("/#\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",$OriginalText, $tags)) { + $tagline = ""; + foreach($tags[2] as $tag) + if (!strpos($Text, "#".$tag)) + $tagline .= "#".$tag." "; + + $Text = $Text."
".$tagline; + } + + } else { $Text = bbcode($Text, $preserve_nl, false, 4); // Libertree doesn't convert a harizontal rule if there isn't a linefeed $Text = str_replace("
", "
", $Text); From e475cf131d96ac1b9fc82ae598acad811cd9e4aa Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 1 Dec 2014 22:49:35 +0100 Subject: [PATCH 03/11] When a post contains a picture then a picture in the added information is displayed only as a small preview picture. --- include/bbcode.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/include/bbcode.php b/include/bbcode.php index 5395548f51..724b8e2fdd 100644 --- a/include/bbcode.php +++ b/include/bbcode.php @@ -3,10 +3,10 @@ require_once("include/oembed.php"); require_once('include/event.php'); function bb_attachment($Text, $plaintext = false, $tryoembed = true) { - $Text = preg_replace_callback("/\[attachment(.*?)\](.*?)\[\/attachment\]/ism", + $Text = preg_replace_callback("/(.*?)\[attachment(.*?)\](.*?)\[\/attachment\]/ism", function ($match) use ($plaintext){ - $attributes = $match[1]; + $attributes = $match[2]; $type = ""; preg_match("/type='(.*?)'/ism", $attributes, $matches); @@ -65,6 +65,11 @@ function bb_attachment($Text, $plaintext = false, $tryoembed = true) { $preview = $matches[1]; } + if (((strpos($match[1], "[img=") !== false) OR (strpos($match[1], "[img]") !== false)) AND ($image != "")) { + $preview = $image; + $image = ""; + } + if ($plaintext) $text = sprintf('%s
', $url, $title); else { @@ -83,10 +88,10 @@ function bb_attachment($Text, $plaintext = false, $tryoembed = true) { $text .= $oembed; - $text .= sprintf('
%s
', trim($match[2])); + $text .= sprintf('
%s
', trim($match[3])); } - return($text); + return($match[1].$text); },$Text); return($Text); From d2d3804e091011f80e1cc1da9b5ed20769e4b177 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 1 Dec 2014 22:51:41 +0100 Subject: [PATCH 04/11] If a link contains BBCode stuff, make a short link out of this to avoid parsing problems --- include/items.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/items.php b/include/items.php index 5b27559357..0398f54588 100644 --- a/include/items.php +++ b/include/items.php @@ -903,6 +903,12 @@ function add_page_info_data($data) { if ($no_photos AND ($data["type"] == "photo")) return(""); + // If the link contains BBCode stuff, make a short link out of this to avoid parsing problems + if (strpos($data["url"], '[') OR strpos($data["url"], ']')) { + require_once("include/network.php"); + $data["url"] = short_link($data["url"]); + } + if (($data["type"] != "photo") AND is_string($data["title"])) $text .= "[bookmark=".$data["url"]."]".trim($data["title"])."[/bookmark]"; From 0c60eb18f76bd9368f8e4a8276ffd343765c6d47 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 1 Dec 2014 22:52:51 +0100 Subject: [PATCH 05/11] Some more typical tracking parameter is removed --- include/network.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/network.php b/include/network.php index eb29a02b66..6a37f4a549 100644 --- a/include/network.php +++ b/include/network.php @@ -1130,7 +1130,7 @@ function original_url($url, $depth=1, $fetchbody = false) { if (in_array($param, array("utm_source", "utm_medium", "utm_term", "utm_content", "utm_campaign", "wt_mc", "pk_campaign", "pk_kwd", "mc_cid", "mc_eid", "fb_action_ids", "fb_action_types", "fb_ref", - "awesm", + "awesm", "wtrid", "woo_campaign", "woo_source", "woo_medium", "woo_content", "woo_term"))) { $pair = $param."=".urlencode($value); From 9124d922e9e5bf2b9dbc7ea187237f2a6776bc28 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 1 Dec 2014 22:54:01 +0100 Subject: [PATCH 06/11] In "parse_url" now links with [ or ] are replaced with shortened versions to avoid parsing problems. --- mod/parse_url.php | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/mod/parse_url.php b/mod/parse_url.php index 564b22c042..06e59b8691 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -187,6 +187,9 @@ function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $co case "description": $siteinfo["text"] = $attr["content"]; break; + case "thumbnail": + $siteinfo["image"] = $attr["content"]; + break; case "twitter:image": $siteinfo["image"] = $attr["content"]; break; @@ -421,6 +424,12 @@ function parse_url_content(&$a) { $url= $siteinfo["url"]; + // If the link contains BBCode stuff, make a short link out of this to avoid parsing problems + if (strpos($url, '[') OR strpos($url, ']')) { + require_once("include/network.php"); + $url = short_link($url); + } + $sitedata = ""; if($siteinfo["title"] == "") { From 8b5412068e55964aae930480b0feecbbb176708c Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 1 Dec 2014 22:56:41 +0100 Subject: [PATCH 07/11] When selecting forums there was an added "span" html element. --- js/acl.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/acl.js b/js/acl.js index 9c55842fdc..487ffafc77 100644 --- a/js/acl.js +++ b/js/acl.js @@ -62,7 +62,7 @@ ACL.prototype.add_mention = function(id) { that.element.val( searchText + that.element.val() ); } else { if ( tinyMCE.activeEditor.getContent({format : 'raw'}).search(searchText) >= 0 ) return; - tinyMCE.activeEditor.dom.add(tinyMCE.activeEditor.getBody(), 'span', {}, searchText); + tinyMCE.activeEditor.dom.add(tinyMCE.activeEditor.getBody(), 'dummy', {}, searchText); } } From f493252ac8c9cef6626585e1e8de5f7d739bdd83 Mon Sep 17 00:00:00 2001 From: Silke Meyer Date: Sat, 6 Dec 2014 20:05:30 +0100 Subject: [PATCH 08/11] Use a sans-serif font-family in breathe --- view/theme/vier/breathe.css | 1 + 1 file changed, 1 insertion(+) diff --git a/view/theme/vier/breathe.css b/view/theme/vier/breathe.css index 3343cac127..049c1bf4b6 100644 --- a/view/theme/vier/breathe.css +++ b/view/theme/vier/breathe.css @@ -3,6 +3,7 @@ body { background-color: #eeeeee !important; + font-family: "Lucida Sans Unicode","Lucida Sans", sans-serif; } header #banner #logo-text { From c398974918347e3491302d63177c787675f53e74 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 8 Dec 2014 11:02:03 +0100 Subject: [PATCH 09/11] Logging to analyse the stale gprobe processes --- include/gprobe.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/gprobe.php b/include/gprobe.php index 0cf32e95fe..36650eb9ae 100644 --- a/include/gprobe.php +++ b/include/gprobe.php @@ -10,7 +10,7 @@ function gprobe_run(&$argv, &$argc){ if(is_null($a)) { $a = new App; } - + if(is_null($db)) { @include(".htconfig.php"); require_once("include/dba.php"); @@ -37,6 +37,8 @@ function gprobe_run(&$argv, &$argc){ dbesc(normalise_link($url)) ); + logger("gprobe start for ".normalise_link($url), LOGGER_DEBUG); + if(! count($r)) { $arr = probe_url($url); @@ -55,7 +57,8 @@ function gprobe_run(&$argv, &$argc){ } if(count($r)) poco_load(0,0,$r[0]['id'], str_replace('/profile/','/poco/',$r[0]['url'])); - + + logger("gprobe end for ".normalise_link($url), LOGGER_DEBUG); return; } From 620ee6be0bdb4e706abfa2172d3dfb3e2d035e25 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 8 Dec 2014 14:26:44 +0100 Subject: [PATCH 10/11] Cleanup the HTML before parsing it. --- include/Scrape.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/Scrape.php b/include/Scrape.php index 99784af336..ad9e030a39 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -263,6 +263,11 @@ function scrape_feed($url) { } try { + // Cleanup invalid HTML + $doc = new DOMDocument(); + @$doc->loadHTML($s); + $s = $doc->saveHTML(); + $dom = HTML5_Parser::parse($s); } catch (DOMException $e) { logger('scrape_feed: parse error: ' . $e); From 04106ff6f52b03d0d5b009f07a7b6ab11107f0c0 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 8 Dec 2014 22:37:49 +0100 Subject: [PATCH 11/11] Moving the clean up code to a central place. --- include/Scrape.php | 5 ----- library/HTML5/Parser.php | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/Scrape.php b/include/Scrape.php index ad9e030a39..99784af336 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -263,11 +263,6 @@ function scrape_feed($url) { } try { - // Cleanup invalid HTML - $doc = new DOMDocument(); - @$doc->loadHTML($s); - $s = $doc->saveHTML(); - $dom = HTML5_Parser::parse($s); } catch (DOMException $e) { logger('scrape_feed: parse error: ' . $e); diff --git a/library/HTML5/Parser.php b/library/HTML5/Parser.php index 5f9ca560e5..c7faf875ad 100644 --- a/library/HTML5/Parser.php +++ b/library/HTML5/Parser.php @@ -17,6 +17,12 @@ class HTML5_Parser * @return Parsed HTML as DOMDocument */ static public function parse($text, $builder = null) { + + // Cleanup invalid HTML + $doc = new DOMDocument(); + @$doc->loadHTML($text); + $text = $doc->saveHTML(); + $tokenizer = new HTML5_Tokenizer($text, $builder); $tokenizer->parse(); return $tokenizer->save();