From 77986d76faf539df20cb598458332ae773bc5631 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:40:46 +0200 Subject: [PATCH 01/12] Preparations for statusnet connector import --- boot.php | 2 ++ include/contact_selectors.php | 4 +++- mod/item.php | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/boot.php b/boot.php index 55318e1ec5..139564ec55 100644 --- a/boot.php +++ b/boot.php @@ -142,6 +142,7 @@ define ( 'NETWORK_GPLUS', 'goog'); // Google+ define ( 'NETWORK_PUMPIO', 'pump'); // pump.io define ( 'NETWORK_TWITTER', 'twit'); // Twitter define ( 'NETWORK_DIASPORA2', 'dspc'); // Diaspora connector +define ( 'NETWORK_STATUSNET', 'stac'); // Statusnet connector define ( 'NETWORK_PHANTOM', 'unkn'); // Place holder @@ -167,6 +168,7 @@ $netgroup_ids = array( NETWORK_PUMPIO => (-13), NETWORK_TWITTER => (-14), NETWORK_DIASPORA2 => (-15), + NETWORK_STATUSNET => (-16), NETWORK_PHANTOM => (-127), ); diff --git a/include/contact_selectors.php b/include/contact_selectors.php index 5e35ecd7e1..dfdd2fa38f 100644 --- a/include/contact_selectors.php +++ b/include/contact_selectors.php @@ -86,7 +86,9 @@ function network_to_name($s) { NETWORK_MAIL2 => t('Email'), NETWORK_GPLUS => t('Google+'), NETWORK_PUMPIO => t('pump.io'), - NETWORK_TWITTER => t('Twitter') + NETWORK_TWITTER => t('Twitter'), + NETWORK_DIASPORA2 => t('Diaspora Connector'), + NETWORK_STATUSNET => t('Statusnet') ); call_hooks('network_to_name', $nets); diff --git a/mod/item.php b/mod/item.php index 346f1a0564..08145d5a0f 100644 --- a/mod/item.php +++ b/mod/item.php @@ -1127,7 +1127,7 @@ function handle_tag($a, &$body, &$inform, &$str_tags, $profile_uid, $tag) { if(count($r)) { $profile = $r[0]['url']; //set newname to nick, find alias - if(($r[0]['network'] === NETWORK_OSTATUS) OR ($r[0]['network'] === NETWORK_TWITTER)) { + if(($r[0]['network'] === NETWORK_OSTATUS) OR ($r[0]['network'] === NETWORK_TWITTER) OR ($r[0]['network'] === NETWORK_STATUSNET)) { $newname = $r[0]['nick']; $stat = true; if($r[0]['alias']) From 63a581bf9bbdfb0853a20a64caff470e64603c92 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:42:12 +0200 Subject: [PATCH 02/12] Removed deprecated twitter code, since twitter had changed its API long time ago. --- include/Scrape.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/include/Scrape.php b/include/Scrape.php index 1df8964f7c..a585047ebd 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -14,7 +14,7 @@ function scrape_dfrn($url) { $s = fetch_url($url); - if(! $s) + if(! $s) return $ret; $headers = $a->get_curl_headers(); @@ -23,7 +23,7 @@ function scrape_dfrn($url) { $lines = explode("\n",$headers); if(count($lines)) { - foreach($lines as $line) { + foreach($lines as $line) { // don't try and run feeds through the html5 parser if(stristr($line,'content-type:') && ((stristr($line,'application/atom+xml')) || (stristr($line,'application/rss+xml')))) return ret; @@ -351,7 +351,8 @@ function probe_url($url, $mode = PROBE_NORMAL) { $has_lrdd = false; $email_conversant = false; - $twitter = ((strpos($url,'twitter.com') !== false) ? true : false); + // Twitter is deactivated since twitter closed its old API + //$twitter = ((strpos($url,'twitter.com') !== false) ? true : false); $lastfm = ((strpos($url,'last.fm/user') !== false) ? true : false); $at_addr = ((strpos($url,'@') !== false) ? true : false); @@ -561,7 +562,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { $vcard['nick'] = $addr_parts[0]; } - if($twitter) { + /* if($twitter) { logger('twitter: setup'); $tid = basename($url); $tapi = 'https://api.twitter.com/1/statuses/user_timeline.rss'; @@ -574,7 +575,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { $vcard['photo'] = 'https://api.twitter.com/1/users/profile_image?screen_name=' . $tid . '&size=bigger'; $vcard['nick'] = $tid; $vcard['fn'] = $tid; - } + } */ if($lastfm) { $profile = $url; @@ -609,7 +610,7 @@ function probe_url($url, $mode = PROBE_NORMAL) { logger('probe_url: scrape_feed ' . (($poll)? $poll : $url) . ' returns: ' . print_r($feedret,true), LOGGER_DATA); if(count($feedret) && ($feedret['feed_atom'] || $feedret['feed_rss'])) { $poll = ((x($feedret,'feed_atom')) ? unamp($feedret['feed_atom']) : unamp($feedret['feed_rss'])); - if(! x($vcard)) + if(! x($vcard)) $vcard = array(); } From 7614ae40ef57c0a2b92e7186791e6f6d380f9d9c Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:44:42 +0200 Subject: [PATCH 03/12] When converting to plain text, links behind @-addresses are now removed --- include/api.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/api.php b/include/api.php index ea1ba27e40..74586850f8 100644 --- a/include/api.php +++ b/include/api.php @@ -2420,10 +2420,11 @@ function api_clean_plain_items($Text) { $Text = bb_CleanPictureLinks($Text); - if ($include_entities == "true") { - $URLSearchString = "^\[\]"; + $URLSearchString = "^\[\]"; - $Text = preg_replace("/#\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",'#$2',$Text); + $Text = preg_replace("/([!#@])\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",'$1$3',$Text); + + if ($include_entities == "true") { $Text = preg_replace("/\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",'[url=$1]$1[/url]',$Text); } From 834e0ff778bf25ce3137b29aa3adead062bf7a65 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:45:40 +0200 Subject: [PATCH 04/12] Diaspora: The HTML paragraph element was treated wrong when importing items from Diaspora --- include/bb2diaspora.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/bb2diaspora.php b/include/bb2diaspora.php index 59d0e88226..5b990bf0df 100644 --- a/include/bb2diaspora.php +++ b/include/bb2diaspora.php @@ -27,7 +27,9 @@ function diaspora2bb($s) { $s = str_replace("\r","",$s); //
is invalid. Replace it with the valid expression - $s = str_replace("
","
",$s); + logger("diaspora2bb: 1: ".$s); + $s = str_replace(array("
", "

", "

"),array("
", "
", "
"),$s); + logger("diaspora2bb: 2: ".$s); $s = preg_replace('/\@\{(.+?)\; (.+?)\@(.+?)\}/','@[url=https://$3/u/$2]$1[/url]',$s); From 8bb81ff83d2dd9e993e8e3251f28c337678ec8d4 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:48:02 +0200 Subject: [PATCH 05/12] The system's load is now checked, when the cronhooks are started. --- include/cronhooks.php | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/include/cronhooks.php b/include/cronhooks.php index 15d49fe547..cf1d13765f 100644 --- a/include/cronhooks.php +++ b/include/cronhooks.php @@ -9,7 +9,7 @@ function cronhooks_run(&$argv, &$argc){ if(is_null($a)) { $a = new App; } - + if(is_null($db)) { @include(".htconfig.php"); require_once("include/dba.php"); @@ -24,6 +24,17 @@ function cronhooks_run(&$argv, &$argc){ load_config('config'); load_config('system'); + $maxsysload = intval(get_config('system','maxloadavg')); + if($maxsysload < 1) + $maxsysload = 50; + if(function_exists('sys_getloadavg')) { + $load = sys_getloadavg(); + if(intval($load[0]) > $maxsysload) { + logger('system: load ' . $load . ' too high. Poller deferred to next scheduled run.'); + return; + } + } + $lockpath = get_config('system','lockpath'); if ($lockpath != '') { $pidfile = new pidfile($lockpath, 'cron.lck'); @@ -38,7 +49,7 @@ function cronhooks_run(&$argv, &$argc){ load_hooks(); logger('cronhooks: start'); - + $d = datetime_convert(); From 5d4719f0096925fc14d8a0504d0045a1b35ef08a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:52:53 +0200 Subject: [PATCH 06/12] The function to fetch data about a site (via the "parse url" function) is moved from diaspora to another place, since it is multi purpose --- include/diaspora.php | 59 ++++++-------------------------------- include/items.php | 67 +++++++++++++++++++++++++++++++++----------- 2 files changed, 60 insertions(+), 66 deletions(-) diff --git a/include/diaspora.php b/include/diaspora.php index 7476a5c26d..feb57a0404 100755 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -7,47 +7,6 @@ require_once('include/contact_selectors.php'); require_once('include/queue_fn.php'); require_once('include/lock.php'); -function diaspora_add_page_info($url) { - require_once("mod/parse_url.php"); - $data = parseurl_getsiteinfo($url, true); - - logger('diaspora_add_page_info: fetch page info for '.$url.' '.print_r($data, true), LOGGER_DATA); - - if (($data["type"] != "link") OR ($data["title"] == $url)) - return(""); - - if (is_string($data["title"])) - $text .= "[bookmark=".$url."]".trim($data["title"])."[/bookmark]"; - - if (sizeof($data["images"]) > 0) { - $imagedata = $data["images"][0]; - $text .= '[img]'.$imagedata["src"].'[/img]'; - } - - if (is_string($data["text"])) - $text .= "[quote]".$data["text"]."[/quote]"; - - return("\n[class=type-".$data["type"]."]".$text."[/class]"); -} - -function diaspora_add_page_info_to_body($body) { - - logger('diaspora_add_page_info_to_body: fetch page info for body '.$body, LOGGER_DATA); - - $URLSearchString = "^\[\]"; - - // Adding these spaces is a quick hack due to my problems with regular expressions :) - preg_match("/[^@#]\[url\]([$URLSearchString]*)\[\/url\]/ism", " ".$body, $matches); - - if (!$matches) - preg_match("/[^@#]\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", " ".$body, $matches); - - if ($matches) - $body .= diaspora_add_page_info($matches[1]); - - return $body; -} - function diaspora_dispatch_public($msg) { $enabled = intval(get_config('system','diaspora_enabled')); @@ -526,7 +485,7 @@ function diaspora_decode($importer,$xml) { $base = $dom->env; elseif($dom->data) $base = $dom; - + if(! $base) { logger('mod-diaspora: unable to locate salmon data in xml '); http_status_exit(400); @@ -654,7 +613,7 @@ function diaspora_request($importer,$xml) { $arr['author-avatar'] = $arr['owner-avatar'] = $self[0]['thumb']; $arr['verb'] = ACTIVITY_FRIEND; $arr['object-type'] = ACTIVITY_OBJ_PERSON; - + $A = '[url=' . $self[0]['url'] . ']' . $self[0]['name'] . '[/url]'; $B = '[url=' . $contact['url'] . ']' . $contact['name'] . '[/url]'; $BPhoto = '[url=' . $contact['url'] . ']' . '[img]' . $contact['thumb'] . '[/img][/url]'; @@ -713,7 +672,7 @@ function diaspora_request($importer,$xml) { 1, 2 ); - + // find the contact record we just created $contact_record = diaspora_get_contact_by_handle($importer['uid'],$sender_handle); @@ -734,7 +693,7 @@ function diaspora_request($importer,$xml) { if($importer['page-flags'] == PAGE_NORMAL) { $hash = random_string() . (string) time(); // Generate a confirm_key - + $ret = q("INSERT INTO `intro` ( `uid`, `contact-id`, `blocked`, `knowyou`, `note`, `hash`, `datetime` ) VALUES ( %d, %d, %d, %d, '%s', '%s', '%s' )", intval($importer['uid']), @@ -753,7 +712,7 @@ function diaspora_request($importer,$xml) { require_once('include/Photo.php'); $photos = import_profile_photo($contact_record['photo'],$importer['uid'],$contact_record['id']); - + // technically they are sharing with us (CONTACT_IS_SHARING), // but if our page-type is PAGE_COMMUNITY or PAGE_SOAPBOX // we are going to change the relationship and make them a follower. @@ -868,7 +827,7 @@ function diaspora_post($importer,$xml,$msg) { $body = diaspora2bb($xml->raw_message); // Add OEmbed and other information to the body - $body = diaspora_add_page_info_to_body($body); + $body = add_page_info_to_body($body, false, true); $datarray = array(); @@ -1025,7 +984,7 @@ function diaspora_reshare($importer,$xml,$msg) { $body = scale_external_images($body); // Add OEmbed and other information to the body - $body = diaspora_add_page_info_to_body($body); + $body = add_page_info_to_body($body, false, true); } else { // Maybe it is a reshare of a photo that will be delivered at a later time (testing) @@ -1445,12 +1404,12 @@ function diaspora_comment($importer,$xml,$msg) { foreach($myconv as $conv) { // now if we find a match, it means we're in this conversation - + if(! link_compare($conv['author-link'],$importer_url)) continue; require_once('include/enotify.php'); - + $conv_parent = $conv['parent']; notification(array( diff --git a/include/items.php b/include/items.php index ac8c34794a..1516c12015 100755 --- a/include/items.php +++ b/include/items.php @@ -856,9 +856,10 @@ function get_atom_elements($feed, $item, $contact = array()) { } if (isset($contact["network"]) AND ($contact["network"] == NETWORK_FEED) AND $contact['fetch_further_information']) { - $res["body"] = $res["title"]."\n\n[class=type-link]".fetch_siteinfo($res['plink'])."[/class]"; + $res["body"] = $res["title"].add_page_info($res['plink']); $res["title"] = ""; - } + } elseif (isset($contact["network"]) AND ($contact["network"] == NETWORK_OSTATUS)) + $res["body"] = add_page_info_to_body($res["body"]); $arr = array('feed' => $feed, 'item' => $item, 'result' => $res); @@ -874,29 +875,63 @@ function get_atom_elements($feed, $item, $contact = array()) { return $res; } -function fetch_siteinfo($url) { - require_once("mod/parse_url.php"); +function add_page_info($url, $no_photos = false) { + require_once("mod/parse_url.php"); + $data = parseurl_getsiteinfo($url, true); - // Fetch site infos - but only from the meta data - $data = parseurl_getsiteinfo($url, true); + logger('add_page_info: fetch page info for '.$url.' '.print_r($data, true), LOGGER_DEBUG); - $text = ""; + // It maybe is a rich content, but if it does have everything that a link has, + // then treat it that way + if (($data["type"] == "rich") AND is_string($data["title"]) AND + is_string($data["text"]) AND (sizeof($data["images"]) > 0)) + $data["type"] = "link"; - if (!is_string($data["text"]) AND (sizeof($data["images"]) == 0) AND ($data["title"] == $url)) + if ((($data["type"] != "link") AND ($data["type"] != "video") AND ($data["type"] != "photo")) OR ($data["title"] == $url)) + return(""); + + if ($no_photos AND ($data["type"] == "photo")) return(""); - if (is_string($data["title"])) - $text .= "[bookmark=".$url."]".trim($data["title"])."[/bookmark]\n"; + if (($data["type"] != "photo") AND is_string($data["title"])) + $text .= "[bookmark=".$url."]".trim($data["title"])."[/bookmark]"; - if (sizeof($data["images"]) > 0) { - $imagedata = $data["images"][0]; - $text .= '[img='.$imagedata["width"].'x'.$imagedata["height"].']'.$imagedata["src"].'[/img]' . "\n"; + if (($data["type"] != "video") AND (sizeof($data["images"]) > 0)) { + $imagedata = $data["images"][0]; + $text .= '[img]'.$imagedata["src"].'[/img]'; + } + + if (($data["type"] != "photo") AND is_string($data["text"])) + $text .= "[quote]".$data["text"]."[/quote]"; + + return("\n[class=type-".$data["type"]."]".$text."[/class]"); +} + +function add_page_info_to_body($body, $texturl = false, $no_photos = false) { + + logger('add_page_info_to_body: fetch page info for body '.$body, LOGGER_DEBUG); + + $URLSearchString = "^\[\]"; + + // Adding these spaces is a quick hack due to my problems with regular expressions :) + preg_match("/[^@#]\[url\]([$URLSearchString]*)\[\/url\]/ism", " ".$body, $matches); + + if (!$matches) + preg_match("/[^@#]\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", " ".$body, $matches); + + // Convert urls without bbcode elements + if (!$matches AND $texturl) { + preg_match("/([^\]\='".'"'."]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", " ".$body, $matches); + + // Yeah, a hack. I really hate regular expressions :) + if ($matches) + $matches[1] = $matches[2]; } - if (is_string($data["text"])) - $text .= "[quote]".$data["text"]."[/quote]"; + if ($matches) + $body .= add_page_info($matches[1], $no_photos); - return($text); + return $body; } function encode_rel_links($links) { From 7b471e097ab10c22c69ab486f4d3208629248f7b Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:54:43 +0200 Subject: [PATCH 07/12] When converting from html to bbcode, now mail links are converted into the [mail] bbcode --- include/html2bbcode.php | 1 + 1 file changed, 1 insertion(+) diff --git a/include/html2bbcode.php b/include/html2bbcode.php index b38da6b0a4..d9255a7ffc 100644 --- a/include/html2bbcode.php +++ b/include/html2bbcode.php @@ -212,6 +212,7 @@ function html2bbcode($message) node2bbcode($doc, 'h5', array(), "\n\n[size=small][b]", "[/b][/size]\n"); node2bbcode($doc, 'h6', array(), "\n\n[size=x-small][b]", "[/b][/size]\n"); + node2bbcode($doc, 'a', array('href'=>'/mailto:(.+)/'), '[mail=$1]', '[/mail]'); node2bbcode($doc, 'a', array('href'=>'/(.+)/'), '[url=$1]', '[/url]'); node2bbcode($doc, 'img', array('src'=>'/(.+)/', 'width'=>'/(\d+)/', 'height'=>'/(\d+)/'), '[img=$2x$3]$1', '[/img]'); From a64948b7f76fc7b23f05d12fd23809ed6d45ff79 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:55:30 +0200 Subject: [PATCH 08/12] New function "original_url" (moved from an addon) that follows all redirects to the original url. --- include/network.php | 70 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/include/network.php b/include/network.php index 4bc50974b0..b85775ba2b 100644 --- a/include/network.php +++ b/include/network.php @@ -189,7 +189,7 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) $base = substr($base,strlen($chunk)); } - if($http_code == 301 || $http_code == 302 || $http_code == 303) { + if($http_code == 301 || $http_code == 302 || $http_code == 303 || $http_code == 307) { $matches = array(); preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches); $newurl = trim(array_pop($matches)); @@ -1105,3 +1105,71 @@ function xml2array($contents, $namespaces = true, $get_attributes=1, $priority = return($xml_array); } + +function original_url($url, $depth=1, $fetchbody = false) { + if ($depth > 10) + return($url); + + $url = trim($url, "'"); + + $siteinfo = array(); + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HEADER, 1); + + if ($fetchbody) + curl_setopt($ch, CURLOPT_NOBODY, 0); + else + curl_setopt($ch, CURLOPT_NOBODY, 1); + + curl_setopt($ch, CURLOPT_TIMEOUT, 10); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch,CURLOPT_USERAGENT,'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0'); + + $header = curl_exec($ch); + $curl_info = @curl_getinfo($ch); + $http_code = $curl_info['http_code']; + curl_close($ch); + + if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302")) + AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) { + if ($curl_info['redirect_url'] != "") + return(original_url($curl_info['redirect_url'], ++$depth, $fetchbody)); + else + return(original_url($curl_info['location'], ++$depth, $fetchbody)); + } + + $pos = strpos($header, "\r\n\r\n"); + + if ($pos) + $body = trim(substr($header, $pos)); + else + $body = $header; + + if (trim($body) == "") + return(original_url($url, ++$depth, true)); + + $doc = new DOMDocument(); + @$doc->loadHTML($body); + + $xpath = new DomXPath($doc); + + $list = $xpath->query("//meta[@content]"); + foreach ($list as $node) { + $attr = array(); + if ($node->attributes->length) + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; + + if (@$attr["http-equiv"] == 'refresh') { + $path = $attr["content"]; + $pathinfo = explode(";", $path); + $content = ""; + foreach ($pathinfo AS $value) + if (substr(strtolower($value), 0, 4) == "url=") + return(original_url(substr($value, 4), ++$depth)); + } + } + + return($url); +} From 96a41a16fd90492a9fca62fc675c9e09e0979a88 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:56:33 +0200 Subject: [PATCH 09/12] oembed: Now it is possible to use the service from embedly with an API key. --- include/oembed.php | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/include/oembed.php b/include/oembed.php index ee042f8ce9..17fc873ccb 100755 --- a/include/oembed.php +++ b/include/oembed.php @@ -12,6 +12,9 @@ function oembed_replacecb($matches){ function oembed_fetch_url($embedurl){ + $embedurl = trim($embedurl, "'"); + $embedurl = trim($embedurl, '"'); + $a = get_app(); $txt = Cache::get($a->videowidth . $embedurl); @@ -48,9 +51,18 @@ function oembed_fetch_url($embedurl){ } if ($txt==false || $txt==""){ - // try oohembed service - $ourl = "http://oohembed.com/oohembed/?url=".urlencode($embedurl).'&maxwidth=' . $a->videowidth; - $txt = fetch_url($ourl); + $embedly = get_config("system", "embedly"); + if ($embedly == "") { + // try oohembed service + $ourl = "http://oohembed.com/oohembed/?url=".urlencode($embedurl).'&maxwidth=' . $a->videowidth; + $txt = fetch_url($ourl); + } else { + // try embedly service + $ourl = "https://api.embed.ly/1/oembed?key=".$embedly."&url=".urlencode($embedurl); + $txt = fetch_url($ourl); + } + + logger("oembed_fetch_url: ".$txt, LOGGER_DEBUG); } $txt=trim($txt); From 12866209e889cea0e2c3e1659d0845388bc4090b Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 10:58:31 +0200 Subject: [PATCH 10/12] parse url: Emtpy values in meta headers are now ignored. Oembed data has priority. Videos are embedded with some further information. --- mod/parse_url.php | 113 +++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 51 deletions(-) diff --git a/mod/parse_url.php b/mod/parse_url.php index 5caf830325..a0608f6363 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -53,6 +53,9 @@ function completeurl($url, $scheme) { function parseurl_getsiteinfo($url, $no_guessing = false) { $siteinfo = array(); + $url = trim($url, "'"); + $url = trim($url, '"'); + $siteinfo["url"] = $url; $siteinfo["type"] = "link"; $ch = curl_init(); @@ -69,7 +72,7 @@ function parseurl_getsiteinfo($url, $no_guessing = false) { $http_code = $curl_info['http_code']; curl_close($ch); - if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302")) + if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302") OR ($curl_info['http_code'] == "303") OR ($curl_info['http_code'] == "307")) AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) { if ($curl_info['redirect_url'] != "") $siteinfo = parseurl_getsiteinfo($curl_info['redirect_url']); @@ -85,19 +88,12 @@ function parseurl_getsiteinfo($url, $no_guessing = false) { if ($oembed_data->type != "error") $siteinfo["type"] = $oembed_data->type; - if ($oembed_data->type == "link") { - if (isset($oembed_data->title)) - $siteinfo["title"] = $oembed_data->title; - if (isset($oembed_data->description)) - $siteinfo["text"] = trim($oembed_data->description); - if (isset($oembed_data->thumbnail_url)) - $siteinfo["image"] = $oembed_data->thumbnail_url; - } - // Fetch the first mentioned charset. Can be in body or header + $charset = ""; if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches)) $charset = trim(array_pop($matches)); - else + + if ($charset == "") $charset = "utf-8"; $pos = strpos($header, "\r\n\r\n"); @@ -164,33 +160,34 @@ function parseurl_getsiteinfo($url, $no_guessing = false) { $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8")); - switch (strtolower($attr["name"])) { - case "fulltitle": - $siteinfo["title"] = $attr["content"]; - break; - case "description": - $siteinfo["text"] = $attr["content"]; - break; - case "twitter:image": - $siteinfo["image"] = $attr["content"]; - break; - case "twitter:card": - if ($siteinfo["type"] == "") - $siteinfo["type"] = $attr["content"]; - break; - case "twitter:description": - $siteinfo["text"] = $attr["content"]; - break; - case "twitter:title": - $siteinfo["title"] = $attr["content"]; - break; - case "dc.title": - $siteinfo["title"] = $attr["content"]; - break; - case "dc.description": - $siteinfo["text"] = $attr["content"]; - break; - } + if ($attr["content"] != "") + switch (strtolower($attr["name"])) { + case "fulltitle": + $siteinfo["title"] = $attr["content"]; + break; + case "description": + $siteinfo["text"] = $attr["content"]; + break; + case "twitter:image": + $siteinfo["image"] = $attr["content"]; + break; + case "twitter:card": + if ($siteinfo["type"] == "") + $siteinfo["type"] = $attr["content"]; + break; + case "twitter:description": + $siteinfo["text"] = $attr["content"]; + break; + case "twitter:title": + $siteinfo["title"] = $attr["content"]; + break; + case "dc.title": + $siteinfo["title"] = $attr["content"]; + break; + case "dc.description": + $siteinfo["text"] = $attr["content"]; + break; + } if ($siteinfo["type"] == "summary") $siteinfo["type"] = "link"; } @@ -205,17 +202,27 @@ function parseurl_getsiteinfo($url, $no_guessing = false) { $attr["content"] = trim(html_entity_decode($attr["content"], ENT_QUOTES, "UTF-8")); - switch (strtolower($attr["property"])) { - case "og:image": - $siteinfo["image"] = $attr["content"]; - break; - case "og:title": - $siteinfo["title"] = $attr["content"]; - break; - case "og:description": - $siteinfo["text"] = $attr["content"]; - break; - } + if ($attr["content"] != "") + switch (strtolower($attr["property"])) { + case "og:image": + $siteinfo["image"] = $attr["content"]; + break; + case "og:title": + $siteinfo["title"] = $attr["content"]; + break; + case "og:description": + $siteinfo["text"] = $attr["content"]; + break; + } + } + + if ($oembed_data->type == "link") { + if (isset($oembed_data->title) AND (trim($oembed_data->title) != "")) + $siteinfo["title"] = $oembed_data->title; + if (isset($oembed_data->description) AND (trim($oembed_data->description) != "")) + $siteinfo["text"] = trim($oembed_data->description); + if (isset($oembed_data->thumbnail_url) AND (trim($oembed_data->thumbnail_url) != "")) + $siteinfo["image"] = $oembed_data->thumbnail_url; } if ((@$siteinfo["image"] == "") AND !$no_guessing) { @@ -290,6 +297,8 @@ function parseurl_getsiteinfo($url, $no_guessing = false) { } } + logger("parseurl_getsiteinfo: Siteinfo for ".$url." ".print_r($siteinfo, true), LOGGER_DEBUG); + return($siteinfo); } @@ -367,6 +376,8 @@ function parse_url_content(&$a) { $siteinfo = parseurl_getsiteinfo($url); + $url= $siteinfo["url"]; + $sitedata = ""; if($siteinfo["title"] == "") { @@ -379,7 +390,7 @@ function parse_url_content(&$a) { $image = ""; - if(sizeof($siteinfo["images"]) > 0){ + if (($siteinfo["type"] != "video") AND (sizeof($siteinfo["images"]) > 0)){ /* Execute below code only if image is present in siteinfo */ $total_images = 0; @@ -421,7 +432,7 @@ function parse_url_content(&$a) { $sitedata .= trim($result); if (($siteinfo["type"] == "video") AND ($url != "")) - echo "[video]".$url."[/video]"; + echo "[class=type-video]".$sitedata."[/class]"; elseif (($siteinfo["type"] != "photo")) echo "[class=type-link]".$sitedata."[/class]"; else From 5f3d5847162774ff9d93847ae5ee3f6102a49117 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 11:02:36 +0200 Subject: [PATCH 11/12] Code beautification and some more logging data - no changes in the functionality. --- mod/like.php | 13 ++++----- mod/pubsub.php | 8 +++--- mod/settings.php | 75 ++++++++++++++++++++++++------------------------ 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/mod/like.php b/mod/like.php index ed6b1e7d08..a2f51b074e 100755 --- a/mod/like.php +++ b/mod/like.php @@ -109,14 +109,13 @@ function like_content(&$a) { $return_path = ((x($_REQUEST,'return')) ? $_REQUEST['return'] : ''); - $r = q("SELECT * FROM `item` WHERE `verb` = '%s' AND `deleted` = 0 - AND `contact-id` = %d AND ( `parent` = '%s' OR `parent-uri` = '%s' OR `thr-parent` = '%s') LIMIT 1", - dbesc($activity), - intval($contact['id']), - dbesc($item_id), - dbesc($item_id), - dbesc($item['uri']) + $r = q("SELECT `id`, `guid` FROM `item` WHERE `verb` = '%s' AND `deleted` = 0 + AND `contact-id` = %d AND `uid` = %d + AND (`parent` = '%s' OR `parent-uri` = '%s' OR `thr-parent` = '%s') LIMIT 1", + dbesc($activity), intval($contact['id']), intval($owner_uid), + dbesc($item_id), dbesc($item_id), dbesc($item['uri']) ); + if(count($r)) { $like_item = $r[0]; diff --git a/mod/pubsub.php b/mod/pubsub.php index 3d8bfca48e..82458f46ac 100644 --- a/mod/pubsub.php +++ b/mod/pubsub.php @@ -1,7 +1,7 @@ argc > 1) && ($a->argv[1] === 'oauth')) { - + if(($a->argc > 2) && ($a->argv[2] === 'add')) { $tpl = get_markup_template("settings_oauth_edit.tpl"); $o .= replace_macros($tpl, array( @@ -608,18 +608,18 @@ function settings_content(&$a) { )); return $o; } - + if(($a->argc > 3) && ($a->argv[2] === 'edit')) { $r = q("SELECT * FROM clients WHERE client_id='%s' AND uid=%d", dbesc($a->argv[3]), local_user()); - + if (!count($r)){ notice(t("You can't edit this application.")); return; } $app = $r[0]; - + $tpl = get_markup_template("settings_oauth_edit.tpl"); $o .= replace_macros($tpl, array( '$form_security_token' => get_form_security_token("settings_oauth"), @@ -634,18 +634,18 @@ function settings_content(&$a) { )); return $o; } - + if(($a->argc > 3) && ($a->argv[2] === 'delete')) { check_form_security_token_redirectOnErr('/settings/oauth', 'settings_oauth', 't'); - + $r = q("DELETE FROM clients WHERE client_id='%s' AND uid=%d", dbesc($a->argv[3]), local_user()); goaway($a->get_baseurl(true)."/settings/oauth/"); - return; + return; } - - + + $r = q("SELECT clients.*, tokens.id as oauth_token, (clients.uid=%d) AS my FROM clients LEFT JOIN tokens ON clients.client_id=tokens.client_id @@ -754,11 +754,11 @@ function settings_content(&$a) { else { $mail_disabled_message = (($mail_disabled) ? t('Email access is disabled on this site.') : ''); } - + $o .= replace_macros($tpl, array( '$form_security_token' => get_form_security_token("settings_connectors"), - + '$title' => t('Connector Settings'), '$diasp_enabled' => $diasp_enabled, @@ -801,14 +801,14 @@ function settings_content(&$a) { $allowed_themes_raw = explode(',',$allowed_themes_str); $allowed_themes = array(); if(count($allowed_themes_raw)) - foreach($allowed_themes_raw as $x) + foreach($allowed_themes_raw as $x) if(strlen(trim($x)) && is_dir("view/theme/$x")) $allowed_themes[] = trim($x); - + $themes = array(); $mobile_themes = array("---" => t('No special theme for mobile devices')); - $files = glob('view/theme/*'); + $files = glob('view/theme/*'); /* */ if($allowed_themes) { foreach($allowed_themes as $th) { $f = $th; @@ -905,13 +905,13 @@ function settings_content(&$a) { $expire_items = get_pconfig(local_user(), 'expire','items'); $expire_items = (($expire_items===false)? '1' : $expire_items); // default if not set: 1 - + $expire_notes = get_pconfig(local_user(), 'expire','notes'); $expire_notes = (($expire_notes===false)? '1' : $expire_notes); // default if not set: 1 $expire_starred = get_pconfig(local_user(), 'expire','starred'); $expire_starred = (($expire_starred===false)? '1' : $expire_starred); // default if not set: 1 - + $expire_photos = get_pconfig(local_user(), 'expire','photos'); $expire_photos = (($expire_photos===false)? '0' : $expire_photos); // default if not set: 0 @@ -931,7 +931,8 @@ function settings_content(&$a) { $post_profilechange = get_pconfig(local_user(), 'system','post_profilechange'); $post_profilechange = (($post_profilechange===false)? '0': $post_profilechange); // default if not set: 0 - + // nowarn_insecure + if(! strlen($a->user['timezone'])) $timezone = date_default_timezone_get(); @@ -939,24 +940,24 @@ function settings_content(&$a) { $pageset_tpl = get_markup_template('pagetypes.tpl'); $pagetype = replace_macros($pageset_tpl, array( - '$page_normal' => array('page-flags', t('Normal Account Page'), PAGE_NORMAL, - t('This account is a normal personal profile'), + '$page_normal' => array('page-flags', t('Normal Account Page'), PAGE_NORMAL, + t('This account is a normal personal profile'), ($a->user['page-flags'] == PAGE_NORMAL)), - - '$page_soapbox' => array('page-flags', t('Soapbox Page'), PAGE_SOAPBOX, + + '$page_soapbox' => array('page-flags', t('Soapbox Page'), PAGE_SOAPBOX, t('Automatically approve all connection/friend requests as read-only fans'), ($a->user['page-flags'] == PAGE_SOAPBOX)), - - '$page_community' => array('page-flags', t('Community Forum/Celebrity Account'), PAGE_COMMUNITY, + + '$page_community' => array('page-flags', t('Community Forum/Celebrity Account'), PAGE_COMMUNITY, t('Automatically approve all connection/friend requests as read-write fans'), ($a->user['page-flags'] == PAGE_COMMUNITY)), - - '$page_freelove' => array('page-flags', t('Automatic Friend Page'), PAGE_FREELOVE, + + '$page_freelove' => array('page-flags', t('Automatic Friend Page'), PAGE_FREELOVE, t('Automatically approve all connection/friend requests as friends'), ($a->user['page-flags'] == PAGE_FREELOVE)), - '$page_prvgroup' => array('page-flags', t('Private Forum [Experimental]'), PAGE_PRVGROUP, - t('Private forum - approved members only'), + '$page_prvgroup' => array('page-flags', t('Private Forum [Experimental]'), PAGE_PRVGROUP, + t('Private forum - approved members only'), ($a->user['page-flags'] == PAGE_PRVGROUP)), @@ -1004,7 +1005,7 @@ function settings_content(&$a) { '$field' => array('blockwall', t('Allow friends to post to your profile page?'), (intval($a->user['blockwall']) ? '0' : '1'), '', array(t('No'),t('Yes'))), )); - + $blocktags = replace_macros($opt_tpl,array( '$field' => array('blocktags', t('Allow friends to tag your posts?'), (intval($a->user['blocktags']) ? '0' : '1'), '', array(t('No'),t('Yes'))), @@ -1134,10 +1135,10 @@ function settings_content(&$a) { '$profile_in_net_dir' => $profile_in_net_dir, '$hide_friends' => $hide_friends, '$hide_wall' => $hide_wall, - '$unkmail' => $unkmail, + '$unkmail' => $unkmail, '$cntunkmail' => array('cntunkmail', t('Maximum private messages per day from unknown people:'), $cntunkmail ,t("\x28to prevent spam abuse\x29")), - - + + '$h_not' => t('Notification Settings'), '$activity_options' => t('By default post a status message when:'), '$post_newfriend' => array('post_newfriend', t('accepting a friend request'), $post_newfriend, ''), @@ -1152,16 +1153,16 @@ function settings_content(&$a) { '$notify6' => array('notify6', t('You receive a friend suggestion'), ($notify & NOTIFY_SUGGEST), NOTIFY_SUGGEST, ''), '$notify7' => array('notify7', t('You are tagged in a post'), ($notify & NOTIFY_TAGSELF), NOTIFY_TAGSELF, ''), '$notify8' => array('notify8', t('You are poked/prodded/etc. in a post'), ($notify & NOTIFY_POKE), NOTIFY_POKE, ''), - - + + '$h_advn' => t('Advanced Account/Page Type Settings'), '$h_descadvn' => t('Change the behaviour of this account for special situations'), '$pagetype' => $pagetype, - + '$relocate' => t('Relocate'), '$relocate_text' => t("If you have moved this profile from another server, and some of your contacts don't receive your updates, try pushing this button."), '$relocate_button' => t("Resend relocate message to contacts"), - + )); call_hooks('settings_form',$o); From 73f049612790f9195324315582cdf3980a67bce3 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 4 Apr 2014 11:03:34 +0200 Subject: [PATCH 12/12] Vier: Added treatment of videos like links. (See the changes in the "parse url" section) --- view/theme/vier/style.css | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/view/theme/vier/style.css b/view/theme/vier/style.css index 41855e4340..2ff3bbd15e 100644 --- a/view/theme/vier/style.css +++ b/view/theme/vier/style.css @@ -1210,19 +1210,20 @@ section.minimal { font-size: 12px; } -.type-link { +.type-link, .type-video { border-top: 1px solid #D2D2D2; border-bottom: 1px solid #D2D2D2; display: block; padding-top: 5px; padding-bottom: 5px; + margin-top: 10px; /* padding-left: 170px; min-height: 90px; position: relative; */ } -.type-link:after { +.type-link:after, .type-video:after { content: "."; display: block; height: 0; @@ -1230,13 +1231,13 @@ position: relative; visibility: hidden; } .wall-item-container .wall-item-content .type-link img, -.type-link img { +.type-link img, .type-video img { max-width: 160px; max-height: 160px; float: left; margin-right: 10px; } -.type-link blockquote { +.type-link blockquote, .type-video blockquote { /* margin-left: 160px; */ margin-left: 0px; max-height: 160px; @@ -1244,17 +1245,17 @@ position: relative; border-left: 0px; padding-left: 0px; } -.type-link .oembed { +.type-link .oembed, .type-video .oembed { font-size: inherit; } -.type-link img { +.type-link img, .type-video img { /* position: absolute; left: 0; top: 0; */ } -.type-link span.oembed + br, -.type-link a + br, -.type-link img + br{ +.type-link span.oembed + br, .type-video span.oembed + br, +.type-link a + br, .type-video a + br, +.type-link img + br, .type-video img + br{ display: none; }