dir/include/Scrape.php
Hypolite Petovan a8dc2e65f4 Merge branch 'master' into feature/redesign-prototype
# Conflicts:
#	.gitignore
#	.htconfig.php
#	.htconfig.php-dist
#	boot.php
#	include/dba.php
#	mod/health.php
#	util/htconfig.vagrant.php
2017-04-20 22:33:41 -04:00

186 lines
5.3 KiB
PHP

<?php
require_once('library/HTML5/Parser.php');
if(! function_exists('attribute_contains')) {
function attribute_contains($attr,$s) {
$a = explode(' ', $attr);
if(count($a) && in_array($s,$a))
return true;
return false;
}}
if(! function_exists('noscrape_dfrn')) {
function noscrape_dfrn($url) {
$submit_noscrape_start = microtime(true);
$data = fetch_url($url);
$submit_noscrape_request_end = microtime(true);
if(empty($data)) return false;
$parms = json_decode($data, true);
if(!$parms || !count($parms)) return false;
$parms['tags'] = implode(' ', (array)$parms['tags']);
$submit_noscrape_end = microtime(true);
$parms['_timings'] = array(
'fetch' => round(($submit_noscrape_request_end - $submit_noscrape_start) * 1000),
'scrape' => round(($submit_noscrape_end - $submit_noscrape_request_end) * 1000)
);
return $parms;
}}
if(! function_exists('scrape_dfrn')) {
function scrape_dfrn($url, $max_nodes=3500) {
$minNodes = 100; //Lets do at least 100 nodes per type.
$timeout = 10; //Timeout will affect batch processing.
//Try and cheat our way into faster profiles.
if(strpos($url, 'tab=profile') === false){
$url .= (strpos($url, '?') > 0 ? '&' : '?').'tab=profile';
}
$scrape_start = microtime(true);
$ret = array();
$s = fetch_url($url, $timeout);
$scrape_fetch_end = microtime(true);
if(! $s)
return $ret;
$dom = HTML5_Parser::parse($s);
if(! $dom)
return $ret;
$items = $dom->getElementsByTagName('meta');
// get DFRN link elements
$nodes_left = max(intval($max_nodes), $minNodes);
$targets = array('hide', 'comm', 'tags');
$targets_left = count($targets);
foreach($items as $item) {
$x = $item->getAttribute('name');
if($x == 'dfrn-global-visibility') {
$z = strtolower(trim($item->getAttribute('content')));
if($z != 'true')
$ret['hide'] = 1;
if($z === 'false')
$ret['explicit-hide'] = 1;
$targets_left = pop_scrape_target($targets, 'hide');
}
if($x == 'friendika.community' || $x == 'friendica.community') {
$z = strtolower(trim($item->getAttribute('content')));
if($z == 'true')
$ret['comm'] = 1;
$targets_left = pop_scrape_target($targets, 'comm');
}
if($x == 'keywords') {
$z = str_replace(',',' ',strtolower(trim($item->getAttribute('content'))));
if(strlen($z))
$ret['tags'] = $z;
$targets_left = pop_scrape_target($targets, 'tags');
}
$nodes_left--;
if($nodes_left <= 0 || $targets_left <= 0) break;
}
$items = $dom->getElementsByTagName('link');
// get DFRN link elements
$nodes_left = max(intval($max_nodes), $minNodes);
foreach($items as $item) {
$x = $item->getAttribute('rel');
if(substr($x,0,5) == "dfrn-")
$ret[$x] = $item->getAttribute('href');
$nodes_left--;
if($nodes_left <= 0) break;
}
// Pull out hCard profile elements
$nodes_left = max(intval($max_nodes), $minNodes);
$items = $dom->getElementsByTagName('*');
$targets = array('fn', 'pdesc', 'photo', 'key', 'locality', 'region', 'postal-code', 'country-name');
$targets_left = count($targets);
foreach($items as $item) {
if(attribute_contains($item->getAttribute('class'), 'vcard')) {
$level2 = $item->getElementsByTagName('*');
foreach($level2 as $x) {
if(attribute_contains($x->getAttribute('class'),'fn')){
$ret['fn'] = $x->textContent;
$targets_left = pop_scrape_target($targets, 'fn');
}
if(attribute_contains($x->getAttribute('class'),'title')){
$ret['pdesc'] = $x->textContent;
$targets_left = pop_scrape_target($targets, 'pdesc');
}
if(attribute_contains($x->getAttribute('class'),'photo')){
$ret['photo'] = $x->getAttribute('src');
$targets_left = pop_scrape_target($targets, 'photo');
}
if(attribute_contains($x->getAttribute('class'),'key')){
$ret['key'] = $x->textContent;
$targets_left = pop_scrape_target($targets, 'key');
}
if(attribute_contains($x->getAttribute('class'),'locality')){
$ret['locality'] = $x->textContent;
$targets_left = pop_scrape_target($targets, 'locality');
}
if(attribute_contains($x->getAttribute('class'),'region')){
$ret['region'] = $x->textContent;
$targets_left = pop_scrape_target($targets, 'region');
}
if(attribute_contains($x->getAttribute('class'),'postal-code')){
$ret['postal-code'] = $x->textContent;
$targets_left = pop_scrape_target($targets, 'postal-code');
}
if(attribute_contains($x->getAttribute('class'),'country-name')){
$ret['country-name'] = $x->textContent;
$targets_left = pop_scrape_target($targets, 'country-name');
}
}
}
$nodes_left--;
if($nodes_left <= 0 || $targets_left <= 0) break;
}
$scrape_end = microtime(true);
$fetch_time = round(($scrape_fetch_end - $scrape_start) * 1000);
$scrape_time = round(($scrape_end - $scrape_fetch_end) * 1000);
$ret['_timings'] = array(
'fetch' => $fetch_time,
'scrape' => $scrape_time
);
return $ret;
}}
if(! function_exists('validate_dfrn')) {
function validate_dfrn($a) {
$errors = 0;
if(! x($a,'key'))
$errors ++;
if(! x($a,'dfrn-request'))
$errors ++;
if(! x($a,'dfrn-confirm'))
$errors ++;
if(! x($a,'dfrn-notify'))
$errors ++;
if(! x($a,'dfrn-poll'))
$errors ++;
return $errors;
}}
if(! function_exists('pop_scrape_target')) {
function pop_scrape_target(&$array, $name) {
$at = array_search($name, $array);
unset($array[$at]);
return count($array);
}}