friendica/dir
6
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Code style cleanup

Browse source
This commit is contained in:
Hypolite Petovan 2017-11-01 02:29:31 -04:00
parent b21b493613
commit be5939e02f
3 changed files with 427 additions and 447 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

91
include/site-health.php
View file

@ -1,5 +1,4 @@
<?php <?php
/* /*
Based on a submitted URL, take note of the site it mentions. Based on a submitted URL, take note of the site it mentions.
Ensures that the site health will be tracked if it wasn't already. Ensures that the site health will be tracked if it wasn't already.
@ -11,7 +10,6 @@
if (!function_exists('notice_site')) { if (!function_exists('notice_site')) {
function notice_site($url, $check_health = false) function notice_site($url, $check_health = false)
{ {
global $a; global $a;
//Parse the domain from the URL. //Parse the domain from the URL.
@ -25,12 +23,10 @@ function notice_site($url, $check_health=false)
//If it exists, see if we need to update any flags / statuses. //If it exists, see if we need to update any flags / statuses.
if (!empty($result) && isset($result[0])) { if (!empty($result) && isset($result[0])) {
$entry = $result[0]; $entry = $result[0];
//If we are allowed to do health checks... //If we are allowed to do health checks...
if ($check_health) { if ($check_health) {
//And the site is in bad health currently, do a check now. //And the site is in bad health currently, do a check now.
//This is because you have a high certainty the site may perform better now. //This is because you have a high certainty the site may perform better now.
if ($entry['health_score'] < -40) { if ($entry['health_score'] < -40) {
@ -42,9 +38,7 @@ function notice_site($url, $check_health=false)
elseif (strtotime($entry['dt_last_probed']) < time() - $a->config['site-health']['min_probe_delay']) { elseif (strtotime($entry['dt_last_probed']) < time() - $a->config['site-health']['min_probe_delay']) {
run_site_probe($entry['id'], $entry); run_site_probe($entry['id'], $entry);
} }
} }
} }
//If it does not exist. //If it does not exist.
@ -67,27 +61,24 @@ function notice_site($url, $check_health=false)
$entry = $result[0]; $entry = $result[0];
run_site_probe($result[0]['id'], $entry); run_site_probe($result[0]['id'], $entry);
} }
} }
} }
//Give other scripts the site health. //Give other scripts the site health.
return isset($entry) ? $entry : false; return isset($entry) ? $entry : false;
}
}} }
//Extracts the site from a given URL. //Extracts the site from a given URL.
if (!function_exists('parse_site_from_url')) { if (!function_exists('parse_site_from_url')) {
function parse_site_from_url($url) function parse_site_from_url($url)
{ {
//Currently a simple implementation, but may improve over time. //Currently a simple implementation, but may improve over time.
#TODO: support subdirectories? #TODO: support subdirectories?
$urlMeta = parse_url($url); $urlMeta = parse_url($url);
return $urlMeta['scheme'] . '://' . $urlMeta['host']; return $urlMeta['scheme'] . '://' . $urlMeta['host'];
}
}} }
//Performs a ping to the given site ID //Performs a ping to the given site ID
//You may need to notice the site first before you know it's ID. //You may need to notice the site first before you know it's ID.
@ -100,7 +91,6 @@ function parse_site_from_url($url)
if (!function_exists('run_site_probe')) { if (!function_exists('run_site_probe')) {
function run_site_probe($id, &$entry_out) function run_site_probe($id, &$entry_out)
{ {
global $a; global $a;
//Get the site information from the DB, based on the ID. //Get the site information from the DB, based on the ID.
@ -123,27 +113,22 @@ function run_site_probe($id, &$entry_out)
//Prepare the CURL call. //Prepare the CURL call.
$handle = curl_init(); $handle = curl_init();
$options = array( $options = array(
//Timeouts //Timeouts
CURLOPT_TIMEOUT => max($a->config['site-health']['probe_timeout'], 1), //Minimum of 1 second timeout. CURLOPT_TIMEOUT => max($a->config['site-health']['probe_timeout'], 1), //Minimum of 1 second timeout.
CURLOPT_CONNECTTIMEOUT => 1, CURLOPT_CONNECTTIMEOUT => 1,
//Redirecting //Redirecting
CURLOPT_FOLLOWLOCATION => true, CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 8, CURLOPT_MAXREDIRS => 8,
//SSL //SSL
CURLOPT_SSL_VERIFYPEER => true, CURLOPT_SSL_VERIFYPEER => true,
// CURLOPT_VERBOSE => true, // CURLOPT_VERBOSE => true,
// CURLOPT_CERTINFO => true, // CURLOPT_CERTINFO => true,
CURLOPT_SSL_VERIFYHOST => 2, CURLOPT_SSL_VERIFYHOST => 2,
CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS,
//Basic request //Basic request
CURLOPT_USERAGENT => 'friendica-directory-probe-1.0', CURLOPT_USERAGENT => 'friendica-directory-probe-1.0',
CURLOPT_RETURNTRANSFER => true, CURLOPT_RETURNTRANSFER => true,
CURLOPT_URL => $probe_location CURLOPT_URL => $probe_location
); );
curl_setopt_array($handle, $options); curl_setopt_array($handle, $options);
@ -161,7 +146,6 @@ function run_site_probe($id, &$entry_out)
//When it's the certificate that doesn't work. //When it's the certificate that doesn't work.
if ($sslcert_issues) { if ($sslcert_issues) {
//Probe again, without strict SSL. //Probe again, without strict SSL.
$options[CURLOPT_SSL_VERIFYPEER] = false; $options[CURLOPT_SSL_VERIFYPEER] = false;
@ -177,7 +161,6 @@ function run_site_probe($id, &$entry_out)
//Store new status. //Store new status.
$curl_statuscode = curl_errno($handle); $curl_statuscode = curl_errno($handle);
} }
//Gather more meta. //Gather more meta.
@ -215,7 +198,6 @@ function run_site_probe($id, &$entry_out)
} }
if (!$parse_failed) { if (!$parse_failed) {
$given_base_url_match = $data->url == $base_url; $given_base_url_match = $data->url == $base_url;
//Record the probe speed in a probes table. //Record the probe speed in a probes table.
@ -258,7 +240,6 @@ function run_site_probe($id, &$entry_out)
if (isset($data->no_scrape_url)) { if (isset($data->no_scrape_url)) {
$parsedDataQuery .= sprintf("`no_scrape_url` = '%s',", dbesc($data->no_scrape_url)); $parsedDataQuery .= sprintf("`no_scrape_url` = '%s',", dbesc($data->no_scrape_url));
} }
} }
//Get the new health. //Get the new health.
@ -285,39 +266,46 @@ function run_site_probe($id, &$entry_out)
if ($result && isset($result[0])) { if ($result && isset($result[0])) {
$entry_out = $result[0]; $entry_out = $result[0];
} }
}
}} }
//Determines the new health score after a probe has been executed. //Determines the new health score after a probe has been executed.
if (!function_exists('health_score_after_probe')) { if (!function_exists('health_score_after_probe')) {
function health_score_after_probe($current, $probe_success, $time = null, $version = null, $ssl_issues = null) function health_score_after_probe($current, $probe_success, $time = null, $version = null, $ssl_issues = null)
{ {
//Probe failed, costs you 30 points. //Probe failed, costs you 30 points.
if(!$probe_success) return max($current-30, -100); if (!$probe_success) {
return max($current - 30, -100);
}
//A good probe gives you 20 points. //A good probe gives you 20 points.
$current += 20; $current += 20;
//Speed scoring. //Speed scoring.
if (intval($time) > 0) { if (intval($time) > 0) {
//Pentaly / bonus points. //Pentaly / bonus points.
if ($time > 800) $current -= 10; //Bad speed. if ($time > 800) {
elseif ($time > 400) $current -= 5; //Still not good. $current -= 10; //Bad speed.
elseif ($time > 250) $current += 0; //This is normal. } elseif ($time > 400) {
elseif ($time > 120) $current += 5; //Good speed. $current -= 5; //Still not good.
else $current += 10; //Excellent speed. } elseif ($time > 250) {
$current += 0; //This is normal.
} elseif ($time > 120) {
$current += 5; //Good speed.
} else {
$current += 10; //Excellent speed.
}
//Cap for bad speeds. //Cap for bad speeds.
if ($time > 800) $current = min(40, $current); if ($time > 800) {
elseif ($time > 400) $current = min(60, $current); $current = min(40, $current);
} elseif ($time > 400) {
$current = min(60, $current);
}
} }
//Version check. //Version check.
if (!empty($version)) { if (!empty($version)) {
$versionParts = explode('.', $version); $versionParts = explode('.', $version);
//Older than 3.x.x? //Older than 3.x.x?
@ -333,7 +321,6 @@ function health_score_after_probe($current, $probe_success, $time=null, $version
#TODO: See if this needs to be more dynamic. #TODO: See if this needs to be more dynamic.
#TODO: See if this is a proper indicator of health. #TODO: See if this is a proper indicator of health.
} }
//SSL problems? That's a big deal. //SSL problems? That's a big deal.
@ -343,19 +330,25 @@ function health_score_after_probe($current, $probe_success, $time=null, $version
//Don't go beyond +100 or -100. //Don't go beyond +100 or -100.
return max(min(100, $current), -100); return max(min(100, $current), -100);
}
}} }
//Changes a score into a name. Used for classes and such. //Changes a score into a name. Used for classes and such.
if (!function_exists('health_score_to_name')) { if (!function_exists('health_score_to_name')) {
function health_score_to_name($score) function health_score_to_name($score)
{ {
if ($score < -50) {
if ($score < -50) return 'very-bad'; return 'very-bad';
elseif ($score < 0) return 'bad'; } elseif ($score < 0) {
elseif ($score < 30) return 'neutral'; return 'bad';
elseif ($score < 50) return 'ok'; } elseif ($score < 30) {
elseif ($score < 80) return 'good'; return 'neutral';
else return 'perfect'; } elseif ($score < 50) {
return 'ok';
}} } elseif ($score < 80) {
return 'good';
} else {
return 'perfect';
}
}
}

125
include/submit.php
View file

@ -1,14 +1,16 @@
<?php <?php
require_once 'datetime.php';
require_once 'site-health.php';
require_once 'Scrape.php';
require_once 'Photo.php';
require_once('datetime.php'); function run_submit($url)
require_once('site-health.php'); {
function run_submit($url) {
global $a; global $a;
if(! strlen($url)) if (!strlen($url)) {
return false; return false;
}
logger('Updating: ' . $url); logger('Updating: ' . $url);
@ -26,6 +28,8 @@ function run_submit($url) {
dbesc($nurl) dbesc($nurl)
); );
$profile_id = null;
if (count($r)) { if (count($r)) {
$profile_exists = true; $profile_exists = true;
$profile_id = $r[0]['id']; $profile_id = $r[0]['id'];
@ -33,10 +37,7 @@ function run_submit($url) {
$r = q("UPDATE `profile` SET $r = q("UPDATE `profile` SET
`available` = 0, `available` = 0,
`updated` = '%s' `updated` = '%s'
WHERE `id` = %d LIMIT 1", WHERE `id` = %d LIMIT 1", dbesc(datetime_convert()), intval($profile_id)
dbesc(datetime_convert()),
intval($profile_id)
); );
} }
@ -53,32 +54,27 @@ function run_submit($url) {
} }
} }
require_once('Scrape.php');
//Skip the scrape? :D //Skip the scrape? :D
$noscrape = $site_health && $site_health['no_scrape_url']; $noscrape = $site_health && $site_health['no_scrape_url'];
if ($noscrape) { if ($noscrape) {
//Find out who to look up. //Find out who to look up.
$which = str_replace($site_health['base_url'], '', $url); $which = str_replace($site_health['base_url'], '', $url);
$noscrape = preg_match('~/profile/([^/]+)~', $which, $matches) === 1; $noscrape = preg_match('~/profile/([^/]+)~', $which, $matches) === 1;
//If that did not fail... //If that did not fail...
if ($noscrape) { if ($noscrape) {
$parms = noscrape_dfrn($site_health['no_scrape_url'].'/'.$matches[1]); $params = noscrape_dfrn($site_health['no_scrape_url'] . '/' . $matches[1]);
$noscrape = !!$parms; //If the result was false, do a scrape after all. $noscrape = !!$params; //If the result was false, do a scrape after all.
} }
} }
if (!$noscrape) { if (!$noscrape) {
$parms = scrape_dfrn($url); $params = scrape_dfrn($url);
} }
// Empty result is due to an offline site. // Empty result is due to an offline site.
if(!count($parms) > 1){ if (!count($params) > 1) {
//For large sites this could lower the health too quickly, so don't track health. //For large sites this could lower the health too quickly, so don't track health.
//But for sites that are already in bad status. Do a cleanup now. //But for sites that are already in bad status. Do a cleanup now.
if ($profile_exists && $site_health['health_score'] < $a->config['maintenance']['remove_profile_health_threshold']) { if ($profile_exists && $site_health['health_score'] < $a->config['maintenance']['remove_profile_health_threshold']) {
@ -87,17 +83,14 @@ function run_submit($url) {
} }
return false; return false;
} elseif (x($params, 'explicit-hide') && $profile_exists) {
}
// We don't care about valid dfrn if the user indicates to be hidden. // We don't care about valid dfrn if the user indicates to be hidden.
elseif($parms['explicit-hide'] && $profile_exists) {
logger('User opted out of the directory.'); logger('User opted out of the directory.');
nuke_record($url); nuke_record($url);
return true; //This is a good update. return true; //This is a good update.
} }
if((x($parms,'hide')) || (! (x($parms,'fn')) && (x($parms,'photo')))) { if ((x($params, 'hide')) || (!(x($params, 'fn')) && (x($params, 'photo')))) {
if ($profile_exists) { if ($profile_exists) {
logger('Profile inferred to be opted out of the directory.'); logger('Profile inferred to be opted out of the directory.');
nuke_record($url); nuke_record($url);
@ -106,17 +99,18 @@ function run_submit($url) {
} }
// This is most likely a problem with the site configuration. Ignore. // This is most likely a problem with the site configuration. Ignore.
if(validate_dfrn($parms)) { if (validate_dfrn($params)) {
logger('Site is unavailable'); logger('Site is unavailable');
return false; return false;
} }
$photo = $parms['photo']; $photo = $params['photo'];
dbesc_array($parms); dbesc_array($params);
if(x($parms,'comm')) if (x($params, 'comm')) {
$parms['comm'] = intval($parms['comm']); $params['comm'] = intval($params['comm']);
}
if ($profile_exists) { if ($profile_exists) {
$r = q("UPDATE `profile` SET $r = q("UPDATE `profile` SET
@ -133,48 +127,46 @@ function run_submit($url) {
`available` = 1, `available` = 1,
`updated` = '%s' `updated` = '%s'
WHERE `id` = %d LIMIT 1", WHERE `id` = %d LIMIT 1",
$params['fn'],
$parms['fn'], $params['pdesc'],
$parms['pdesc'], $params['locality'],
$parms['locality'], $params['region'],
$parms['region'], $params['postal-code'],
$parms['postal-code'], $params['country-name'],
$parms['country-name'],
dbesc($url), dbesc($url),
dbesc($nurl), dbesc($nurl),
intval($parms['comm']), intval($params['comm']),
$parms['tags'], $params['tags'],
dbesc(datetime_convert()), dbesc(datetime_convert()),
intval($profile_id) intval($profile_id)
); );
logger('Update returns: ' . $r); logger('Update returns: ' . $r);
} else {
}
else {
$r = q("INSERT INTO `profile` ( `name`, `pdesc`, `locality`, `region`, `postal-code`, `country-name`, `homepage`, `nurl`, `comm`, `tags`, `created`, `updated` ) $r = q("INSERT INTO `profile` ( `name`, `pdesc`, `locality`, `region`, `postal-code`, `country-name`, `homepage`, `nurl`, `comm`, `tags`, `created`, `updated` )
VALUES ( '%s', '%s', '%s', '%s' , '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s' )", VALUES ( '%s', '%s', '%s', '%s' , '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s' )",
$parms['fn'], $params['fn'],
$parms['pdesc'], x($params, 'pdesc') ? $params['pdesc'] : '',
$parms['locality'], x($params, 'locality') ? $params['locality'] : '',
$parms['region'], x($params, 'region') ? $params['region'] : '',
$parms['postal-code'], x($params, 'postal-code') ? $params['postal-code'] : '',
$parms['country-name'], x($params, 'country-name') ? $params['country-name'] : '',
dbesc($url), dbesc($url),
dbesc($nurl), dbesc($nurl),
intval($parms['comm']), intval($params['comm']),
$parms['tags'], x($params, 'tags') ? $params['tags'] : '',
dbesc(datetime_convert()), dbesc(datetime_convert()),
dbesc(datetime_convert()) dbesc(datetime_convert())
); );
logger('Insert returns: ' . $r); logger('Insert returns: ' . $r);
$r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' or `nurl` = '%s' ) order by id asc", $r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' or `nurl` = '%s' ) ORDER BY `id` ASC",
dbesc($url), dbesc($url),
dbesc($nurl) dbesc($nurl)
); );
if(count($r)) if (count($r)) {
$profile_id = $r[count($r) - 1]['id']; $profile_id = $r[count($r) - 1]['id'];
}
if (count($r) > 1) { if (count($r) > 1) {
q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1", q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1",
@ -184,12 +176,10 @@ function run_submit($url) {
intval($r[0]['id']) intval($r[0]['id'])
); );
} }
} }
if($parms['tags']) { if ($params['tags']) {
$arr = explode(' ', $parms['tags']); $arr = explode(' ', $params['tags']);
if(count($arr)) {
foreach ($arr as $t) { foreach ($arr as $t) {
$t = strip_tags(trim($t)); $t = strip_tags(trim($t));
$t = substr($t, 0, 254); $t = substr($t, 0, 254);
@ -208,12 +198,9 @@ function run_submit($url) {
} }
} }
} }
}
$submit_photo_start = microtime(true); $submit_photo_start = microtime(true);
require_once("Photo.php");
$photo_failure = false; $photo_failure = false;
$status = false; $status = false;
@ -225,13 +212,11 @@ function run_submit($url) {
$img->scaleImageSquare(80); $img->scaleImageSquare(80);
$r = $img->store($profile_id); $r = $img->store($profile_id);
} }
$r = q("UPDATE `profile` SET `photo` = '%s' WHERE `id` = %d LIMIT 1", $r = q("UPDATE `profile` SET `photo` = '%s' WHERE `id` = %d LIMIT 1", dbesc($a->get_baseurl() . '/photo/' . $profile_id . '.jpg'),
dbesc($a->get_baseurl() . '/photo/' . $profile_id . '.jpg'),
intval($profile_id) intval($profile_id)
); );
$status = true; $status = true;
} } else {
else{
nuke_record($url); nuke_record($url);
return false; return false;
} }
@ -241,23 +226,23 @@ function run_submit($url) {
$time = round(($submit_end - $submit_start) * 1000); $time = round(($submit_end - $submit_start) * 1000);
//Record the scrape speed in a scrapes table. //Record the scrape speed in a scrapes table.
if($site_health && $status) q( if ($site_health && $status) {
q(
"INSERT INTO `site-scrape` (`site_health_id`, `dt_performed`, `request_time`, `scrape_time`, `photo_time`, `total_time`)" . "INSERT INTO `site-scrape` (`site_health_id`, `dt_performed`, `request_time`, `scrape_time`, `photo_time`, `total_time`)" .
"VALUES (%u, NOW(), %u, %u, %u, %u)", "VALUES (%u, NOW(), %u, %u, %u, %u)",
$site_health['id'], $site_health['id'],
$parms['_timings']['fetch'], $params['_timings']['fetch'],
$parms['_timings']['scrape'], $params['_timings']['scrape'],
$photo_time, $photo_time,
$time $time
); );
return $status;
} }
return $status;
}
function nuke_record($url) { function nuke_record($url)
{
$nurl = str_replace(array('https:', '//www.'), array('http:', '//'), $url); $nurl = str_replace(array('https:', '//www.'), array('http:', '//'), $url);
$r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' OR `nurl` = '%s' ) ", $r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' OR `nurl` = '%s' ) ",

10
mod/submit.php
View file

@ -1,10 +1,12 @@
<?php <?php
require_once('include/submit.php'); require_once 'include/submit.php';
require_once('include/sync.php'); require_once 'include/sync.php';
function submit_content(&$a) { use Friendica\Directory\App;
function submit_content(App &$a)
{
//Decode the URL. //Decode the URL.
$url = hex2bin(notags(trim($_GET['url']))); $url = hex2bin(notags(trim($_GET['url'])));
@ -13,6 +15,6 @@ function submit_content(&$a) {
//Run the submit sequence. //Run the submit sequence.
run_submit($url); run_submit($url);
exit;
exit;
} }