friendica
/
dir
6
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code style cleanup

This commit is contained in:
Hypolite Petovan 2017-11-01 02:29:31 -04:00
parent b21b493613
commit be5939e02f
3 changed files with 427 additions and 447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

649
include/site-health.php
View File

@ -1,5 +1,4 @@
<?php <?php
/* /*
Based on a submitted URL, take note of the site it mentions. Based on a submitted URL, take note of the site it mentions.
Ensures that the site health will be tracked if it wasn't already. Ensures that the site health will be tracked if it wasn't already.
@ -7,228 +6,211 @@
Do not enable it unless you have enough execution time to do so. Do not enable it unless you have enough execution time to do so.
But when you do, it's better to check for health whenever a site submits something. But when you do, it's better to check for health whenever a site submits something.
After all, the highest chance for the server to be online is when it submits activity. After all, the highest chance for the server to be online is when it submits activity.
*/ */
if(! function_exists('notice_site')){ if (!function_exists('notice_site')) {
function notice_site($url, $check_health=false) function notice_site($url, $check_health = false)
{ {
global $a;
global $a;
//Parse the domain from the URL.
//Parse the domain from the URL. $site = parse_site_from_url($url);
$site = parse_site_from_url($url);
//Search for it in the site-health table.
//Search for it in the site-health table. $result = q(
$result = q( "SELECT * FROM `site-health` WHERE `base_url`= '%s' ORDER BY `id` ASC LIMIT 1",
"SELECT * FROM `site-health` WHERE `base_url`= '%s' ORDER BY `id` ASC LIMIT 1", dbesc($site)
dbesc($site) );
);
//If it exists, see if we need to update any flags / statuses.
//If it exists, see if we need to update any flags / statuses. if (!empty($result) && isset($result[0])) {
if(!empty($result) && isset($result[0])){ $entry = $result[0];
$entry = $result[0]; //If we are allowed to do health checks...
if ($check_health) {
//If we are allowed to do health checks... //And the site is in bad health currently, do a check now.
if($check_health){ //This is because you have a high certainty the site may perform better now.
if ($entry['health_score'] < -40) {
//And the site is in bad health currently, do a check now. run_site_probe($entry['id'], $entry);
//This is because you have a high certainty the site may perform better now. }
if($entry['health_score'] < -40){
run_site_probe($entry['id'], $entry); //Or if the site has not been probed for longer than the minimum delay.
} //This is to make sure not everything is postponed to the batches.
elseif (strtotime($entry['dt_last_probed']) < time() - $a->config['site-health']['min_probe_delay']) {
//Or if the site has not been probed for longer than the minimum delay. run_site_probe($entry['id'], $entry);
//This is to make sure not everything is postponed to the batches. }
elseif(strtotime($entry['dt_last_probed']) < time()-$a->config['site-health']['min_probe_delay']){ }
run_site_probe($entry['id'], $entry); }
}
//If it does not exist.
} else {
} //Add it and make sure it is ready for probing.
q(
//If it does not exist. "INSERT INTO `site-health` (`base_url`, `dt_first_noticed`) VALUES ('%s', NOW())",
else{ dbesc($site)
);
//Add it and make sure it is ready for probing.
q( //And in case we should probe now, do so.
"INSERT INTO `site-health` (`base_url`, `dt_first_noticed`) VALUES ('%s', NOW())", if ($check_health) {
dbesc($site)
); $result = q(
"SELECT * FROM `site-health` WHERE `base_url`= '%s' ORDER BY `id` ASC LIMIT 1",
//And in case we should probe now, do so. dbesc($site)
if($check_health){ );
if (!empty($result) && isset($result[0])) {
$result = q( $entry = $result[0];
"SELECT * FROM `site-health` WHERE `base_url`= '%s' ORDER BY `id` ASC LIMIT 1", run_site_probe($result[0]['id'], $entry);
dbesc($site) }
); }
if(!empty($result) && isset($result[0])){ }
$entry = $result[0];
run_site_probe($result[0]['id'], $entry); //Give other scripts the site health.
} return isset($entry) ? $entry : false;
}
} }
}
//Give other scripts the site health.
return isset($entry) ? $entry : false;
}}
//Extracts the site from a given URL. //Extracts the site from a given URL.
if(! function_exists('parse_site_from_url')){ if (!function_exists('parse_site_from_url')) {
function parse_site_from_url($url) function parse_site_from_url($url)
{ {
//Currently a simple implementation, but may improve over time.
//Currently a simple implementation, but may improve over time. #TODO: support subdirectories?
#TODO: support subdirectories? $urlMeta = parse_url($url);
$urlMeta = parse_url($url); return $urlMeta['scheme'] . '://' . $urlMeta['host'];
return $urlMeta['scheme'].'://'.$urlMeta['host']; }
}
}}
//Performs a ping to the given site ID //Performs a ping to the given site ID
//You may need to notice the site first before you know it's ID. //You may need to notice the site first before you know it's ID.
//TODO: Probe server location using IP address or using the info the friendica server provides (preferred). //TODO: Probe server location using IP address or using the info the friendica server provides (preferred).
// If IP needs to be used only provide country information. // If IP needs to be used only provide country information.
//TODO: Check SSLLabs Grade //TODO: Check SSLLabs Grade
// Check needs to be asynchronous, meaning that the check at SSLLabs will be initiated in one run while // Check needs to be asynchronous, meaning that the check at SSLLabs will be initiated in one run while
// the results must be fetched later. It might be good to mark sites, where a check has been inititated // the results must be fetched later. It might be good to mark sites, where a check has been inititated
// f.e. using the ssl_grade field. In the next run, results of these sites could be fetched. // f.e. using the ssl_grade field. In the next run, results of these sites could be fetched.
if(! function_exists('run_site_probe')){ if (!function_exists('run_site_probe')) {
function run_site_probe($id, &$entry_out) function run_site_probe($id, &$entry_out)
{ {
global $a;
global $a;
//Get the site information from the DB, based on the ID.
//Get the site information from the DB, based on the ID. $result = q(
$result = q( "SELECT * FROM `site-health` WHERE `id`= %u ORDER BY `id` ASC LIMIT 1",
"SELECT * FROM `site-health` WHERE `id`= %u ORDER BY `id` ASC LIMIT 1", intval($id)
intval($id) );
);
//Abort the probe if site is not known.
//Abort the probe if site is not known. if (!$result || !isset($result[0])) {
if(!$result || !isset($result[0])){ logger('Unknown site-health ID being probed: ' . $id);
logger('Unknown site-health ID being probed: '.$id); throw new \Exception('Unknown site-health ID being probed: ' . $id);
throw new \Exception('Unknown site-health ID being probed: '.$id); }
}
//Shortcut.
//Shortcut. $entry = $result[0];
$entry = $result[0]; $base_url = $entry['base_url'];
$base_url = $entry['base_url']; $probe_location = $base_url . '/friendica/json';
$probe_location = $base_url.'/friendica/json';
//Prepare the CURL call.
//Prepare the CURL call. $handle = curl_init();
$handle = curl_init(); $options = array(
$options = array( //Timeouts
CURLOPT_TIMEOUT => max($a->config['site-health']['probe_timeout'], 1), //Minimum of 1 second timeout.
//Timeouts CURLOPT_CONNECTTIMEOUT => 1,
CURLOPT_TIMEOUT => max($a->config['site-health']['probe_timeout'], 1), //Minimum of 1 second timeout. //Redirecting
CURLOPT_CONNECTTIMEOUT => 1, CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 8,
//Redirecting //SSL
CURLOPT_FOLLOWLOCATION => true, CURLOPT_SSL_VERIFYPEER => true,
CURLOPT_MAXREDIRS => 8, // CURLOPT_VERBOSE => true,
// CURLOPT_CERTINFO => true,
//SSL CURLOPT_SSL_VERIFYHOST => 2,
CURLOPT_SSL_VERIFYPEER => true, CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS,
// CURLOPT_VERBOSE => true, //Basic request
// CURLOPT_CERTINFO => true, CURLOPT_USERAGENT => 'friendica-directory-probe-1.0',
CURLOPT_SSL_VERIFYHOST => 2, CURLOPT_RETURNTRANSFER => true,
CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS, CURLOPT_URL => $probe_location
);
//Basic request curl_setopt_array($handle, $options);
CURLOPT_USERAGENT => 'friendica-directory-probe-1.0',
CURLOPT_RETURNTRANSFER => true, //Probe the site.
CURLOPT_URL => $probe_location $probe_start = microtime(true);
$probe_data = curl_exec($handle);
); $probe_end = microtime(true);
curl_setopt_array($handle, $options);
//Check for SSL problems.
//Probe the site. $curl_statuscode = curl_errno($handle);
$probe_start = microtime(true); $sslcert_issues = in_array($curl_statuscode, array(
$probe_data = curl_exec($handle); 60, //Could not authenticate certificate with known CA's
$probe_end = microtime(true); 83 //Issuer check failed
));
//Check for SSL problems.
$curl_statuscode = curl_errno($handle); //When it's the certificate that doesn't work.
$sslcert_issues = in_array($curl_statuscode, array( if ($sslcert_issues) {
60, //Could not authenticate certificate with known CA's //Probe again, without strict SSL.
83 //Issuer check failed $options[CURLOPT_SSL_VERIFYPEER] = false;
));
//Replace the handle.
//When it's the certificate that doesn't work. curl_close($handle);
if($sslcert_issues){ $handle = curl_init();
curl_setopt_array($handle, $options);
//Probe again, without strict SSL.
$options[CURLOPT_SSL_VERIFYPEER] = false; //Probe.
$probe_start = microtime(true);
//Replace the handle. $probe_data = curl_exec($handle);
curl_close($handle); $probe_end = microtime(true);
$handle = curl_init();
curl_setopt_array($handle, $options); //Store new status.
$curl_statuscode = curl_errno($handle);
//Probe. }
$probe_start = microtime(true);
$probe_data = curl_exec($handle); //Gather more meta.
$probe_end = microtime(true); $time = round(($probe_end - $probe_start) * 1000);
$status = curl_getinfo($handle, CURLINFO_HTTP_CODE);
//Store new status. $type = curl_getinfo($handle, CURLINFO_CONTENT_TYPE);
$curl_statuscode = curl_errno($handle); $info = curl_getinfo($handle);
} //Done with CURL now.
curl_close($handle);
//Gather more meta.
$time = round(($probe_end - $probe_start) * 1000); #TODO: if the site redirects elsewhere, notice this site and record an issue.
$status = curl_getinfo($handle, CURLINFO_HTTP_CODE); $effective_base_url = parse_site_from_url($info['url']);
$type = curl_getinfo($handle, CURLINFO_CONTENT_TYPE); $wrong_base_url = $effective_base_url !== $entry['base_url'];
$info = curl_getinfo($handle);
try {
//Done with CURL now. $data = json_decode($probe_data);
curl_close($handle); } catch (\Exception $ex) {
$data = false;
#TODO: if the site redirects elsewhere, notice this site and record an issue. }
$effective_base_url = parse_site_from_url($info['url']);
$wrong_base_url = $effective_base_url !== $entry['base_url']; $parse_failed = !$data;
try{ $parsedDataQuery = '';
$data = json_decode($probe_data);
}catch(\Exception $ex){ logger('Effective Base URL: ' . $effective_base_url);
$data = false;
} if ($wrong_base_url) {
$parsedDataQuery .= sprintf(
$parse_failed = !$data; "`effective_base_url` = '%s',",
dbesc($effective_base_url)
$parsedDataQuery = ''; );
} else {
logger('Effective Base URL: ' . $effective_base_url); $parsedDataQuery .= "`effective_base_url` = NULL,";
}
if($wrong_base_url){
$parsedDataQuery .= sprintf( if (!$parse_failed) {
"`effective_base_url` = '%s',", $given_base_url_match = $data->url == $base_url;
dbesc($effective_base_url)
); //Record the probe speed in a probes table.
}else{ q(
$parsedDataQuery .= "`effective_base_url` = NULL,"; "INSERT INTO `site-probe` (`site_health_id`, `dt_performed`, `request_time`)" .
} "VALUES (%u, NOW(), %u)",
$entry['id'],
if(!$parse_failed){ $time
);
$given_base_url_match = $data->url == $base_url;
//Update any health calculations or otherwise processed data.
//Record the probe speed in a probes table. $parsedDataQuery .= sprintf(
q( "`dt_last_seen` = NOW(),
"INSERT INTO `site-probe` (`site_health_id`, `dt_performed`, `request_time`)".
"VALUES (%u, NOW(), %u)",
$entry['id'],
$time
);
//Update any health calculations or otherwise processed data.
$parsedDataQuery .= sprintf(
"`dt_last_seen` = NOW(),
`name` = '%s', `name` = '%s',
`version` = '%s', `version` = '%s',
`plugins` = '%s', `plugins` = '%s',
@ -237,125 +219,136 @@ function run_site_probe($id, &$entry_out)
`admin_name` = '%s', `admin_name` = '%s',
`admin_profile` = '%s', `admin_profile` = '%s',
", ",
dbesc($data->site_name), dbesc($data->site_name),
dbesc($data->version), dbesc($data->version),
dbesc(implode("\r\n", $data->plugins)), dbesc(implode("\r\n", $data->plugins)),
dbesc($data->register_policy), dbesc($data->register_policy),
dbesc($data->info), dbesc($data->info),
dbesc($data->admin->name), dbesc($data->admin->name),
dbesc($data->admin->profile) dbesc($data->admin->profile)
); );
//Did we use HTTPS? //Did we use HTTPS?
$urlMeta = parse_url($probe_location); $urlMeta = parse_url($probe_location);
if($urlMeta['scheme'] == 'https'){ if ($urlMeta['scheme'] == 'https') {
$parsedDataQuery .= sprintf("`ssl_state` = b'%u',", $sslcert_issues ? '0' : '1'); $parsedDataQuery .= sprintf("`ssl_state` = b'%u',", $sslcert_issues ? '0' : '1');
} else { } else {
$parsedDataQuery .= "`ssl_state` = NULL,"; $parsedDataQuery .= "`ssl_state` = NULL,";
} }
//Do we have a no scrape supporting node? :D //Do we have a no scrape supporting node? :D
if(isset($data->no_scrape_url)){ if (isset($data->no_scrape_url)) {
$parsedDataQuery .= sprintf("`no_scrape_url` = '%s',", dbesc($data->no_scrape_url)); $parsedDataQuery .= sprintf("`no_scrape_url` = '%s',", dbesc($data->no_scrape_url));
} }
}
}
//Get the new health.
//Get the new health. $version = $parse_failed ? '' : $data->version;
$version = $parse_failed ? '' : $data->version; $health = health_score_after_probe($entry['health_score'], !$parse_failed, $time, $version, $sslcert_issues);
$health = health_score_after_probe($entry['health_score'], !$parse_failed, $time, $version, $sslcert_issues);
//Update the health.
//Update the health. q("UPDATE `site-health` SET
q("UPDATE `site-health` SET
`health_score` = '%d', `health_score` = '%d',
$parsedDataQuery $parsedDataQuery
`dt_last_probed` = NOW() `dt_last_probed` = NOW()
WHERE `id` = %d LIMIT 1", WHERE `id` = %d LIMIT 1",
$health, $health,
$entry['id'] $entry['id']
); );
//Get the site information from the DB, based on the ID. //Get the site information from the DB, based on the ID.
$result = q( $result = q(
"SELECT * FROM `site-health` WHERE `id`= %u ORDER BY `id` ASC LIMIT 1", "SELECT * FROM `site-health` WHERE `id`= %u ORDER BY `id` ASC LIMIT 1",
$entry['id'] $entry['id']
); );
//Return updated entry data. //Return updated entry data.
if($result && isset($result[0])){ if ($result && isset($result[0])) {
$entry_out = $result[0]; $entry_out = $result[0];
} }
}
}} }
//Determines the new health score after a probe has been executed. //Determines the new health score after a probe has been executed.
if(! function_exists('health_score_after_probe')){ if (!function_exists('health_score_after_probe')) {
function health_score_after_probe($current, $probe_success, $time=null, $version=null, $ssl_issues=null) function health_score_after_probe($current, $probe_success, $time = null, $version = null, $ssl_issues = null)
{ {
//Probe failed, costs you 30 points.
//Probe failed, costs you 30 points. if (!$probe_success) {
if(!$probe_success) return max($current-30, -100); return max($current - 30, -100);
}
//A good probe gives you 20 points.
$current += 20; //A good probe gives you 20 points.
$current += 20;
//Speed scoring.
if(intval($time) > 0){ //Speed scoring.
if (intval($time) > 0) {
//Pentaly / bonus points. //Pentaly / bonus points.
if ($time > 800) $current -= 10; //Bad speed. if ($time > 800) {
elseif ($time > 400) $current -= 5; //Still not good. $current -= 10; //Bad speed.
elseif ($time > 250) $current += 0; //This is normal. } elseif ($time > 400) {
elseif ($time > 120) $current += 5; //Good speed. $current -= 5; //Still not good.
else $current += 10; //Excellent speed. } elseif ($time > 250) {
$current += 0; //This is normal.
//Cap for bad speeds. } elseif ($time > 120) {
if ($time > 800) $current = min(40, $current); $current += 5; //Good speed.
elseif ($time > 400) $current = min(60, $current); } else {
$current += 10; //Excellent speed.
} }
//Version check. //Cap for bad speeds.
if(!empty($version)){ if ($time > 800) {
$current = min(40, $current);
$versionParts = explode('.', $version); } elseif ($time > 400) {
$current = min(60, $current);
//Older than 3.x.x? }
//Your score can not go above 30 health. }
if(intval($versionParts[0]) < 3){
$current = min($current, 30); //Version check.
} if (!empty($version)) {
$versionParts = explode('.', $version);
//Older than 3.3.x?
elseif(intval($versionParts[1] < 3)){ //Older than 3.x.x?
$current -= 5; //Somewhat outdated. //Your score can not go above 30 health.
} if (intval($versionParts[0]) < 3) {
$current = min($current, 30);
#TODO: See if this needs to be more dynamic. }
#TODO: See if this is a proper indicator of health.
//Older than 3.3.x?
} elseif (intval($versionParts[1] < 3)) {
$current -= 5; //Somewhat outdated.
//SSL problems? That's a big deal. }
if($ssl_issues === true){
$current -= 10; #TODO: See if this needs to be more dynamic.
} #TODO: See if this is a proper indicator of health.
}
//Don't go beyond +100 or -100.
return max(min(100, $current), -100); //SSL problems? That's a big deal.
if ($ssl_issues === true) {
}} $current -= 10;
}
//Don't go beyond +100 or -100.
return max(min(100, $current), -100);
}
}
//Changes a score into a name. Used for classes and such. //Changes a score into a name. Used for classes and such.
if(! function_exists('health_score_to_name')){ if (!function_exists('health_score_to_name')) {
function health_score_to_name($score) function health_score_to_name($score)
{ {
if ($score < -50) {
if ($score < -50) return 'very-bad'; return 'very-bad';
elseif ($score < 0) return 'bad'; } elseif ($score < 0) {
elseif ($score < 30) return 'neutral'; return 'bad';
elseif ($score < 50) return 'ok'; } elseif ($score < 30) {
elseif ($score < 80) return 'good'; return 'neutral';
else return 'perfect'; } elseif ($score < 50) {
return 'ok';
}} } elseif ($score < 80) {
return 'good';
} else {
return 'perfect';
}
}
}

197
include/submit.php
View File

@ -1,14 +1,16 @@
<?php <?php
require_once 'datetime.php';
require_once 'site-health.php';
require_once 'Scrape.php';
require_once 'Photo.php';
require_once('datetime.php'); function run_submit($url)
require_once('site-health.php'); {
function run_submit($url) {
global $a; global $a;
if(! strlen($url)) if (!strlen($url)) {
return false; return false;
}
logger('Updating: ' . $url); logger('Updating: ' . $url);
@ -17,7 +19,7 @@ function run_submit($url) {
$submit_start = microtime(true); $submit_start = microtime(true);
$nurl = str_replace(array('https:','//www.'), array('http:','//'), $url); $nurl = str_replace(array('https:', '//www.'), array('http:', '//'), $url);
$profile_exists = false; $profile_exists = false;
@ -26,24 +28,23 @@ function run_submit($url) {
dbesc($nurl) dbesc($nurl)
); );
if(count($r)) { $profile_id = null;
if (count($r)) {
$profile_exists = true; $profile_exists = true;
$profile_id = $r[0]['id']; $profile_id = $r[0]['id'];
$r = q("UPDATE `profile` SET $r = q("UPDATE `profile` SET
`available` = 0, `available` = 0,
`updated` = '%s' `updated` = '%s'
WHERE `id` = %d LIMIT 1", WHERE `id` = %d LIMIT 1", dbesc(datetime_convert()), intval($profile_id)
dbesc(datetime_convert()),
intval($profile_id)
); );
} }
//Remove duplicates. //Remove duplicates.
if(count($r) > 1){ if (count($r) > 1) {
for($i=1; $i<count($r); $i++){ for ($i = 1; $i < count($r); $i++) {
logger('Removed duplicate profile '.intval($r[$i]['id'])); logger('Removed duplicate profile ' . intval($r[$i]['id']));
q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1", q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1",
intval($r[$i]['id']) intval($r[$i]['id'])
); );
@ -53,72 +54,65 @@ function run_submit($url) {
} }
} }
require_once('Scrape.php');
//Skip the scrape? :D //Skip the scrape? :D
$noscrape = $site_health && $site_health['no_scrape_url']; $noscrape = $site_health && $site_health['no_scrape_url'];
if($noscrape){ if ($noscrape) {
//Find out who to look up. //Find out who to look up.
$which = str_replace($site_health['base_url'], '', $url); $which = str_replace($site_health['base_url'], '', $url);
$noscrape = preg_match('~/profile/([^/]+)~', $which, $matches) === 1; $noscrape = preg_match('~/profile/([^/]+)~', $which, $matches) === 1;
//If that did not fail... //If that did not fail...
if($noscrape){ if ($noscrape) {
$parms = noscrape_dfrn($site_health['no_scrape_url'].'/'.$matches[1]); $params = noscrape_dfrn($site_health['no_scrape_url'] . '/' . $matches[1]);
$noscrape = !!$parms; //If the result was false, do a scrape after all. $noscrape = !!$params; //If the result was false, do a scrape after all.
} }
} }
if(!$noscrape){ if (!$noscrape) {
$parms = scrape_dfrn($url); $params = scrape_dfrn($url);
} }
//Empty result is due to an offline site. // Empty result is due to an offline site.
if(!count($parms) > 1){ if (!count($params) > 1) {
//For large sites this could lower the health too quickly, so don't track health. //For large sites this could lower the health too quickly, so don't track health.
//But for sites that are already in bad status. Do a cleanup now. //But for sites that are already in bad status. Do a cleanup now.
if($profile_exists && $site_health['health_score'] < $a->config['maintenance']['remove_profile_health_threshold']){ if ($profile_exists && $site_health['health_score'] < $a->config['maintenance']['remove_profile_health_threshold']) {
logger('Nuked bad health record.'); logger('Nuked bad health record.');
nuke_record($url); nuke_record($url);
} }
return false; return false;
} elseif (x($params, 'explicit-hide') && $profile_exists) {
} // We don't care about valid dfrn if the user indicates to be hidden.
//We don't care about valid dfrn if the user indicates to be hidden.
elseif($parms['explicit-hide'] && $profile_exists) {
logger('User opted out of the directory.'); logger('User opted out of the directory.');
nuke_record($url); nuke_record($url);
return true; //This is a good update. return true; //This is a good update.
} }
if((x($parms,'hide')) || (! (x($parms,'fn')) && (x($parms,'photo')))) { if ((x($params, 'hide')) || (!(x($params, 'fn')) && (x($params, 'photo')))) {
if($profile_exists) { if ($profile_exists) {
logger('Profile inferred to be opted out of the directory.'); logger('Profile inferred to be opted out of the directory.');
nuke_record($url); nuke_record($url);
} }
return true; //This is a good update. return true; //This is a good update.
} }
//This is most likely a problem with the site configuration. Ignore. // This is most likely a problem with the site configuration. Ignore.
if(validate_dfrn($parms)) { if (validate_dfrn($params)) {
logger('Site is unavailable'); logger('Site is unavailable');
return false; return false;
} }
$photo = $parms['photo']; $photo = $params['photo'];
dbesc_array($parms); dbesc_array($params);
if(x($parms,'comm')) if (x($params, 'comm')) {
$parms['comm'] = intval($parms['comm']); $params['comm'] = intval($params['comm']);
}
if($profile_exists) { if ($profile_exists) {
$r = q("UPDATE `profile` SET $r = q("UPDATE `profile` SET
`name` = '%s', `name` = '%s',
`pdesc` = '%s', `pdesc` = '%s',
@ -133,50 +127,48 @@ function run_submit($url) {
`available` = 1, `available` = 1,
`updated` = '%s' `updated` = '%s'
WHERE `id` = %d LIMIT 1", WHERE `id` = %d LIMIT 1",
$params['fn'],
$parms['fn'], $params['pdesc'],
$parms['pdesc'], $params['locality'],
$parms['locality'], $params['region'],
$parms['region'], $params['postal-code'],
$parms['postal-code'], $params['country-name'],
$parms['country-name'],
dbesc($url), dbesc($url),
dbesc($nurl), dbesc($nurl),
intval($parms['comm']), intval($params['comm']),
$parms['tags'], $params['tags'],
dbesc(datetime_convert()), dbesc(datetime_convert()),
intval($profile_id) intval($profile_id)
); );
logger('Update returns: ' . $r); logger('Update returns: ' . $r);
} else {
}
else {
$r = q("INSERT INTO `profile` ( `name`, `pdesc`, `locality`, `region`, `postal-code`, `country-name`, `homepage`, `nurl`, `comm`, `tags`, `created`, `updated` ) $r = q("INSERT INTO `profile` ( `name`, `pdesc`, `locality`, `region`, `postal-code`, `country-name`, `homepage`, `nurl`, `comm`, `tags`, `created`, `updated` )
VALUES ( '%s', '%s', '%s', '%s' , '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s' )", VALUES ( '%s', '%s', '%s', '%s' , '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s' )",
$parms['fn'], $params['fn'],
$parms['pdesc'], x($params, 'pdesc') ? $params['pdesc'] : '',
$parms['locality'], x($params, 'locality') ? $params['locality'] : '',
$parms['region'], x($params, 'region') ? $params['region'] : '',
$parms['postal-code'], x($params, 'postal-code') ? $params['postal-code'] : '',
$parms['country-name'], x($params, 'country-name') ? $params['country-name'] : '',
dbesc($url), dbesc($url),
dbesc($nurl), dbesc($nurl),
intval($parms['comm']), intval($params['comm']),
$parms['tags'], x($params, 'tags') ? $params['tags'] : '',
dbesc(datetime_convert()), dbesc(datetime_convert()),
dbesc(datetime_convert()) dbesc(datetime_convert())
); );
logger('Insert returns: ' . $r); logger('Insert returns: ' . $r);
$r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' or `nurl` = '%s' ) order by id asc", $r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' or `nurl` = '%s' ) ORDER BY `id` ASC",
dbesc($url), dbesc($url),
dbesc($nurl) dbesc($nurl)
); );
if(count($r)) if (count($r)) {
$profile_id = $r[count($r) - 1]['id']; $profile_id = $r[count($r) - 1]['id'];
}
if(count($r) > 1) { if (count($r) > 1) {
q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1", q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1",
intval($r[0]['id']) intval($r[0]['id'])
); );
@ -184,27 +176,24 @@ function run_submit($url) {
intval($r[0]['id']) intval($r[0]['id'])
); );
} }
} }
if($parms['tags']) { if ($params['tags']) {
$arr = explode(' ', $parms['tags']); $arr = explode(' ', $params['tags']);
if(count($arr)) { foreach ($arr as $t) {
foreach($arr as $t) { $t = strip_tags(trim($t));
$t = strip_tags(trim($t)); $t = substr($t, 0, 254);
$t = substr($t,0,254);
if(strlen($t)) { if (strlen($t)) {
$r = q("SELECT `id` FROM `tag` WHERE `term` = '%s' and `nurl` = '%s' LIMIT 1", $r = q("SELECT `id` FROM `tag` WHERE `term` = '%s' and `nurl` = '%s' LIMIT 1",
dbesc($t),
dbesc($nurl)
);
if (!count($r)) {
$r = q("INSERT INTO `tag` (`term`, `nurl`) VALUES ('%s', '%s') ",
dbesc($t), dbesc($t),
dbesc($nurl) dbesc($nurl)
); );
if(! count($r)) {
$r = q("INSERT INTO `tag` (`term`, `nurl`) VALUES ('%s', '%s') ",
dbesc($t),
dbesc($nurl)
);
}
} }
} }
} }
@ -212,26 +201,22 @@ function run_submit($url) {
$submit_photo_start = microtime(true); $submit_photo_start = microtime(true);
require_once("Photo.php");
$photo_failure = false; $photo_failure = false;
$status = false; $status = false;
if($profile_id) { if ($profile_id) {
$img_str = fetch_url($photo,true); $img_str = fetch_url($photo, true);
$img = new Photo($img_str); $img = new Photo($img_str);
if($img) { if ($img) {
$img->scaleImageSquare(80); $img->scaleImageSquare(80);
$r = $img->store($profile_id); $r = $img->store($profile_id);
} }
$r = q("UPDATE `profile` SET `photo` = '%s' WHERE `id` = %d LIMIT 1", $r = q("UPDATE `profile` SET `photo` = '%s' WHERE `id` = %d LIMIT 1", dbesc($a->get_baseurl() . '/photo/' . $profile_id . '.jpg'),
dbesc($a->get_baseurl() . '/photo/' . $profile_id . '.jpg'),
intval($profile_id) intval($profile_id)
); );
$status = true; $status = true;
} } else {
else{
nuke_record($url); nuke_record($url);
return false; return false;
} }
@ -241,32 +226,32 @@ function run_submit($url) {
$time = round(($submit_end - $submit_start) * 1000); $time = round(($submit_end - $submit_start) * 1000);
//Record the scrape speed in a scrapes table. //Record the scrape speed in a scrapes table.
if($site_health && $status) q( if ($site_health && $status) {
"INSERT INTO `site-scrape` (`site_health_id`, `dt_performed`, `request_time`, `scrape_time`, `photo_time`, `total_time`)". q(
"VALUES (%u, NOW(), %u, %u, %u, %u)", "INSERT INTO `site-scrape` (`site_health_id`, `dt_performed`, `request_time`, `scrape_time`, `photo_time`, `total_time`)" .
$site_health['id'], "VALUES (%u, NOW(), %u, %u, %u, %u)",
$parms['_timings']['fetch'], $site_health['id'],
$parms['_timings']['scrape'], $params['_timings']['fetch'],
$photo_time, $params['_timings']['scrape'],
$time $photo_time,
); $time
);
}
return $status; return $status;
} }
function nuke_record($url)
function nuke_record($url) { {
$nurl = str_replace(array('https:', '//www.'), array('http:', '//'), $url);
$nurl = str_replace(array('https:','//www.'), array('http:','//'), $url);
$r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' OR `nurl` = '%s' ) ", $r = q("SELECT `id` FROM `profile` WHERE ( `homepage` = '%s' OR `nurl` = '%s' ) ",
dbesc($url), dbesc($url),
dbesc($nurl) dbesc($nurl)
); );
if(count($r)) { if (count($r)) {
foreach($r as $rr) { foreach ($r as $rr) {
q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1", q("DELETE FROM `photo` WHERE `profile-id` = %d LIMIT 1",
intval($rr['id']) intval($rr['id'])
); );

28
mod/submit.php
View File

@ -1,18 +1,20 @@
<?php <?php
require_once('include/submit.php'); require_once 'include/submit.php';
require_once('include/sync.php'); require_once 'include/sync.php';
function submit_content(&$a) { use Friendica\Directory\App;
//Decode the URL. function submit_content(App &$a)
{
//Decode the URL.
$url = hex2bin(notags(trim($_GET['url']))); $url = hex2bin(notags(trim($_GET['url'])));
//Currently we simply push RAW URL's to our targets.
sync_push($url);
//Run the submit sequence.
run_submit($url);
exit;
} //Currently we simply push RAW URL's to our targets.
sync_push($url);
//Run the submit sequence.
run_submit($url);
exit;
}