This standalone software is meant to provide a global public directory of Friendica profiles across nodes.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 
friendica-directory/src/classes/Pollers/Server.php

479 lines
13 KiB

<?php
namespace Friendica\Directory\Pollers;
use GuzzleHttp\Psr7\Uri;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\TransferStats;
/**
* @author Hypolite Petovan <hypolite@mrpetovan.com>
*/
class Server
{
/**
* @var \Atlas\Pdo\Connection
*/
private $atlas;
/**
* @var \GuzzleHttp\ClientInterface
*/
private $http;
/**
* @var \Friendica\Directory\Models\ProfilePollQueue
*/
private $profilePollQueueModel;
/**
* @var \Friendica\Directory\Models\Server
*/
private $serverModel;
/**
* @var \Psr\SimpleCache\CacheInterface
*/
private $simplecache;
/**
* @var \Psr\Log\LoggerInterface
*/
private $logger;
/**
* @var array
*/
private $settings = [
'probe_timeout' => 5
];
public function __construct(
\Atlas\Pdo\Connection $atlas,
\GuzzleHttp\ClientInterface $http,
\Friendica\Directory\Models\ProfilePollQueue $profilePollQueueModel,
\Friendica\Directory\Models\Server $serverModel,
\Psr\SimpleCache\CacheInterface $simplecache,
\Psr\Log\LoggerInterface $logger,
array $settings)
{
$this->atlas = $atlas;
$this->http = $http;
$this->profilePollQueueModel = $profilePollQueueModel;
$this->serverModel = $serverModel;
$this->simplecache = $simplecache;
$this->logger = $logger;
$this->settings = array_merge($this->settings, $settings);
}
public function __invoke(string $polled_url): int
{
$this->logger->info('Poll server with URL: ' . $polled_url);
$host = parse_url($polled_url, PHP_URL_HOST);
if (!$host) {
$this->logger->warning('Missing hostname in polled server URL: ' . $polled_url);
return 0;
}
if (!\Friendica\Directory\Utils\Network::isPublicHost($host)) {
$this->logger->warning('Private/reserved IP in polled server URL: ' . $polled_url);
return 0;
}
$server = $this->serverModel->getByUrlAlias($polled_url);
if (
$server
&& substr($polled_url, 0, 7) == 'http://'
&& substr($server['base_url'], 0, 8) == 'https://'
) {
$this->logger->info('Favoring the HTTPS version of server with URL: ' . $polled_url);
return $server['id'];
}
if ($server) {
$this->atlas->perform('UPDATE `server` SET `available` = 0 WHERE `id` = :server_id', ['server_id' => $server['id']]);
}
$probe_result = $this->getProbeResult($polled_url);
$parse_success = !empty($probe_result['data']['url']);
$avg_ping = null;
if ($parse_success) {
$base_url = $probe_result['data']['url'];
// Maybe we know the server under the canonical URL?
if (!$server) {
$server = $this->serverModel->getByUrlAlias($base_url);
}
if (!$server) {
$this->atlas->perform('INSERT INTO `server` SET
`base_url` = :base_url,
`first_noticed` = NOW(),
`available` = 0,
`health_score` = 50',
['base_url' => $polled_url]
);
$server = [
'id' => $this->atlas->lastInsertId(),
'base_url' => $base_url,
'health_score' => 50
];
}
$this->serverModel->addAliasToServer($server['id'], $polled_url);
$this->serverModel->addAliasToServer($server['id'], $base_url);
$avg_ping = $this->getAvgPing($base_url);
if ($probe_result['time'] && $avg_ping) {
$speed_score = max(1, $avg_ping > 10 ? $probe_result['time'] / $avg_ping : $probe_result['time'] / 50);
} else {
$speed_score = null;
}
$this->atlas->perform('INSERT INTO `site_probe`
SET `server_id` = :server_id,
`request_time` = :request_time,
`avg_ping` = :avg_ping,
`speed_score` = :speed_score,
`timestamp` = NOW()',
[
'server_id' => $server['id'],
'request_time' => $probe_result['time'],
'avg_ping' => $avg_ping,
'speed_score' => $speed_score
]
);
if (isset($probe_result['data']['addons'])) {
$addons = $probe_result['data']['addons'];
} else {
// Backward compatibility
$addons = $probe_result['data']['plugins'];
}
if ($probe_result['data']['admin']['profile']) {
$subscribe = $this->getSubscribeUrl($probe_result['data']['url'], $probe_result['data']['admin']['profile']);
}
$this->atlas->perform(
'UPDATE `server`
SET `available` = 1,
`last_seen` = NOW(),
`base_url` = :base_url,
`name` = :name,
`language` = :language,
`version` = :version,
`addons` = :addons,
`reg_policy` = :reg_policy,
`info` = :info,
`admin_name` = :admin_name,
`admin_profile` = :admin_profile,
`noscrape_url` = :noscrape_url,
`subscribe_url` = :subscribe_url,
`ssl_state` = :ssl_state
WHERE `id` = :server_id',
[
'server_id' => $server['id'],
'base_url' => strtolower($probe_result['data']['url']),
'name' => $probe_result['data']['site_name'],
'language' => $probe_result['data']['language'] ?? null,
'version' => $probe_result['data']['version'],
'addons' => implode(',', $addons),
'reg_policy' => $probe_result['data']['register_policy'],
'info' => $probe_result['data']['info'],
'admin_name' => $probe_result['data']['admin']['name'],
'admin_profile' => $probe_result['data']['admin']['profile'],
'noscrape_url' => $probe_result['data']['no_scrape_url'] ?? null,
'subscribe_url' => $subscribe ?? null,
'ssl_state' => $probe_result['ssl_state']
]
);
//Add the admin to the directory
if (!empty($probe_result['data']['admin']['profile'])) {
$result = $this->profilePollQueueModel->add($probe_result['data']['admin']['profile']);
$this->logger->debug('Profile queue add URL: ' . $probe_result['data']['admin']['profile'] . ' - ' . $result);
}
$this->discoverPoco($base_url);
} else {
$this->logger->debug('Parse unsuccessful', ['$polled_url' => $polled_url, '$probe_result' => $probe_result]);
}
if ($server) {
//Get the new health.
$version = $parse_success ? $probe_result['data']['version'] : '';
$health_score = $this->computeHealthScore(
$server['health_score'],
$parse_success,
$probe_result['time'],
$version,
$probe_result['ssl_state'],
$avg_ping,
$probe_result['data']['info'] ?? null
);
$this->atlas->perform(
'UPDATE `server` SET `health_score` = :health_score WHERE `id` = :server_id',
['health_score' => $health_score, 'server_id' => $server['id']]
);
}
if ($parse_success) {
$this->logger->info('Server poll successful');
} else {
$this->logger->info('Server poll unsuccessful');
}
return $parse_success ? $server['id'] : 0;
}
/**
* @param string $base_url
* @return float|null
*/
private function getAvgPing(string $base_url)
{
$net_ping = \Net_Ping::factory();
$net_ping->setArgs(['count' => 5]);
$ping_result = $net_ping->ping(parse_url($base_url, PHP_URL_HOST));
if (is_a($ping_result, 'Net_Ping_Result')) {
$avg_ping = $ping_result->getAvg();
} else {
$avg_ping = null;
}
unset($net_ping);
return $avg_ping;
}
private function getProbeResult(string $base_url): array
{
$curl_info = null;
$options = [
'timeout' => max($this->settings['probe_timeout'], 1),
'on_stats' => function (TransferStats $transferStats) use (&$curl_info) {
$curl_info = $transferStats->getHandlerStats();
}
];
$sslcert_issues = false;
try {
//Probe the site.
$probe_start = microtime(true);
$probe_data = $this->http->get($base_url . '/friendica/json', $options)->getBody()->getContents();
$probe_end = microtime(true);
} catch (RequestException $e) {
if (!in_array($e->getHandlerContext()['errno'], [
60, //Could not authenticate certificate with known CA's
83 //Issuer check failed
])) {
throw $e;
}
$sslcert_issues = true;
//When it's the certificate that doesn't work, we probe again without strict SSL.
$options['verify'] = false;
$probe_start = microtime(true);
$probe_data = $this->http->get($base_url . '/friendica/json', $options)->getBody()->getContents();
$probe_end = microtime(true);
}
$time = round(($probe_end - $probe_start) * 1000);
try {
$data = json_decode($probe_data, true);
} catch (\Exception $ex) {
$data = false;
}
$ssl_state = 0;
if (parse_url($base_url, PHP_URL_SCHEME) == 'https') {
if ($sslcert_issues) {
$ssl_state = -1;
} else {
$ssl_state = 1;
}
}
return ['data' => $data, 'time' => $time, 'curl_info' => $curl_info, 'ssl_state' => $ssl_state];
}
private function computeHealthScore(
int $original_health,
bool $probe_success,
?int $time,
?string $version,
?int $ssl_state,
?float $avg_ping,
?string $description
): int
{
//Probe failed, costs you 30 points.
if (!$probe_success) {
return max($original_health - 30, -100);
}
//A good probe gives you 10 points.
$delta = 10;
$max_health = 100;
//Speed scoring.
if (intval($time) > 0) {
//Penalty / bonus points.
if ($time > 800) {
$delta -= 10; //Bad speed.
} elseif ($time > 400) {
$delta -= 5; //Still not good.
} elseif ($time > 250) {
$delta += 0; //This is normal.
} elseif ($time > 120) {
$delta += 5; //Good speed.
} else {
$delta += 10; //Excellent speed.
}
//Cap for bad speeds.
if ($time > 800) {
$max_health = 40;
} elseif ($time > 400) {
$max_health = 60;
}
}
if ($ssl_state == 1) {
$delta += 10;
} elseif ($ssl_state == -1) {
$delta -= 10;
}
//Version check.
if (!empty($version)) {
$versionParts = explode('.', $version);
if (intval($versionParts[0]) == 3) {
$max_health = 30; // Really old version
} else {
$stable_version = $this->simplecache->get('stable_version');
if (!$stable_version) {
$stable_version = trim(file_get_contents('https://git.friendi.ca/friendica/friendica/raw/branch/stable/VERSION'));
$this->simplecache->set('stable_version', $stable_version);
}
$dev_version = $this->simplecache->get('dev_version');
if (!$dev_version) {
$dev_version = trim(file_get_contents('https://git.friendi.ca/friendica/friendica/raw/branch/develop/VERSION'));
$this->simplecache->set('dev_version', $dev_version);
}
$rc_version = str_replace('-dev', '-rc', $dev_version);
if ($version == $dev_version || $version == $rc_version) {
$old_max_health = $max_health;
$new_max_health = 95; //Develop/RC can be unstable
$max_health = min($old_max_health, $new_max_health);
} elseif ($version !== $stable_version) {
$delta = min($delta, 0) - 10; // Losing score as time passes if node isn't updated
}
}
}
// No description available penalty
if (!$description) {
$max_health = min(75, $max_health);
}
// No ping penalty
if (!$avg_ping) {
$max_health -= 5;
}
return max(min($max_health, $original_health + $delta), -100);
}
function discoverPoco($base_url): void
{
$uri = Uri::withQueryValues(new Uri($base_url . '/poco'), ['fields' => 'urls', 'count' => 1000]);
$response = $this->http->request('GET', $uri);
$this->logger->debug('WebRequest: ' . $uri . ' Status: ' . $response->getStatusCode());
if ($response->getStatusCode() != 200) {
$this->logger->info('Unsuccessful poco request: ' . $uri);
return;
}
try {
$pocoFetchData = json_decode($response->getBody()->getContents());
} catch (\Throwable $e) {
$this->logger->notice('Invalid JSON string for PoCo URL: ' . $uri);
return;
}
if (!isset($pocoFetchData->entry)) {
$this->logger->notice('Invalid JSON structure for PoCo URL: ' . $uri);
return;
}
foreach($pocoFetchData->entry as $entry) {
if (empty($entry->urls)) {
continue;
}
foreach ($entry->urls as $url) {
if (!empty($url->type) && !empty($url->value) && $url->type == 'profile') {
$result = $this->profilePollQueueModel->add($url->value);
if ($result === 0) {
$this->logger->info('Discovered profile URL ' . $url->value);
}
}
}
}
}
public function getSubscribeUrl($base_url, $profile)
{
$uri = Uri::withQueryValues(new Uri($base_url . '/xrd'), ['uri' => $profile]);
$response = $this->http->request('GET', $uri, ['headers' => ['Accept' => 'application/jrd+json']]);
$xrdJsonData = $response->getBody()->getContents();
$this->logger->debug('WebRequest: ' . $uri . ' Status: ' . $response->getStatusCode());
if ($response->getStatusCode() != 200) {
$this->logger->info('Unsuccessful XRD request: ' . $uri);
return null;
}
try {
$xrdData = json_decode($xrdJsonData);
} catch (\Throwable $e) {
$this->logger->notice('Invalid JSON string for XRD URL: ' . $uri);
return null;
}
if (!isset($xrdData->links)) {
$this->logger->notice('Invalid JSON structure for XRD URL: ' . $uri);
return null;
}
foreach ($xrdData->links as $link) {
if ($link->rel == 'http://ostatus.org/schema/1.0/subscribe') {
return $link->template ?? null;
}
}
return null;
}
}