From a574146f042d52e2acbec3bd4ce98ad1eeaf6057 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Wed, 4 Jan 2023 11:38:08 -0500 Subject: [PATCH 1/4] Add UriInterface-enabled cleanUri method in Model\GServer - Tests! --- src/Model/GServer.php | 30 +++++++++++-- tests/src/Model/GServerTest.php | 76 +++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 tests/src/Model/GServerTest.php diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 1878befa9f..9e32c7b0cb 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -45,6 +45,7 @@ use Friendica\Util\Strings; use Friendica\Util\XML; use Friendica\Network\HTTPException; use GuzzleHttp\Psr7\Uri; +use Psr\Http\Message\UriInterface; /** * This class handles GServer related functions @@ -442,18 +443,41 @@ class GServer * * @return string cleaned URL * @throws Exception + * @deprecated since 2023.03 Use cleanUri instead */ public static function cleanURL(string $dirtyUrl): string { try { - $url = str_replace('/index.php', '', trim($dirtyUrl, '/')); - return (string)(new Uri($url))->withUserInfo('')->withQuery('')->withFragment(''); + return (string)self::cleanUri(new Uri($dirtyUrl)); } catch (\Throwable $e) { - Logger::warning('Invalid URL', ['dirtyUrl' => $dirtyUrl, 'url' => $url]); + Logger::warning('Invalid URL', ['dirtyUrl' => $dirtyUrl]); return ''; } } + /** + * Remove unwanted content from the given URI + * + * @param UriInterface $dirtyUri + * + * @return UriInterface cleaned URI + * @throws Exception + */ + public static function cleanUri(UriInterface $dirtyUri): string + { + return $dirtyUri + ->withUserInfo('') + ->withQuery('') + ->withFragment('') + ->withPath( + preg_replace( + '#(?:^|/)index\.php#', + '', + rtrim($dirtyUri->getPath(), '/') + ) + ); + } + /** * Detect server data (type, protocol, version number, ...) * The detected data is then updated or inserted in the gserver table. diff --git a/tests/src/Model/GServerTest.php b/tests/src/Model/GServerTest.php new file mode 100644 index 0000000000..a56f4ed6ff --- /dev/null +++ b/tests/src/Model/GServerTest.php @@ -0,0 +1,76 @@ +. + * + */ + +namespace Friendica\Test\src\Model; + +use Friendica\Model\GServer; +use GuzzleHttp\Psr7\Uri; +use Psr\Http\Message\UriInterface; + +class GServerTest extends \PHPUnit\Framework\TestCase +{ + public function dataCleanUri(): array + { + return [ + 'full-monty' => [ + 'expected' => new Uri('https://example.com/path'), + 'dirtyUri' => new Uri('https://user:password@example.com/path?query=string#fragment'), + ], + 'index.php' => [ + 'expected' => new Uri('https://example.com'), + 'dirtyUri' => new Uri('https://example.com/index.php'), + ], + 'index.php-2' => [ + 'expected' => new Uri('https://example.com/path/to/resource'), + 'dirtyUri' => new Uri('https://example.com/index.php/path/to/resource'), + ], + 'index.php-path' => [ + 'expected' => new Uri('https://example.com/path/to'), + 'dirtyUri' => new Uri('https://example.com/path/to/index.php'), + ], + 'index.php-path-2' => [ + 'expected' => new Uri('https://example.com/path/to/path/to/resource'), + 'dirtyUri' => new Uri('https://example.com/path/to/index.php/path/to/resource'), + ], + 'index.php-slash' => [ + 'expected' => new Uri('https://example.com'), + 'dirtyUri' => new Uri('https://example.com/index.php/'), + ], + 'index.php-slash-2' => [ + 'expected' => new Uri('https://example.com/path/to/resource'), + 'dirtyUri' => new Uri('https://example.com/index.php/path/to/resource/'), + ], + ]; + } + + /** + * @dataProvider dataCleanUri + * + * @param UriInterface $expected + * @param UriInterface $dirtyUri + * @return void + * @throws \Exception + */ + public function testCleanUri(UriInterface $expected, UriInterface $dirtyUri) + { + $this->assertEquals($expected, GServer::cleanUri($dirtyUri)); + } +} From a907d6c87b05c475953e25205e453a77fbf45274 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Wed, 4 Jan 2023 11:38:46 -0500 Subject: [PATCH 2/4] Add UriInterface-enabled isUriBlocked method in Util\Network --- src/Util/Network.php | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/Util/Network.php b/src/Util/Network.php index b7954d8e4d..f9f32d4222 100644 --- a/src/Util/Network.php +++ b/src/Util/Network.php @@ -29,6 +29,7 @@ use Friendica\Network\HTTPClient\Client\HttpClientAccept; use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Network\HTTPException\NotModifiedException; use GuzzleHttp\Psr7\Uri; +use Psr\Http\Message\UriInterface; class Network { @@ -177,11 +178,28 @@ class Network * @param string $url The url to check the domain from * * @return boolean + * + * @deprecated since 2023.03 Use isUriBlocked instead */ public static function isUrlBlocked(string $url): bool { - $host = @parse_url($url, PHP_URL_HOST); - if (!$host) { + try { + return self::isUriBlocked(new Uri($url)); + } catch (\Throwable $e) { + Logger::warning('Invalid URL', ['url' => $url]); + return false; + } + } + + /** + * Checks if the provided URI domain is on the domain blocklist. + * + * @param UriInterface $uri + * @return boolean + */ + public static function isUriBlocked(UriInterface $uri): bool + { + if (!$uri->getHost()) { return false; } @@ -191,7 +209,7 @@ class Network } foreach ($domain_blocklist as $domain_block) { - if (fnmatch(strtolower($domain_block['domain']), strtolower($host))) { + if (fnmatch(strtolower($domain_block['domain']), strtolower($uri->getHost()))) { return true; } } From 1f3c07c06f53877735f5dc8d686120ff8be5c56b Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Wed, 4 Jan 2023 11:39:22 -0500 Subject: [PATCH 3/4] Drop UpdateGServer worker task if domain is blocked --- src/Model/GServer.php | 17 ++++++++++------- src/Worker/UpdateGServer.php | 32 +++++++++++++++++++++++++++++++- src/Worker/UpdateGServers.php | 5 +++-- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 9e32c7b0cb..a3a3d1abbc 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -44,6 +44,7 @@ use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; use Friendica\Network\HTTPException; +use Friendica\Worker\UpdateGServer; use GuzzleHttp\Psr7\Uri; use Psr\Http\Message\UriInterface; @@ -100,11 +101,11 @@ class GServer */ public static function add(string $url, bool $only_nodeinfo = false) { - if (self::getID($url, false)) { + if (self::getID($url)) { return; } - Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $url, $only_nodeinfo); + UpdateGServer::add(Worker::PRIORITY_LOW, $url, $only_nodeinfo); } /** @@ -192,8 +193,9 @@ class GServer return false; } else { if (strtotime($gserver['next_contact']) < time()) { - Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false); + UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']); } + return self::isDefunct($gserver); } } @@ -211,8 +213,9 @@ class GServer return true; } else { if (strtotime($gserver['next_contact']) < time()) { - Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false); + UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']); } + return !$gserver['failed'] && in_array($gserver['network'], Protocol::FEDERATED); } } @@ -253,7 +256,7 @@ class GServer } if (!empty($server) && (empty($gserver) || strtotime($gserver['next_contact']) < time())) { - Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $server, false); + UpdateGServer::add(Worker::PRIORITY_LOW, $server); } return $reachable; @@ -376,7 +379,7 @@ class GServer Logger::info('Reset failed status for server', ['url' => $gserver['url']]); if (strtotime($gserver['next_contact']) < time()) { - Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false); + UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']); } } } @@ -394,7 +397,7 @@ class GServer Logger::info('Set failed status for server', ['url' => $gserver['url']]); if (strtotime($gserver['next_contact']) < time()) { - Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false); + UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']); } } } diff --git a/src/Worker/UpdateGServer.php b/src/Worker/UpdateGServer.php index 9b111fceda..f6f33113f3 100644 --- a/src/Worker/UpdateGServer.php +++ b/src/Worker/UpdateGServer.php @@ -22,9 +22,14 @@ namespace Friendica\Worker; use Friendica\Core\Logger; +use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\Model\GServer; +use Friendica\Network\HTTPException\InternalServerErrorException; +use Friendica\Util\Network; use Friendica\Util\Strings; +use GuzzleHttp\Psr7\Uri; +use Psr\Http\Message\UriInterface; class UpdateGServer { @@ -34,8 +39,9 @@ class UpdateGServer * @param string $server_url Server URL * @param boolean $only_nodeinfo Only use nodeinfo for server detection * @return void + * @throws \Exception */ - public static function execute(string $server_url, bool $only_nodeinfo = false) + public static function execute(string $server_url, bool $only_nodeinfo) { if (empty($server_url)) { return; @@ -47,6 +53,11 @@ class UpdateGServer return; } + // Silently dropping the worker task if the server domain is blocked + if (Network::isUrlBlocked($filtered)) { + return; + } + if (($filtered != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) { GServer::setFailureByUrl($server_url); return; @@ -61,4 +72,23 @@ class UpdateGServer $ret = GServer::check($filtered, '', true, $only_nodeinfo); Logger::info('Updated gserver', ['url' => $filtered, 'result' => $ret]); } + + /** + * @param array|int $run_parameters Priority constant or array of options described in Worker::add + * @param string $serverUrl + * @param bool $onlyNodeInfo Only use NodeInfo for server detection + * @return int + * @throws InternalServerErrorException + */ + public static function add($run_parameters, string $serverUrl, bool $onlyNodeInfo = false): int + { + // Dropping the worker task if the server domain is blocked + if (Network::isUrlBlocked($serverUrl)) { + return 0; + } + + // We have to convert the Uri back to string because worker parameters are saved in JSON format which + // doesn't allow for structured objects. + return Worker::add($run_parameters, 'UpdateGServer', $serverUrl, $onlyNodeInfo); + } } diff --git a/src/Worker/UpdateGServers.php b/src/Worker/UpdateGServers.php index ef76ca9bf5..12f3ff10e1 100644 --- a/src/Worker/UpdateGServers.php +++ b/src/Worker/UpdateGServers.php @@ -27,6 +27,7 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Util\DateTimeFormat; use Friendica\Util\Strings; +use GuzzleHttp\Psr7\Uri; class UpdateGServers { @@ -63,12 +64,12 @@ class UpdateGServers // There are duplicated "url" but not "nurl". So we check both addresses instead of just overwriting them, // since that would mean loosing data. if (!empty($gserver['url'])) { - if (Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'])) { + if (UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url'])) { $count++; } } if (!empty($gserver['nurl']) && ($gserver['nurl'] != Strings::normaliseLink($gserver['url']))) { - if (Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['nurl'])) { + if (UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['nurl'])) { $count++; } } From 647ab1d04ac1dfba46888fed156dd416c7aa5b54 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Wed, 4 Jan 2023 11:42:54 -0500 Subject: [PATCH 4/4] Drop UpdateContact worker task if contact is blocked --- src/Worker/UpdateContact.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Worker/UpdateContact.php b/src/Worker/UpdateContact.php index 633058428d..c2e4432820 100644 --- a/src/Worker/UpdateContact.php +++ b/src/Worker/UpdateContact.php @@ -25,6 +25,7 @@ use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Model\Contact; use Friendica\Network\HTTPException\InternalServerErrorException; +use Friendica\Util\Network; class UpdateContact { @@ -38,6 +39,11 @@ class UpdateContact */ public static function execute(int $contact_id) { + // Silently dropping the task if the contact is blocked + if (Contact::isBlocked($contact_id)) { + return; + } + $success = Contact::updateFromProbe($contact_id); Logger::info('Updated from probe', ['id' => $contact_id, 'success' => $success]); @@ -55,6 +61,11 @@ class UpdateContact throw new \InvalidArgumentException('Invalid value provided for contact_id'); } + // Dropping the task if the contact is blocked + if (Contact::isBlocked($contact_id)) { + return 0; + } + return Worker::add($run_parameters, 'UpdateContact', $contact_id); } }