Merge pull request #12615 from MrPetovan/task/12601-update-gserver-block

Drop UpdateGServer worker task if domain is blocked
This commit is contained in:
Michael Vogel 2023-01-05 12:37:00 +01:00 committed by GitHub
commit 2f28ec7af0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 179 additions and 16 deletions

View file

@ -44,7 +44,9 @@ use Friendica\Util\Network;
use Friendica\Util\Strings; use Friendica\Util\Strings;
use Friendica\Util\XML; use Friendica\Util\XML;
use Friendica\Network\HTTPException; use Friendica\Network\HTTPException;
use Friendica\Worker\UpdateGServer;
use GuzzleHttp\Psr7\Uri; use GuzzleHttp\Psr7\Uri;
use Psr\Http\Message\UriInterface;
/** /**
* This class handles GServer related functions * This class handles GServer related functions
@ -99,11 +101,11 @@ class GServer
*/ */
public static function add(string $url, bool $only_nodeinfo = false) public static function add(string $url, bool $only_nodeinfo = false)
{ {
if (self::getID($url, false)) { if (self::getID($url)) {
return; return;
} }
Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $url, $only_nodeinfo); UpdateGServer::add(Worker::PRIORITY_LOW, $url, $only_nodeinfo);
} }
/** /**
@ -191,8 +193,9 @@ class GServer
return false; return false;
} else { } else {
if (strtotime($gserver['next_contact']) < time()) { if (strtotime($gserver['next_contact']) < time()) {
Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false); UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
} }
return self::isDefunct($gserver); return self::isDefunct($gserver);
} }
} }
@ -210,8 +213,9 @@ class GServer
return true; return true;
} else { } else {
if (strtotime($gserver['next_contact']) < time()) { if (strtotime($gserver['next_contact']) < time()) {
Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false); UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
} }
return !$gserver['failed'] && in_array($gserver['network'], Protocol::FEDERATED); return !$gserver['failed'] && in_array($gserver['network'], Protocol::FEDERATED);
} }
} }
@ -252,7 +256,7 @@ class GServer
} }
if (!empty($server) && (empty($gserver) || strtotime($gserver['next_contact']) < time())) { if (!empty($server) && (empty($gserver) || strtotime($gserver['next_contact']) < time())) {
Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $server, false); UpdateGServer::add(Worker::PRIORITY_LOW, $server);
} }
return $reachable; return $reachable;
@ -375,7 +379,7 @@ class GServer
Logger::info('Reset failed status for server', ['url' => $gserver['url']]); Logger::info('Reset failed status for server', ['url' => $gserver['url']]);
if (strtotime($gserver['next_contact']) < time()) { if (strtotime($gserver['next_contact']) < time()) {
Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false); UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
} }
} }
} }
@ -393,7 +397,7 @@ class GServer
Logger::info('Set failed status for server', ['url' => $gserver['url']]); Logger::info('Set failed status for server', ['url' => $gserver['url']]);
if (strtotime($gserver['next_contact']) < time()) { if (strtotime($gserver['next_contact']) < time()) {
Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false); UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url']);
} }
} }
} }
@ -442,18 +446,41 @@ class GServer
* *
* @return string cleaned URL * @return string cleaned URL
* @throws Exception * @throws Exception
* @deprecated since 2023.03 Use cleanUri instead
*/ */
public static function cleanURL(string $dirtyUrl): string public static function cleanURL(string $dirtyUrl): string
{ {
try { try {
$url = str_replace('/index.php', '', trim($dirtyUrl, '/')); return (string)self::cleanUri(new Uri($dirtyUrl));
return (string)(new Uri($url))->withUserInfo('')->withQuery('')->withFragment('');
} catch (\Throwable $e) { } catch (\Throwable $e) {
Logger::warning('Invalid URL', ['dirtyUrl' => $dirtyUrl, 'url' => $url]); Logger::warning('Invalid URL', ['dirtyUrl' => $dirtyUrl]);
return ''; return '';
} }
} }
/**
* Remove unwanted content from the given URI
*
* @param UriInterface $dirtyUri
*
* @return UriInterface cleaned URI
* @throws Exception
*/
public static function cleanUri(UriInterface $dirtyUri): string
{
return $dirtyUri
->withUserInfo('')
->withQuery('')
->withFragment('')
->withPath(
preg_replace(
'#(?:^|/)index\.php#',
'',
rtrim($dirtyUri->getPath(), '/')
)
);
}
/** /**
* Detect server data (type, protocol, version number, ...) * Detect server data (type, protocol, version number, ...)
* The detected data is then updated or inserted in the gserver table. * The detected data is then updated or inserted in the gserver table.

View file

@ -29,6 +29,7 @@ use Friendica\Network\HTTPClient\Client\HttpClientAccept;
use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Network\HTTPClient\Client\HttpClientOptions;
use Friendica\Network\HTTPException\NotModifiedException; use Friendica\Network\HTTPException\NotModifiedException;
use GuzzleHttp\Psr7\Uri; use GuzzleHttp\Psr7\Uri;
use Psr\Http\Message\UriInterface;
class Network class Network
{ {
@ -177,11 +178,28 @@ class Network
* @param string $url The url to check the domain from * @param string $url The url to check the domain from
* *
* @return boolean * @return boolean
*
* @deprecated since 2023.03 Use isUriBlocked instead
*/ */
public static function isUrlBlocked(string $url): bool public static function isUrlBlocked(string $url): bool
{ {
$host = @parse_url($url, PHP_URL_HOST); try {
if (!$host) { return self::isUriBlocked(new Uri($url));
} catch (\Throwable $e) {
Logger::warning('Invalid URL', ['url' => $url]);
return false;
}
}
/**
* Checks if the provided URI domain is on the domain blocklist.
*
* @param UriInterface $uri
* @return boolean
*/
public static function isUriBlocked(UriInterface $uri): bool
{
if (!$uri->getHost()) {
return false; return false;
} }
@ -191,7 +209,7 @@ class Network
} }
foreach ($domain_blocklist as $domain_block) { foreach ($domain_blocklist as $domain_block) {
if (fnmatch(strtolower($domain_block['domain']), strtolower($host))) { if (fnmatch(strtolower($domain_block['domain']), strtolower($uri->getHost()))) {
return true; return true;
} }
} }

View file

@ -25,6 +25,7 @@ use Friendica\Core\Logger;
use Friendica\Core\Worker; use Friendica\Core\Worker;
use Friendica\Model\Contact; use Friendica\Model\Contact;
use Friendica\Network\HTTPException\InternalServerErrorException; use Friendica\Network\HTTPException\InternalServerErrorException;
use Friendica\Util\Network;
class UpdateContact class UpdateContact
{ {
@ -38,6 +39,11 @@ class UpdateContact
*/ */
public static function execute(int $contact_id) public static function execute(int $contact_id)
{ {
// Silently dropping the task if the contact is blocked
if (Contact::isBlocked($contact_id)) {
return;
}
$success = Contact::updateFromProbe($contact_id); $success = Contact::updateFromProbe($contact_id);
Logger::info('Updated from probe', ['id' => $contact_id, 'success' => $success]); Logger::info('Updated from probe', ['id' => $contact_id, 'success' => $success]);
@ -55,6 +61,11 @@ class UpdateContact
throw new \InvalidArgumentException('Invalid value provided for contact_id'); throw new \InvalidArgumentException('Invalid value provided for contact_id');
} }
// Dropping the task if the contact is blocked
if (Contact::isBlocked($contact_id)) {
return 0;
}
return Worker::add($run_parameters, 'UpdateContact', $contact_id); return Worker::add($run_parameters, 'UpdateContact', $contact_id);
} }
} }

View file

@ -22,9 +22,14 @@
namespace Friendica\Worker; namespace Friendica\Worker;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\Database\DBA; use Friendica\Database\DBA;
use Friendica\Model\GServer; use Friendica\Model\GServer;
use Friendica\Network\HTTPException\InternalServerErrorException;
use Friendica\Util\Network;
use Friendica\Util\Strings; use Friendica\Util\Strings;
use GuzzleHttp\Psr7\Uri;
use Psr\Http\Message\UriInterface;
class UpdateGServer class UpdateGServer
{ {
@ -34,8 +39,9 @@ class UpdateGServer
* @param string $server_url Server URL * @param string $server_url Server URL
* @param boolean $only_nodeinfo Only use nodeinfo for server detection * @param boolean $only_nodeinfo Only use nodeinfo for server detection
* @return void * @return void
* @throws \Exception
*/ */
public static function execute(string $server_url, bool $only_nodeinfo = false) public static function execute(string $server_url, bool $only_nodeinfo)
{ {
if (empty($server_url)) { if (empty($server_url)) {
return; return;
@ -47,6 +53,11 @@ class UpdateGServer
return; return;
} }
// Silently dropping the worker task if the server domain is blocked
if (Network::isUrlBlocked($filtered)) {
return;
}
if (($filtered != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) { if (($filtered != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) {
GServer::setFailureByUrl($server_url); GServer::setFailureByUrl($server_url);
return; return;
@ -61,4 +72,23 @@ class UpdateGServer
$ret = GServer::check($filtered, '', true, $only_nodeinfo); $ret = GServer::check($filtered, '', true, $only_nodeinfo);
Logger::info('Updated gserver', ['url' => $filtered, 'result' => $ret]); Logger::info('Updated gserver', ['url' => $filtered, 'result' => $ret]);
} }
/**
* @param array|int $run_parameters Priority constant or array of options described in Worker::add
* @param string $serverUrl
* @param bool $onlyNodeInfo Only use NodeInfo for server detection
* @return int
* @throws InternalServerErrorException
*/
public static function add($run_parameters, string $serverUrl, bool $onlyNodeInfo = false): int
{
// Dropping the worker task if the server domain is blocked
if (Network::isUrlBlocked($serverUrl)) {
return 0;
}
// We have to convert the Uri back to string because worker parameters are saved in JSON format which
// doesn't allow for structured objects.
return Worker::add($run_parameters, 'UpdateGServer', $serverUrl, $onlyNodeInfo);
}
} }

View file

@ -27,6 +27,7 @@ use Friendica\Database\DBA;
use Friendica\DI; use Friendica\DI;
use Friendica\Util\DateTimeFormat; use Friendica\Util\DateTimeFormat;
use Friendica\Util\Strings; use Friendica\Util\Strings;
use GuzzleHttp\Psr7\Uri;
class UpdateGServers class UpdateGServers
{ {
@ -63,12 +64,12 @@ class UpdateGServers
// There are duplicated "url" but not "nurl". So we check both addresses instead of just overwriting them, // There are duplicated "url" but not "nurl". So we check both addresses instead of just overwriting them,
// since that would mean loosing data. // since that would mean loosing data.
if (!empty($gserver['url'])) { if (!empty($gserver['url'])) {
if (Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['url'])) { if (UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['url'])) {
$count++; $count++;
} }
} }
if (!empty($gserver['nurl']) && ($gserver['nurl'] != Strings::normaliseLink($gserver['url']))) { if (!empty($gserver['nurl']) && ($gserver['nurl'] != Strings::normaliseLink($gserver['url']))) {
if (Worker::add(Worker::PRIORITY_LOW, 'UpdateGServer', $gserver['nurl'])) { if (UpdateGServer::add(Worker::PRIORITY_LOW, $gserver['nurl'])) {
$count++; $count++;
} }
} }

View file

@ -0,0 +1,76 @@
<?php
/**
* @copyright Copyright (C) 2010-2023, the Friendica project
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Test\src\Model;
use Friendica\Model\GServer;
use GuzzleHttp\Psr7\Uri;
use Psr\Http\Message\UriInterface;
class GServerTest extends \PHPUnit\Framework\TestCase
{
public function dataCleanUri(): array
{
return [
'full-monty' => [
'expected' => new Uri('https://example.com/path'),
'dirtyUri' => new Uri('https://user:password@example.com/path?query=string#fragment'),
],
'index.php' => [
'expected' => new Uri('https://example.com'),
'dirtyUri' => new Uri('https://example.com/index.php'),
],
'index.php-2' => [
'expected' => new Uri('https://example.com/path/to/resource'),
'dirtyUri' => new Uri('https://example.com/index.php/path/to/resource'),
],
'index.php-path' => [
'expected' => new Uri('https://example.com/path/to'),
'dirtyUri' => new Uri('https://example.com/path/to/index.php'),
],
'index.php-path-2' => [
'expected' => new Uri('https://example.com/path/to/path/to/resource'),
'dirtyUri' => new Uri('https://example.com/path/to/index.php/path/to/resource'),
],
'index.php-slash' => [
'expected' => new Uri('https://example.com'),
'dirtyUri' => new Uri('https://example.com/index.php/'),
],
'index.php-slash-2' => [
'expected' => new Uri('https://example.com/path/to/resource'),
'dirtyUri' => new Uri('https://example.com/index.php/path/to/resource/'),
],
];
}
/**
* @dataProvider dataCleanUri
*
* @param UriInterface $expected
* @param UriInterface $dirtyUri
* @return void
* @throws \Exception
*/
public function testCleanUri(UriInterface $expected, UriInterface $dirtyUri)
{
$this->assertEquals($expected, GServer::cleanUri($dirtyUri));
}
}