forked from friendica/friendica-addons
Blockbot: reworked user agent parser
This commit is contained in:
parent
741c142a05
commit
ce2d9030d1
|
|
@ -2,7 +2,7 @@
|
|||
/**
|
||||
* Name: blockbot
|
||||
* Description: Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.
|
||||
* Version: 0.2
|
||||
* Version: 1.0
|
||||
* Author: Philipp Holzer <admin@philipp.info>
|
||||
* Author: Michael Vogel <https://pirati.ca/profile/heluecht>
|
||||
*
|
||||
|
|
@ -15,6 +15,7 @@ use Friendica\Core\Logger;
|
|||
use Friendica\Core\Renderer;
|
||||
use Friendica\Core\System;
|
||||
use Friendica\Network\HTTPException\ForbiddenException;
|
||||
use Friendica\Util\Network;
|
||||
|
||||
require_once __DIR__ . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'autoload.php';
|
||||
|
||||
|
|
@ -46,14 +47,374 @@ function blockbot_addon_admin_post()
|
|||
DI::config()->set('blockbot', 'training', $_POST['training'] ?? false);
|
||||
}
|
||||
|
||||
function blockbot_reject()
|
||||
{
|
||||
throw new ForbiddenException('Bots are not allowed. If you consider this a mistake, create an issue at https://github.com/friendica/friendica');
|
||||
}
|
||||
|
||||
function blockbot_init_1()
|
||||
{
|
||||
if (empty($_SERVER['HTTP_USER_AGENT'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
$logdata = ['agent' => $_SERVER['HTTP_USER_AGENT'], 'uri' => $_SERVER['REQUEST_URI']];
|
||||
$crawlerDetect = new CrawlerDetect();
|
||||
|
||||
$isCrawler = $crawlerDetect->isCrawler();
|
||||
$cleaned = blockbot_remove_known_parts($_SERVER['HTTP_USER_AGENT']);
|
||||
$logdata = ['isCrawler' => $isCrawler, 'cleaned' => $cleaned, 'agent' => $_SERVER['HTTP_USER_AGENT'], 'uri' => $_SERVER['REQUEST_URI']];
|
||||
|
||||
blockbot_save('all-agents', $_SERVER['HTTP_USER_AGENT'], $cleaned);
|
||||
|
||||
if (empty($cleaned) && !$isCrawler) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (blockbot_is_bot($_SERVER['HTTP_USER_AGENT'])) {
|
||||
blockbot_reject();
|
||||
}
|
||||
|
||||
if (blockbot_is_socialmedia_agents($cleaned, $_SERVER['HTTP_USER_AGENT'])) {
|
||||
if (!DI::config()->get('blockbot', 'socialmedia_agents')) {
|
||||
blockbot_reject();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (blockbot_is_library($cleaned, $_SERVER['HTTP_USER_AGENT'])) {
|
||||
if (!DI::config()->get('blockbot', 'http_libraries')) {
|
||||
blockbot_reject();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (blockbot_is_good_agent($cleaned, $_SERVER['HTTP_USER_AGENT'])) {
|
||||
if (!DI::config()->get('blockbot', 'good_crawlers')) {
|
||||
blockbot_reject();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// This switch here is only meant for developers who want to add more bots to the list above, it is not safe for production.
|
||||
if (!DI::config()->get('blockbot', 'training')) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (blockbot_is_wanted_agent($cleaned, $_SERVER['HTTP_USER_AGENT'])) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!$isCrawler) {
|
||||
if (!empty($cleaned)) {
|
||||
logger::debug('Legacy good user agent detected', $logdata);
|
||||
blockbot_save('legacy-good', $_SERVER['HTTP_USER_AGENT'], $cleaned);
|
||||
}
|
||||
return;
|
||||
} elseif (empty($cleaned)) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger::notice('Blocked bot', $logdata);
|
||||
blockbot_save('blocked-bot', $_SERVER['HTTP_USER_AGENT'], $cleaned);
|
||||
blockbot_reject();
|
||||
}
|
||||
|
||||
function blockbot_save($database, $userAgent, $cleaned)
|
||||
{
|
||||
if (!function_exists('dba_open')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$ressource = dba_open(System::getTempPath() . '/' . $database, 'cl');
|
||||
$result = dba_fetch($userAgent, $ressource);
|
||||
if ($result === false) {
|
||||
dba_insert($userAgent, $cleaned, $ressource);
|
||||
}
|
||||
dba_close($ressource);
|
||||
}
|
||||
|
||||
function blockbot_remove_known_parts($agent)
|
||||
{
|
||||
if (Network::isValidHttpUrl($agent)) {
|
||||
return $agent;
|
||||
}
|
||||
|
||||
$agent = str_replace('))', ')', $agent);
|
||||
$patterns = [
|
||||
"Friendica '.*' \S+; https?://\S+" => 'Friendica',
|
||||
'FriendicaDirectory/\S+ GuzzleHttp/\S+ curl/\S+ PHP/\S+' => 'FriendicaDirectory',
|
||||
'.*\(golang net/http; Activity-Relay \S+; \S+\)' => 'Activity-Relay',
|
||||
'Nextcloud Social \S+' => 'Nextcloud Social',
|
||||
'\S+ - Mobilizon \S+' => 'Mobilizon',
|
||||
'\S+ - Mobilizon' => 'Mobilizon',
|
||||
'GNU social/\S+ \(.*\)' => 'GNU social',
|
||||
'MastodonAndroid/\S+' => 'MastodonAndroid',
|
||||
'Mozilla/5.0 \(compatible; zot\)' => 'zot',
|
||||
'Camo Asset Proxy \S+' => 'Camo Asset Proxy',
|
||||
'AodeRelay \(ap-relay/\S+; \+https?://\S+\)' => 'AodeRelay',
|
||||
'Wget/\S+ \S+ \(Red Hat modified\)' => 'Wget',
|
||||
'PixelFedBot/\S+ \(Pixelfed/\S+; \+https?://\S+\)' => 'Pixelfed',
|
||||
'\(Pixelfed/\S+; \+https?://\S+\)' => 'Pixelfed',
|
||||
'Boost/\S+ \(Linux;Android \S+\)' => 'Boost',
|
||||
'Apache-HttpClient/\S+ \(Java/\S+\)' => 'Apache-HttpClient',
|
||||
'python/federation/\S+' => 'federation',
|
||||
'undefined/undefined \(\+http://a.gup.pe\)' => 'gup.pe',
|
||||
'Poduptime/Production from https?://\S+' => 'Poduptime',
|
||||
'GNUsocialBot \S+ - https?://\S+' => 'GNUsocialBot',
|
||||
'honksnonk/\S+; \S+' => 'honksnonk',
|
||||
];
|
||||
$oldagent = $agent;
|
||||
foreach ($patterns as $pattern => $replacement) {
|
||||
$agent = preg_replace('=' . $pattern . '=i', $replacement, $agent);
|
||||
}
|
||||
|
||||
$patterns = [
|
||||
'Dalvik/\S+ \(Linux; U; Android \S+; [^;]* Build/\S+\)',
|
||||
'Mozilla\S+ Slackware\S+ \([^)]*\)', 'Mozilla/\S+ \([^)]*\)', 'Opera/\S+ \([^)]*\)',
|
||||
'iTunes/\S+ \([^)]*\)', 'Links \([^)]*\)', 'UCWEB/\S+ \([^)]*\)',
|
||||
];
|
||||
do {
|
||||
$oldagent = $agent;
|
||||
foreach ($patterns as $pattern) {
|
||||
$agent = preg_replace('=' . $pattern . '=i', '', $agent);
|
||||
}
|
||||
} while ($agent != $oldagent);
|
||||
|
||||
// Search patterns of known agents
|
||||
$patterns = [
|
||||
'\(KHTML, like Gecko, Foregenix\)',
|
||||
'KHTML/\S+ \(like Gecko\)', '\(KHTML, like Gecko\)',
|
||||
'\(KHTML like Gecko\)', 'like iPhone OS \S+ Mac OS X',
|
||||
'Gecko/\S+', 'Presto/\S+', 'SamsungBrowser/\S+',
|
||||
'KHTML/\S+ \(like Gecko\)', 'Whale/\S+', 'YaBrowser/\S+', 'Yowser/\S+',
|
||||
'Mobile/\S+', 'CriOS/\S+', 'SA/\S+', 'honksnonk/\S+; \S+',
|
||||
'libwww-FM/\S+', 'SSL-MM/\S+', 'Pre/\S+', 'U3/\S+', 'UCBrowser/\S+',
|
||||
'AppleWebKit/\S+', 'Version/\S+', 'Mobile Safari/\S+', 'Safari/\S+', 'Chrome/\S+',
|
||||
'Firefox/\S+', 'Core/\S+', 'QQBrowser/\S+', 'OPR/\S+', 'EdgA/\S+', 'Edge/\S+',
|
||||
'Edg/\S+', 'QtWebEngine/\S+', 'Maxthon/\S+', 'Amethyst/\S+', 'CherryPick/\S+',
|
||||
'Minefield/\S+', 'Waterfox/\S+', 'Iceweasel/\S+', 'Fennec/\S+', 'webOSBrowser/\S+',
|
||||
'Quark/\S+', 'Observatory/\S+', 'SeaMonkey/\S+', 'Chromium/\S+',
|
||||
'Puffin/\S+', 'Kindle/\S+', 'FirePHP/\S+', 'Avant Browser/\S+',
|
||||
'OpenSSL/\S+', 'Lynx/\S+', 'NokiaBrowser/\S+', 'FxiOS/\S+', 'AtContent/\S+',
|
||||
'Minimo/\S+', 'GNUTLS/\S+', 'Python/\S+', 'Mobile Safari [\d.]+',
|
||||
'Vivaldi/\S+', 'Mozilla Firebird/\S+', 'Epiphany\S+', 'WebVideo\S+',
|
||||
'wOSBrowser\S+', 'TouchPad\S+', 'MxBrowser\S+', 'Opera [\d.]'
|
||||
|
||||
];
|
||||
|
||||
do {
|
||||
$oldagent = $agent;
|
||||
foreach ($patterns as $pattern) {
|
||||
$agent = preg_replace('=(^|\W)' . $pattern . '($|\W)=i', '$1$2', $agent);
|
||||
}
|
||||
} while ($agent != $oldagent);
|
||||
$patterns = [
|
||||
'python-httpx/\S+ \((\w+)/\S+; \+https?://\S+\)',
|
||||
'http.rb/\S+ \(\S+/\S+ (\w+)/\S+; \+https?://\S+\) Bot',
|
||||
'http.rb/\S+ \(\S+/\S+ (\w+)/\S+; \+https?://\S+\)',
|
||||
'http.rb/\S+ \((\w+)/\S+; \+https?://\S+\) Bot',
|
||||
'http.rb/\S+ \((\w+)/\S+; \+https?://\S+\)',
|
||||
'http.rb/\S+ \((\w+)/\S+; https?://\S+\)',
|
||||
'http.rb/\S+ \(([\w\s]+)/\S+\)',
|
||||
'python-requests \((\w+)/\S+; \+https?://\S+\)',
|
||||
'python-httpx/\S+ \(\S+; (\w+)/\S+; \+https?://\S+\)',
|
||||
'http.fetch \((\w+); \+https?://\S+ <\S+>; Timeline enrichment\); Bot',
|
||||
'http.fetch \((\w+); \+https?://\S+ <\S+>; Timeline enrichment\)',
|
||||
'(\w+)/\S+ CFNetwork/\S+ Darwin/\S+',
|
||||
'(\w+)/\S+ Android/\S+ OkHttp/\S+',
|
||||
'(\w+)/\S+; https?://\S+; ActivityPub',
|
||||
'(\w+) \(\S+; \+https?://\S+\)',
|
||||
'(\w+) at \S+ \(\+https?://\S+\)',
|
||||
'\S+ \(\+https?://\S+\) (\w+)/\S+ \S+', '\S+ \(\+https?://\S+\) (\w+)/\S+',
|
||||
'(\w+)/\S* \(\+https?://\S+\)', '(\w+) \(\+https?://\S+\)',
|
||||
'(\w+)/\S+ \(https?://\S+\), \w+/\S+ \(https?://\S+\)',
|
||||
'(\w+)/\S+ \(https?://\S+\)', '(\w+) \(https?://\S+\)', '(\w+)/\S* https?://\S+',
|
||||
'(\w+) \S+; https?://\S+ <\S+>, \S+ \S+; https?://\S+ <\S+>; Bot',
|
||||
'(\w+) \S+; https?://\S+ <\S+>; Bot', '(\w+) \S+; https?://\S+ <\S*>',
|
||||
'(\w+)/\S+; \+https?://\S+', '(\w+)/; \+https?://\S+',
|
||||
'(\w+)/[\w._]+ \+https?://\S+',
|
||||
'(\w+)/[\w._]+ \(\S+@\S+\)',
|
||||
'(\w+)/[\w._]+\S*',
|
||||
];
|
||||
do {
|
||||
$oldagent = $agent;
|
||||
foreach ($patterns as $pattern) {
|
||||
$agent = preg_replace('=' . $pattern . '=i', '$1', $agent);
|
||||
}
|
||||
} while ($agent != $oldagent);
|
||||
// 'Pre',
|
||||
// Some more known parts that we can remove
|
||||
//@todo Search for words only
|
||||
$search = [
|
||||
'3gpp-gba', '(Edition Yx)', '(Unsupported)', 'Konqueror (WebEnginePart)',
|
||||
'like Gecko', 'Kubuntu', 'Ubuntu', 'Fedora', 'Debian', 'UNTRUSTED',
|
||||
];
|
||||
do {
|
||||
$oldtext = $agent;
|
||||
$agent = str_ireplace($search, ' ', $agent);
|
||||
} while ($oldtext != $agent);
|
||||
$agent = trim($agent, ',./ ');
|
||||
return trim($agent);
|
||||
}
|
||||
|
||||
function blockbot_is_wanted_agent(string $cleaned, string $agent): bool
|
||||
{
|
||||
$agents = [
|
||||
'Friendica', 'AodeRelay', 'Pixelfed', 'zot', 'Mobilizon', 'federation',
|
||||
'Mastodon', 'Akkoma', 'Pleroma', 'Misskey', 'Firefish', 'WordPress',
|
||||
'Iceshrimp', 'PeerTube', 'FoundKey', 'Calckey', 'gotosocial', 'Wget',
|
||||
'microblogpub', 'Takahe', 'DiasporaFederation', 'MbinBot', 'snac', 'funkwhale',
|
||||
'kbinBot', 'Mbin', 'lotide', 'lemmy-stats-crawler', 'Lemmy', 'MastodonInstances',
|
||||
'lemmy-explorer-crawler', 'MisskeyMediaProxy', 'FedditLemmyverseCrawler',
|
||||
'facebookexternalua', 'ActivityRelay', 'Pachli', 'go-camo', 'OpenGraphReader',
|
||||
'buzzrelay', 'TootDeck-Worker', 'diaspora-connection-tester', 'Fedilab',
|
||||
'ActivityPub', 'camo-rs asset proxy', 'Fedibird', 'FediFetcher', 'FediList Agent',
|
||||
'CSSCheck', 'Tiny Tiny RSS', 'Tusky', 'FediDB', 'Plume', "fediverse's stats",
|
||||
'gup.pe', 'MastodonAndroid', 'Activity-Relay', 'GNU social', 'app.wafrn.net',
|
||||
'enby-town', 'Fediseer', 'Nextcloud Social', 'Ruby, mastodon 0.1.1',
|
||||
'FediCheck.online v1.0', 'Camo Asset Proxy', 'Boost', 'honksnonk',
|
||||
'PieFed', 'Brighteon', 'Mammoth', 'Juick', 'Ruby, mastodon 0.1.1',
|
||||
'Simple Social Network',
|
||||
];
|
||||
if (in_array($cleaned, $agents)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$partial = ['Tiny Tiny RSS'];
|
||||
if (blockbot_match(array_merge($agents, $partial), $cleaned)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$legacy_agents = [
|
||||
'curl', 'zgrab', 'Go-http-client', 'curb', 'github.com', 'reqwest', 'Feedly/',
|
||||
'Python-urllib/', 'Liferea/', 'aiohttp/', 'WordPress.com Reader', 'hackney/',
|
||||
'Faraday v', 'okhttp', 'UniversalFeedParser', 'PixelFedBot', 'python-requests',
|
||||
'WordPress/', 'http.rb/', 'Apache-HttpClient/', 'WordPress.com;', 'Dispatch/',
|
||||
'Ruby', 'Java/', 'libwww-perl/', 'FeedlyApp/', 'lua-resty-http/', 'PostmanRuntime/',
|
||||
'W3C_Validator/', 'NetNewsWire', 'FeedValidator/', 'theoldreader.com', 'axios/',
|
||||
'Paw/', 'fedi.inex.dev', 'index.community crawler', 'Lynx/', 'incestoma ',
|
||||
'SpaceCowboys Android RSS Reader', 'NewsBlur Feed Finder', 'Lemmy/', 'rss2tg bot;',
|
||||
'; HTTrack ', 'NewsBlur Feed Fetcher', 'NewsBlur Page Fetcher', 'FreshRSS/',
|
||||
'BookWyrm/', 'Reeder/',
|
||||
];
|
||||
if (in_array($cleaned, $legacy_agents)) {
|
||||
Logger::debug('Legacy wanted direct match', ['cleaned' => $cleaned, 'agent' => $agent]);
|
||||
blockbot_save('legacy-wanted-direct', $agent, $cleaned);
|
||||
return true;
|
||||
}
|
||||
|
||||
$match = blockbot_match(array_merge($agents, $legacy_agents), $cleaned);
|
||||
if ($match) {
|
||||
Logger::debug('Legacy wanted partial match', ['cleaned' => $cleaned, 'agent' => $agent]);
|
||||
blockbot_save('legacy-wanted-partial', $agent, $cleaned);
|
||||
}
|
||||
return $match;
|
||||
}
|
||||
|
||||
function blockbot_is_good_agent(string $cleaned, string $agent): bool
|
||||
{
|
||||
$agents = [
|
||||
'Zabbix', 'fediblock.manalejandro.com', 'node', 'FeedBurner',
|
||||
'lemmy-stats-crawler', 'Poduptime', 'GNUsocialBot', 'mastofeed.com',
|
||||
'https://wiki.communitydata.science/CommunityData:Fediverse_research',
|
||||
"fediverse's stats", 'link_thumbnailer', 'easy-feed-oven', 'fediverse',
|
||||
];
|
||||
if (in_array($cleaned, $agents)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$partial = ['rest-client'];
|
||||
if (blockbot_match(array_merge($agents, $partial), $cleaned)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$legacy_agents = [
|
||||
'fediverse.space crawler', 'fediverse.network crawler', 'Active_Pods_CheckBot_3.0',
|
||||
'Social-Relay/', 'Test Certificate Info', 'Uptimebot/', 'UptimeRobot/', 'PTST/', 'FediFetcher',
|
||||
'FedditLemmyverseCrawler/', 'lemmy-explorer-crawler/', 'URIports Validator',
|
||||
'rss-is-dead.lol web bot;', 'fedistatsCrawler/', 'W3C_CSS_Validator_JFouffa/',
|
||||
'IABot/', 'Slackbot 1', 'BeeperBot/', 'Matrix-Media-Repo/', 'P3P Validator',
|
||||
'KeybaseBot;', 'Observatory/', 'CSSCheck/', 'rss-is-dead.lol feed bot;',
|
||||
];
|
||||
if (in_array($cleaned, $legacy_agents)) {
|
||||
Logger::debug('Legacy good direct match', ['cleaned' => $cleaned, 'agent' => $agent]);
|
||||
blockbot_save('legacy-good-direct', $agent, $cleaned);
|
||||
return true;
|
||||
}
|
||||
|
||||
$match = blockbot_match(array_merge($agents, $legacy_agents), $cleaned);
|
||||
if ($match) {
|
||||
Logger::debug('Legacy good partial match', ['cleaned' => $cleaned, 'agent' => $agent]);
|
||||
blockbot_save('legacy-good-partial', $agent, $cleaned);
|
||||
}
|
||||
return $match;
|
||||
}
|
||||
|
||||
function blockbot_is_socialmedia_agents(string $cleaned, string $agent): bool
|
||||
{
|
||||
$agents = [
|
||||
'TelegramBot (like TwitterBot)', 'Twitterbot', 'Slack-ImgProxy', 'Slackbot-LinkExpanding',
|
||||
'WhatsApp', 'facebookexternalhit', 'SkypeUriPreview Preview', 'SummalyBot', 'Iframely',
|
||||
'Tumblr', 'Synapse',
|
||||
];
|
||||
if (in_array($cleaned, $agents)) {
|
||||
return true;
|
||||
}
|
||||
$legacy_agents = [
|
||||
'github-camo', 'Bluesky Cardyb/', 'XING-contenttabreceiver/',
|
||||
'LinkedInBot/', 'Instagram ', 'Discordbot/', 'Slack-ImgProxy',
|
||||
];
|
||||
|
||||
if (in_array($cleaned, $legacy_agents)) {
|
||||
Logger::debug('Legacy social media direct match', ['cleaned' => $cleaned, 'agent' => $agent]);
|
||||
blockbot_save('legacy-social-media-direct', $agent, $cleaned);
|
||||
return true;
|
||||
}
|
||||
|
||||
$match = blockbot_match(array_merge($agents, $legacy_agents), $cleaned);
|
||||
if ($match) {
|
||||
Logger::debug('Legacy social media partial match', ['cleaned' => $cleaned, 'agent' => $agent]);
|
||||
blockbot_save('legacy-social-media-partial', $agent, $cleaned);
|
||||
}
|
||||
return $match;
|
||||
}
|
||||
|
||||
function blockbot_is_library(string $cleaned, string $agent): bool
|
||||
{
|
||||
$agents = [
|
||||
'curl', 'Ruby', 'Go-http-client', 'python-httpx', 'undici',
|
||||
'Python-urllib', 'okhttp', 'python-requests', 'python-asks', 'caveman-sieve',
|
||||
'ReactorNetty', 'GuzzleHttp', 'Embed PHP library', 'python-urllib3',
|
||||
'EventMachine HttpClient', 'HTMLParser', 'node-fetch', 'fasthttp',
|
||||
'Fuzz Faster U Fool', 'gvfs', 'Embarcadero URI Client', 'grub-client',
|
||||
'Deno', 'mint', 'axios', 'cutycapt', 'Java', 'Apache-HttpClient',
|
||||
'Crystal', 'com.apple.WebKit.Networking', 'aiohttp', 'Mona',
|
||||
];
|
||||
if (in_array($cleaned, $agents)) {
|
||||
return true;
|
||||
}
|
||||
$legacy_agents = [
|
||||
'ReactorNetty/', 'GuzzleHttp/', 'Embed PHP library', 'python-urllib3/',
|
||||
'EventMachine HttpClient', 'HTMLParser/', 'node-fetch', 'fasthttp', 'python-httpx/',
|
||||
'Fuzz Faster U Fool', 'gvfs/', 'Embarcadero URI Client/', 'grub-client'
|
||||
];
|
||||
|
||||
if (in_array($cleaned, $legacy_agents)) {
|
||||
Logger::debug('Legacy good direct match', ['cleaned' => $cleaned, 'agent' => $agent]);
|
||||
blockbot_save('legacy-good-direct', $agent, $cleaned);
|
||||
return true;
|
||||
}
|
||||
|
||||
$match = blockbot_match(array_merge($agents, $legacy_agents), $cleaned);
|
||||
if ($match) {
|
||||
Logger::debug('Legacy good partial match', ['cleaned' => $cleaned, 'agent' => $agent]);
|
||||
blockbot_save('legacy-good-partial', $agent, $cleaned);
|
||||
}
|
||||
return $match;
|
||||
}
|
||||
|
||||
function blockbot_is_bot(string $agent): bool
|
||||
{
|
||||
// List of known unwanted crawlers.
|
||||
$agents = [
|
||||
'SemrushBot', 's~feedly-nikon3', 'Qwantify/Bleriot/', 'ltx71', 'Sogou web spider/',
|
||||
|
|
@ -65,7 +426,7 @@ function blockbot_init_1()
|
|||
'CrowdTanglebot/', 'Mediapartners-Google', 'Baiduspider', 'datagnionbot',
|
||||
'MegaIndex.ru/', 'SMUrlExpander', 'Hatena-Favicon/', 'Wappalyzer', 'FlipboardProxy/',
|
||||
'NetcraftSurveyAgent/', 'Dataprovider.com', 'SMTBot/', 'Nimbostratus-Bot/',
|
||||
'DuckDuckGo-Favicons-Bot/', 'IndieWebCards/', 'proximic', 'netEstate NE Crawler',
|
||||
'DuckDuckGo/', 'DuckDuckGo-Favicons-Bot/', 'IndieWebCards/', 'proximic', 'netEstate NE Crawler',
|
||||
'AhrefsBot/', 'YandexBot/', 'Exabot/', 'Mediumbot-MetaTagFetcher/',
|
||||
'SurdotlyBot/', 'BingPreview/', 'SabsimBot/', 'CCBot/', 'WbSrch/',
|
||||
'DuckDuckBot-Https/', 'HTTP Banner Detection', 'YandexImages/', 'archive.org_bot',
|
||||
|
|
@ -92,121 +453,21 @@ function blockbot_init_1()
|
|||
'ImagesiftBot;', 'webtech/', 'Bloglines/', 'Netcraft Web Server Survey', 'Spawning-AI',
|
||||
'NLUX_IAHarvester/', 'bots.retroverse.social', 'RSSingBot', 'Chrome-Lighthouse',
|
||||
't3versionsBot/', 'scaninfo@paloaltonetworks.com', 'intelx.io_bot', 'Google-Read-Aloud',
|
||||
'bot Mozilla',
|
||||
'bot Mozilla', 'rayven/', 'com.apple.Safari.SearchHelper',
|
||||
];
|
||||
|
||||
if (DI::config()->get('blockbot', 'block_gab')) {
|
||||
$agents[] = 'GabSocial/';
|
||||
}
|
||||
|
||||
// List of "good" crawlers, mostly from the fediverse.
|
||||
$good_agents = [
|
||||
'fediverse.space crawler', 'fediverse.network crawler', 'Active_Pods_CheckBot_3.0',
|
||||
'Social-Relay/', 'Test Certificate Info', 'Uptimebot/', 'GNUSocialBot', 'UptimeRobot/',
|
||||
'PTST/', 'Zabbix', 'Poduptime/', 'FediFetcher', 'lemmy-stats-crawler',
|
||||
'FedditLemmyverseCrawler/', 'lemmy-explorer-crawler/', 'URIports Validator',
|
||||
'rss-is-dead.lol web bot;', 'fedistatsCrawler/', 'W3C_CSS_Validator_JFouffa/',
|
||||
'IABot/', 'Slackbot 1', 'BeeperBot/', 'Matrix-Media-Repo/', 'P3P Validator',
|
||||
'KeybaseBot;', 'Observatory/', 'CSSCheck/', 'FeedBurner/', 'rss-is-dead.lol feed bot;'
|
||||
];
|
||||
|
||||
if (!DI::config()->get('blockbot', 'good_crawlers')) {
|
||||
$agents = array_merge($agents, $good_agents);
|
||||
} elseif (blockbot_match($good_agents)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// List of agents from social media systems that fetch preview data via opem graph or twitter cards.
|
||||
$socialmedia_agents = ['Twitterbot', 'facebookexternalhit/', 'SkypeUriPreview Preview/',
|
||||
'TelegramBot', 'WhatsApp/', 'github-camo', 'Bluesky Cardyb/', 'XING-contenttabreceiver/',
|
||||
'LinkedInBot/', 'Instagram ', 'Synapse (bot; ', 'Discordbot/', 'SummalyBot/',
|
||||
'Slackbot-LinkExpanding', 'Slack-ImgProxy', 'Iframely/',
|
||||
];
|
||||
|
||||
if (!DI::config()->get('blockbot', 'socialmedia_agents')) {
|
||||
$agents = array_merge($agents, $socialmedia_agents);
|
||||
} elseif (blockbot_match($socialmedia_agents)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// HTTP Libraries
|
||||
$http_libraries = ['ReactorNetty/', 'GuzzleHttp/', 'Embed PHP library', 'python-urllib3/',
|
||||
'EventMachine HttpClient', 'HTMLParser/', 'node-fetch', 'fasthttp', 'python-httpx/',
|
||||
'Fuzz Faster U Fool', 'gvfs/', 'Embarcadero URI Client/', 'grub-client'
|
||||
];
|
||||
|
||||
if (!DI::config()->get('blockbot', 'http_libraries')) {
|
||||
$agents = array_merge($agents, $http_libraries);
|
||||
} elseif (blockbot_match($http_libraries)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (blockbot_match($agents)) {
|
||||
throw new ForbiddenException('Bots are not allowed. If you consider this a mistake, create an issue at https://github.com/friendica/friendica');
|
||||
}
|
||||
|
||||
// This switch here is only meant for developers who want to add more bots to the list above, it is not safe for production.
|
||||
if (!DI::config()->get('blockbot', 'training')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$crawlerDetect = new CrawlerDetect();
|
||||
|
||||
if (!$crawlerDetect->isCrawler()) {
|
||||
logger::debug('Good user agent detected', $logdata);
|
||||
return;
|
||||
}
|
||||
|
||||
// List of known "good" agents, mostly used by Fediverse systems, feed readers, ...
|
||||
$agents = [
|
||||
'curl', 'zgrab', 'Go-http-client', 'curb', 'github.com', 'reqwest', 'Feedly/',
|
||||
'Python-urllib/', 'Liferea/', 'aiohttp/', 'WordPress.com Reader', 'hackney/',
|
||||
'Faraday v', 'okhttp', 'UniversalFeedParser', 'PixelFedBot', 'python-requests',
|
||||
'WordPress/', 'http.rb/', 'Apache-HttpClient/', 'WordPress.com;', 'Pleroma',
|
||||
'Dispatch/', 'Ruby', 'Java/', 'libwww-perl/', 'Mastodon/', 'FeedlyApp/',
|
||||
'lua-resty-http/', 'Tiny Tiny RSS/', 'Wget/', 'PostmanRuntime/',
|
||||
'W3C_Validator/', 'NetNewsWire', 'FeedValidator/', 'theoldreader.com', 'axios/',
|
||||
'Paw/', 'PeerTube/', 'fedi.inex.dev', 'FediDB/', 'index.community crawler',
|
||||
'Slackbot-LinkExpanding', 'Firefish/', 'Takahe/', 'Akkoma ', 'Misskey/', 'Lynx/',
|
||||
'camo-rs asset proxy', 'gotosocial/', 'incestoma ', 'SpaceCowboys Android RSS Reader',
|
||||
'NewsBlur Feed Finder', 'Lemmy/', 'enby-town/', 'rss2tg bot;', '; HTTrack ',
|
||||
'MbinBot', 'kbinBot', 'Pixelfed/', 'NewsBlur Feed Fetcher', 'NewsBlur Page Fetcher',
|
||||
'facebookexternalua', 'FreshRSS/', 'BookWyrm/', 'Reeder/', 'microblogpub/',
|
||||
];
|
||||
|
||||
if (blockbot_match($agents)) {
|
||||
logger::info('False positive', $logdata);
|
||||
return;
|
||||
}
|
||||
|
||||
blockbot_save('blocked-bot', $_SERVER['HTTP_USER_AGENT']);
|
||||
|
||||
logger::notice('Blocked bot', $logdata);
|
||||
throw new ForbiddenException('Bots are not allowed. If you consider this a mistake, create an issue at https://github.com/friendica/friendica');
|
||||
return blockbot_match($agents, $agent);
|
||||
}
|
||||
|
||||
function blockbot_save($database, $userAgent)
|
||||
{
|
||||
if (!function_exists('dba_open')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$ressource = dba_open(System::getTempPath() . '/' . $database, 'cl');
|
||||
$result = dba_fetch($userAgent, $ressource);
|
||||
if ($result === false) {
|
||||
dba_insert($userAgent, 1, $ressource);
|
||||
} else {
|
||||
dba_replace($userAgent, ++$result, $ressource);
|
||||
}
|
||||
dba_close($ressource);
|
||||
}
|
||||
|
||||
function blockbot_match(array $agents)
|
||||
function blockbot_match(array $agents, string $request_agent)
|
||||
{
|
||||
foreach ($agents as $agent) {
|
||||
if (stristr($_SERVER['HTTP_USER_AGENT'], $agent)) {
|
||||
if (stristr($request_agent, $agent)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue