friendica
/
friendica-addons
7
2
Fork 13
Code Issues 3 Pull Requests 2 Releases 16 Wiki Activity

Added config for "good" crawlers

This commit is contained in:
Michael 2019-07-28 08:13:53 +00:00
parent 7c8c09836e
commit 7c29e96170
1 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
blockbot/blockbot.php
View File

@ -33,14 +33,16 @@ function blockbot_addon_admin(&$a, &$o) {
$o = Renderer::replaceMacros($t, [ $o = Renderer::replaceMacros($t, [
'$submit' => L10n::t('Save Settings'), '$submit' => L10n::t('Save Settings'),
'$training' => ['training', L10n::t('Training mode'), Config::get('blockbot', 'training'), "Activates the training mode. This is only meant for developing purposes. Don't activate this on a production machine."], '$good_crawlers' => ['good_crawlers', L10n::t('Allow "good" crawlers'), Config::get('blockbot', 'good_crawlers'), "Don't block fediverse crawlers, relay servers and other bots with good purposes."],
'$block_gab' => ['block_gab', L10n::t('Block GabSocial'), Config::get('blockbot', 'block_gab'), 'Block the software GabSocial. This will block every access for that software. You can block dedicated gab instances in the blocklist settings in the admin section.'], '$block_gab' => ['block_gab', L10n::t('Block GabSocial'), Config::get('blockbot', 'block_gab'), 'Block the software GabSocial. This will block every access for that software. You can block dedicated gab instances in the blocklist settings in the admin section.'],
'$training' => ['training', L10n::t('Training mode'), Config::get('blockbot', 'training'), "Activates the training mode. This is only meant for developing purposes. Don't activate this on a production machine. This can cut communication with some systems."],
]); ]);
} }
function blockbot_addon_admin_post(&$a) { function blockbot_addon_admin_post(&$a) {
Config::set('blockbot', 'training', $_POST['training'] ?? false); Config::set('blockbot', 'good_crawlers', $_POST['good_crawlers'] ?? false);
Config::set('blockbot', 'block_gab', $_POST['block_gab'] ?? false); Config::set('blockbot', 'block_gab', $_POST['block_gab'] ?? false);
Config::set('blockbot', 'training', $_POST['training'] ?? false);
info(L10n::t('Settings updated.'). EOL); info(L10n::t('Settings updated.'). EOL);
} }
@ -51,6 +53,10 @@ function blockbot_init_1(App $a) {
$logdata = ['agent' => $_SERVER['HTTP_USER_AGENT'], 'uri' => $_SERVER['REQUEST_URI']]; $logdata = ['agent' => $_SERVER['HTTP_USER_AGENT'], 'uri' => $_SERVER['REQUEST_URI']];
// List of "good" crawlers
$good_agents = ['fediverse.space crawler', 'fediverse.network crawler', 'Active_Pods_CheckBot_3.0',
'Social-Relay/', 'Test Certificate Info', 'Uptimebot/'];
// List of known crawlers. // List of known crawlers.
$agents = ['SemrushBot', 's~feedly-nikon3', 'Qwantify/Bleriot/', 'ltx71', 'Sogou web spider/', $agents = ['SemrushBot', 's~feedly-nikon3', 'Qwantify/Bleriot/', 'ltx71', 'Sogou web spider/',
'Diffbot/', 'Twitterbot/', 'YisouSpider', 'evc-batch/', 'LivelapBot/', 'TrendsmapResolver/', 'Diffbot/', 'Twitterbot/', 'YisouSpider', 'evc-batch/', 'LivelapBot/', 'TrendsmapResolver/',
@ -70,6 +76,10 @@ function blockbot_init_1(App $a) {
'Cliqzbot/', 'YaK/', 'Mediatoolkitbot', 'Snacktory', 'FunWebProducts', 'oBot/', 'Cliqzbot/', 'YaK/', 'Mediatoolkitbot', 'Snacktory', 'FunWebProducts', 'oBot/',
'7Siters/', 'KOCMOHABT', 'Google-SearchByImage']; '7Siters/', 'KOCMOHABT', 'Google-SearchByImage'];
if (!Config::get('blockbot', 'good_crawlers')) {
$agents = array_merge($agents, $good_agents);
}
if (Config::get('blockbot', 'block_gab')) { if (Config::get('blockbot', 'block_gab')) {
$agents[] = 'GabSocial/'; $agents[] = 'GabSocial/';
} }
@ -93,13 +103,16 @@ function blockbot_init_1(App $a) {
} }
// List of false positives' strings of known "good" agents. // List of false positives' strings of known "good" agents.
$agents = ['fediverse.network crawler', 'Active_Pods_CheckBot_3.0', 'Social-Relay/', $agents = ['curl', 'zgrab', 'Go-http-client', 'curb', 'github.com', 'reqwest', 'Feedly/',
'curl', 'zgrab', 'Go-http-client', 'curb', 'github.com', 'reqwest', 'Feedly/',
'Python-urllib/', 'Liferea/', 'aiohttp/', 'WordPress.com Reader', 'hackney/', 'Python-urllib/', 'Liferea/', 'aiohttp/', 'WordPress.com Reader', 'hackney/',
'Faraday v', 'okhttp', 'UniversalFeedParser', 'PixelFedBot', 'python-requests', 'Faraday v', 'okhttp', 'UniversalFeedParser', 'PixelFedBot', 'python-requests',
'WordPress/', 'http.rb/', 'Apache-HttpClient/', 'WordPress.com;', 'Pleroma', 'WordPress/', 'http.rb/', 'Apache-HttpClient/', 'WordPress.com;', 'Pleroma',
'Dispatch/', 'Ruby', 'Uptimebot/', 'Java/', 'libwww-perl/', 'Mastodon/', 'Dispatch/', 'Ruby', 'Java/', 'libwww-perl/', 'Mastodon/',
'lua-resty-http/', 'Test Certificate Info']; 'lua-resty-http/'];
if (Config::get('blockbot', 'good_crawlers')) {
$agents = array_merge($agents, $good_agents);
}
foreach ($agents as $agent) { foreach ($agents as $agent) {
if (stristr($_SERVER['HTTP_USER_AGENT'], $agent)) { if (stristr($_SERVER['HTTP_USER_AGENT'], $agent)) {