From 7c8c09836eac3ade1e886ceb402dc73849ddd753 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 28 Jul 2019 07:49:30 +0000 Subject: [PATCH 1/4] Admin config setting added for blockbot, added option to block gabsocial --- blockbot/blockbot.php | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/blockbot/blockbot.php b/blockbot/blockbot.php index 3cac5042..2e35f210 100644 --- a/blockbot/blockbot.php +++ b/blockbot/blockbot.php @@ -2,8 +2,9 @@ /** * Name: blockbot * Description: Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header. - * Version: 0.1 + * Version: 0.2 * Author: Philipp Holzer + * Author: Michael Vogel * */ @@ -13,6 +14,8 @@ use Friendica\Core\Hook; use Friendica\Core\System; use Jaybizzle\CrawlerDetect\CrawlerDetect; use Friendica\Core\Logger; +use Friendica\Core\Renderer; +use Friendica\Core\L10n; require_once __DIR__ . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'autoload.php'; @@ -25,6 +28,22 @@ function blockbot_uninstall() { Hook::unregister('init_1', __FILE__, 'blockbot_init_1'); } +function blockbot_addon_admin(&$a, &$o) { + $t = Renderer::getMarkupTemplate("admin.tpl", "addon/blockbot/"); + + $o = Renderer::replaceMacros($t, [ + '$submit' => L10n::t('Save Settings'), + '$training' => ['training', L10n::t('Training mode'), Config::get('blockbot', 'training'), "Activates the training mode. This is only meant for developing purposes. Don't activate this on a production machine."], + '$block_gab' => ['block_gab', L10n::t('Block GabSocial'), Config::get('blockbot', 'block_gab'), 'Block the software GabSocial. This will block every access for that software. You can block dedicated gab instances in the blocklist settings in the admin section.'], + ]); +} + +function blockbot_addon_admin_post(&$a) { + Config::set('blockbot', 'training', $_POST['training'] ?? false); + Config::set('blockbot', 'block_gab', $_POST['block_gab'] ?? false); + info(L10n::t('Settings updated.'). EOL); +} + function blockbot_init_1(App $a) { if (empty($_SERVER['HTTP_USER_AGENT'])) { return; @@ -49,7 +68,11 @@ function blockbot_init_1(App $a) { 'ArchiveTeam ArchiveBot/', 'yacybot', 'https://developers.google.com/+/web/snippet/', 'Scrapy/', 'github-camo', 'MJ12bot/', 'DotBot/', 'Pinterestbot/', 'Jooblebot/', 'Cliqzbot/', 'YaK/', 'Mediatoolkitbot', 'Snacktory', 'FunWebProducts', 'oBot/', - '7Siters/']; + '7Siters/', 'KOCMOHABT', 'Google-SearchByImage']; + + if (Config::get('blockbot', 'block_gab')) { + $agents[] = 'GabSocial/'; + } foreach ($agents as $agent) { if (stristr($_SERVER['HTTP_USER_AGENT'], $agent)) { From 7c29e96170b2f0a95daf5daf8782c3e759cd3b5b Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 28 Jul 2019 08:13:53 +0000 Subject: [PATCH 2/4] Added config for "good" crawlers --- blockbot/blockbot.php | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/blockbot/blockbot.php b/blockbot/blockbot.php index 2e35f210..19f341a6 100644 --- a/blockbot/blockbot.php +++ b/blockbot/blockbot.php @@ -33,14 +33,16 @@ function blockbot_addon_admin(&$a, &$o) { $o = Renderer::replaceMacros($t, [ '$submit' => L10n::t('Save Settings'), - '$training' => ['training', L10n::t('Training mode'), Config::get('blockbot', 'training'), "Activates the training mode. This is only meant for developing purposes. Don't activate this on a production machine."], + '$good_crawlers' => ['good_crawlers', L10n::t('Allow "good" crawlers'), Config::get('blockbot', 'good_crawlers'), "Don't block fediverse crawlers, relay servers and other bots with good purposes."], '$block_gab' => ['block_gab', L10n::t('Block GabSocial'), Config::get('blockbot', 'block_gab'), 'Block the software GabSocial. This will block every access for that software. You can block dedicated gab instances in the blocklist settings in the admin section.'], + '$training' => ['training', L10n::t('Training mode'), Config::get('blockbot', 'training'), "Activates the training mode. This is only meant for developing purposes. Don't activate this on a production machine. This can cut communication with some systems."], ]); } function blockbot_addon_admin_post(&$a) { - Config::set('blockbot', 'training', $_POST['training'] ?? false); + Config::set('blockbot', 'good_crawlers', $_POST['good_crawlers'] ?? false); Config::set('blockbot', 'block_gab', $_POST['block_gab'] ?? false); + Config::set('blockbot', 'training', $_POST['training'] ?? false); info(L10n::t('Settings updated.'). EOL); } @@ -51,6 +53,10 @@ function blockbot_init_1(App $a) { $logdata = ['agent' => $_SERVER['HTTP_USER_AGENT'], 'uri' => $_SERVER['REQUEST_URI']]; + // List of "good" crawlers + $good_agents = ['fediverse.space crawler', 'fediverse.network crawler', 'Active_Pods_CheckBot_3.0', + 'Social-Relay/', 'Test Certificate Info', 'Uptimebot/']; + // List of known crawlers. $agents = ['SemrushBot', 's~feedly-nikon3', 'Qwantify/Bleriot/', 'ltx71', 'Sogou web spider/', 'Diffbot/', 'Twitterbot/', 'YisouSpider', 'evc-batch/', 'LivelapBot/', 'TrendsmapResolver/', @@ -70,6 +76,10 @@ function blockbot_init_1(App $a) { 'Cliqzbot/', 'YaK/', 'Mediatoolkitbot', 'Snacktory', 'FunWebProducts', 'oBot/', '7Siters/', 'KOCMOHABT', 'Google-SearchByImage']; + if (!Config::get('blockbot', 'good_crawlers')) { + $agents = array_merge($agents, $good_agents); + } + if (Config::get('blockbot', 'block_gab')) { $agents[] = 'GabSocial/'; } @@ -93,13 +103,16 @@ function blockbot_init_1(App $a) { } // List of false positives' strings of known "good" agents. - $agents = ['fediverse.network crawler', 'Active_Pods_CheckBot_3.0', 'Social-Relay/', - 'curl', 'zgrab', 'Go-http-client', 'curb', 'github.com', 'reqwest', 'Feedly/', + $agents = ['curl', 'zgrab', 'Go-http-client', 'curb', 'github.com', 'reqwest', 'Feedly/', 'Python-urllib/', 'Liferea/', 'aiohttp/', 'WordPress.com Reader', 'hackney/', 'Faraday v', 'okhttp', 'UniversalFeedParser', 'PixelFedBot', 'python-requests', 'WordPress/', 'http.rb/', 'Apache-HttpClient/', 'WordPress.com;', 'Pleroma', - 'Dispatch/', 'Ruby', 'Uptimebot/', 'Java/', 'libwww-perl/', 'Mastodon/', - 'lua-resty-http/', 'Test Certificate Info']; + 'Dispatch/', 'Ruby', 'Java/', 'libwww-perl/', 'Mastodon/', + 'lua-resty-http/']; + + if (Config::get('blockbot', 'good_crawlers')) { + $agents = array_merge($agents, $good_agents); + } foreach ($agents as $agent) { if (stristr($_SERVER['HTTP_USER_AGENT'], $agent)) { From 6ccddb0051be411f8fbce8fec5abc9685e37c36d Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 28 Jul 2019 08:14:42 +0000 Subject: [PATCH 3/4] Added template --- blockbot/templates/admin.tpl | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 blockbot/templates/admin.tpl diff --git a/blockbot/templates/admin.tpl b/blockbot/templates/admin.tpl new file mode 100644 index 00000000..e6b37743 --- /dev/null +++ b/blockbot/templates/admin.tpl @@ -0,0 +1,4 @@ +{{include file="field_checkbox.tpl" field=$good_crawlers}} +{{include file="field_checkbox.tpl" field=$block_gab}} +{{include file="field_checkbox.tpl" field=$training}} +
From 92837758fd4600ebbe89e415d87ab00d2eba5411 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 28 Jul 2019 17:21:46 +0200 Subject: [PATCH 4/4] Update blockbot/templates/admin.tpl Co-Authored-By: Hypolite Petovan --- blockbot/templates/admin.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blockbot/templates/admin.tpl b/blockbot/templates/admin.tpl index e6b37743..a6de1bd3 100644 --- a/blockbot/templates/admin.tpl +++ b/blockbot/templates/admin.tpl @@ -1,4 +1,4 @@ {{include file="field_checkbox.tpl" field=$good_crawlers}} {{include file="field_checkbox.tpl" field=$block_gab}} {{include file="field_checkbox.tpl" field=$training}} -
+