oldkid/friendica-addons
1
0
Fork 0
forked from friendica/friendica-addons
Code Pull requests Activity

Merge pull request 'Blockbot: Changed loglevels' (#1360) from heluecht/friendica-addons:blockbot into develop

Browse source 
Reviewed-on: friendica/friendica-addons#1360
This commit is contained in:
Hypolite Petovan 2023-03-06 04:27:49 +01:00
parent 2dd57a2f40 8641c30530
commit b46e8ec4b2
1 changed files with 14 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
blockbot/blockbot.php
View file

@ -8,9 +8,7 @@
* *
*/ */
use Friendica\App;
use Friendica\Core\Hook; use Friendica\Core\Hook;
use Friendica\Core\System;
use Friendica\DI; use Friendica\DI;
use Jaybizzle\CrawlerDetect\CrawlerDetect; use Jaybizzle\CrawlerDetect\CrawlerDetect;
use Friendica\Core\Logger; use Friendica\Core\Logger;
@ -52,12 +50,15 @@ function blockbot_init_1()
$logdata = ['agent' => $_SERVER['HTTP_USER_AGENT'], 'uri' => $_SERVER['REQUEST_URI']]; $logdata = ['agent' => $_SERVER['HTTP_USER_AGENT'], 'uri' => $_SERVER['REQUEST_URI']];
// List of "good" crawlers // List of "good" crawlers
$good_agents = ['fediverse.space crawler', 'fediverse.network crawler', 'Active_Pods_CheckBot_3.0', $good_agents = [
'fediverse.space crawler', 'fediverse.network crawler', 'Active_Pods_CheckBot_3.0',
'Social-Relay/', 'Test Certificate Info', 'Uptimebot/', 'GNUSocialBot', 'UptimeRobot/', 'Social-Relay/', 'Test Certificate Info', 'Uptimebot/', 'GNUSocialBot', 'UptimeRobot/',
'PTST/', 'Zabbix', 'Poduptime/']; 'PTST/', 'Zabbix', 'Poduptime/'
];
// List of known crawlers. // List of known crawlers.
$agents = ['SemrushBot', 's~feedly-nikon3', 'Qwantify/Bleriot/', 'ltx71', 'Sogou web spider/', $agents = [
'SemrushBot', 's~feedly-nikon3', 'Qwantify/Bleriot/', 'ltx71', 'Sogou web spider/',
'Diffbot/', 'Twitterbot/', 'YisouSpider', 'evc-batch/', 'LivelapBot/', 'TrendsmapResolver/', 'Diffbot/', 'Twitterbot/', 'YisouSpider', 'evc-batch/', 'LivelapBot/', 'TrendsmapResolver/',
'PaperLiBot/', 'Nuzzel', 'um-LN/', 'Google Favicon', 'Datanyze', 'BLEXBot/', '360Spider', 'PaperLiBot/', 'Nuzzel', 'um-LN/', 'Google Favicon', 'Datanyze', 'BLEXBot/', '360Spider',
'adscanner/', 'HeadlessChrome', 'wpif', 'startmebot/', 'Googlebot/', 'Applebot/', 'adscanner/', 'HeadlessChrome', 'wpif', 'startmebot/', 'Googlebot/', 'Applebot/',
@ -79,7 +80,8 @@ function blockbot_init_1()
'Google-Apps-Script; beanserver;', 'woorankreview/', 'Seekport Crawler;', 'AHC/', 'Google-Apps-Script; beanserver;', 'woorankreview/', 'Seekport Crawler;', 'AHC/',
'SkypeUriPreview Preview/', 'Semanticbot/', 'Embed PHP library', 'XoviOnpageCrawler;', 'SkypeUriPreview Preview/', 'Semanticbot/', 'Embed PHP library', 'XoviOnpageCrawler;',
'GetHPinfo.com-Bot/', 'BoardReader Favicon Fetcher', 'Google-Adwords-Instant', 'newspaper/', 'GetHPinfo.com-Bot/', 'BoardReader Favicon Fetcher', 'Google-Adwords-Instant', 'newspaper/',
'YurichevBot/', 'Crawling at Home Project', 'InfoTigerBot/']; 'YurichevBot/', 'Crawling at Home Project', 'InfoTigerBot/'
];
if (!DI::config()->get('blockbot', 'good_crawlers')) { if (!DI::config()->get('blockbot', 'good_crawlers')) {
$agents = array_merge($agents, $good_agents); $agents = array_merge($agents, $good_agents);
@ -114,7 +116,8 @@ function blockbot_init_1()
} }
// List of false positives' strings of known "good" agents. // List of false positives' strings of known "good" agents.
$agents = ['curl', 'zgrab', 'Go-http-client', 'curb', 'github.com', 'reqwest', 'Feedly/', $agents = [
'curl', 'zgrab', 'Go-http-client', 'curb', 'github.com', 'reqwest', 'Feedly/',
'Python-urllib/', 'Liferea/', 'aiohttp/', 'WordPress.com Reader', 'hackney/', 'Python-urllib/', 'Liferea/', 'aiohttp/', 'WordPress.com Reader', 'hackney/',
'Faraday v', 'okhttp', 'UniversalFeedParser', 'PixelFedBot', 'python-requests', 'Faraday v', 'okhttp', 'UniversalFeedParser', 'PixelFedBot', 'python-requests',
'WordPress/', 'http.rb/', 'Apache-HttpClient/', 'WordPress.com;', 'Pleroma', 'WordPress/', 'http.rb/', 'Apache-HttpClient/', 'WordPress.com;', 'Pleroma',
@ -122,7 +125,8 @@ function blockbot_init_1()
'lua-resty-http/', 'Tiny Tiny RSS/', 'Wget/', 'PostmanRuntime/', 'lua-resty-http/', 'Tiny Tiny RSS/', 'Wget/', 'PostmanRuntime/',
'W3C_Validator/', 'NetNewsWire', 'FeedValidator/', 'theoldreader.com', 'axios/', 'W3C_Validator/', 'NetNewsWire', 'FeedValidator/', 'theoldreader.com', 'axios/',
'Paw/', 'PeerTube/', 'fedi.inex.dev', 'FediDB/', 'index.community crawler', 'Paw/', 'PeerTube/', 'fedi.inex.dev', 'FediDB/', 'index.community crawler',
'Slackbot-LinkExpanding']; 'Slackbot-LinkExpanding'
];
if (DI::config()->get('blockbot', 'good_crawlers')) { if (DI::config()->get('blockbot', 'good_crawlers')) {
$agents = array_merge($agents, $good_agents); $agents = array_merge($agents, $good_agents);
@ -130,11 +134,11 @@ function blockbot_init_1()
foreach ($agents as $agent) { foreach ($agents as $agent) {
if (stristr($_SERVER['HTTP_USER_AGENT'], $agent)) { if (stristr($_SERVER['HTTP_USER_AGENT'], $agent)) {
logger::notice('False positive', $logdata); logger::info('False positive', $logdata);
return; return;
} }
} }
logger::info('Blocked bot', $logdata); logger::notice('Blocked bot', $logdata);
throw new ForbiddenException('Bots are not allowed'); throw new ForbiddenException('Bots are not allowed');
} }