Blockbot: New user agents, storage for possible bot user agents

Reviewed-on: friendica/friendica-addons#1496

blockbot/blockbot.php

@@ -13,6 +13,7 @@ use Friendica\DI;
 use Jaybizzle\CrawlerDetect\CrawlerDetect;
 use Friendica\Core\Logger;
 use Friendica\Core\Renderer;
+use Friendica\Core\System;
 use Friendica\Network\HTTPException\ForbiddenException;
 
 require_once __DIR__ . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'autoload.php';
@@ -87,7 +88,11 @@ function blockbot_init_1()
 		'Facebot', 'Googlebot-Video/', 'msnbot/', 'Offline Explorer/', 'YandexNews/', 'msnbot-media/',
 		'EmailWolf', 'Download Demon/', 'FeedFetcher-Google;', 'WebCopier', '+ONB_Bot_Btrix',
 		'scoopit-crawler/', 'ia_archiver', 'Quora-Bot/', 'WebwikiBot/', 'FullStoryBot/',
-		'wpbot/', 'SearchExpress', 'DuckDuckBot/', 'Google Web Preview',
+		'wpbot/', 'SearchExpress', 'DuckDuckBot/', 'Google Web Preview', 'Amazonbot/',
+		'ImagesiftBot;', 'webtech/', 'Bloglines/', 'Netcraft Web Server Survey', 'Spawning-AI',
+		'NLUX_IAHarvester/', 'bots.retroverse.social', 'RSSingBot', 'Chrome-Lighthouse',
+		't3versionsBot/', 'scaninfo@paloaltonetworks.com', 'intelx.io_bot', 'Google-Read-Aloud',
+		'bot Mozilla',
 	];
 
 	if (DI::config()->get('blockbot', 'block_gab')) {
@@ -102,7 +107,7 @@ function blockbot_init_1()
 		'FedditLemmyverseCrawler/', 'lemmy-explorer-crawler/', 'URIports Validator',
 		'rss-is-dead.lol web bot;', 'fedistatsCrawler/', 'W3C_CSS_Validator_JFouffa/',
 		'IABot/', 'Slackbot 1', 'BeeperBot/', 'Matrix-Media-Repo/', 'P3P Validator',
-		'KeybaseBot;',
+		'KeybaseBot;', 'Observatory/', 'CSSCheck/', 'FeedBurner/', 'rss-is-dead.lol feed bot;'
 	];
 
 	if (!DI::config()->get('blockbot', 'good_crawlers')) {
@@ -126,7 +131,8 @@ function blockbot_init_1()
 	
 	// HTTP Libraries
 	$http_libraries = ['ReactorNetty/', 'GuzzleHttp/', 'Embed PHP library', 'python-urllib3/',
-		'EventMachine HttpClient', 'HTMLParser/'
+		'EventMachine HttpClient', 'HTMLParser/', 'node-fetch', 'fasthttp', 'python-httpx/',
+		'Fuzz Faster U Fool', 'gvfs/', 'Embarcadero URI Client/', 'grub-client'
 	];
 
 	if (!DI::config()->get('blockbot', 'http_libraries')) {
@@ -165,6 +171,7 @@ function blockbot_init_1()
 		'camo-rs asset proxy', 'gotosocial/', 'incestoma ', 'SpaceCowboys Android RSS Reader',
 		'NewsBlur Feed Finder', 'Lemmy/', 'enby-town/', 'rss2tg bot;', '; HTTrack ',
 		'MbinBot', 'kbinBot', 'Pixelfed/', 'NewsBlur Feed Fetcher', 'NewsBlur Page Fetcher',
+		'facebookexternalua', 'FreshRSS/', 'BookWyrm/', 'Reeder/',
 	];
 
 	if (blockbot_match($agents)) {
@@ -172,10 +179,28 @@ function blockbot_init_1()
 		return;
 	}
 
+	blockbot_save('blocked-bot', $_SERVER['HTTP_USER_AGENT']);
+
 	logger::notice('Blocked bot', $logdata);
 	throw new ForbiddenException('Bots are not allowed. If you consider this a mistake, create an issue at https://github.com/friendica/friendica');
 }
 
+function blockbot_save($database, $userAgent)
+{
+	if (!function_exists('dba_open')) {
+		return;
+	}
+
+	$ressource = dba_open(System::getTempPath() . '/' . $database, 'cl');
+	$result = dba_fetch($userAgent, $ressource);
+	if ($result === false) {
+		dba_insert($userAgent, 1, $ressource);
+	} else {
+		dba_replace($userAgent, ++$result, $ressource);
+	}
+	dba_close($ressource);
+}
+
 function blockbot_match(array $agents)
 {
 	foreach ($agents as $agent) {

curweather/curweather.php

@@ -9,12 +9,10 @@
  *
  */
 
-use Friendica\App;
 use Friendica\Core\Cache\Enum\Duration;
 use Friendica\Core\Hook;
 use Friendica\Core\Renderer;
 use Friendica\DI;
-use Friendica\Util\Proxy as ProxyUtils;
 
 function curweather_install()
 {
@@ -127,7 +125,7 @@ function curweather_network_mod_init(string &$body)
 		$t = Renderer::getMarkupTemplate("widget.tpl", "addon/curweather/" );
 		$curweather = Renderer::replaceMacros($t, [
 			'$title' => DI::l10n()->t("Current Weather"),
-			'$icon' => ProxyUtils::proxifyUrl('http://openweathermap.org/img/w/'.$res['icon'].'.png'),
+			'$icon' => 'http://openweathermap.org/img/w/'.$res['icon'].'.png',
 			'$city' => $res['city'],
 			'$lon' => $res['lon'],
 			'$lat' => $res['lat'],

forumdirectory/forumdirectory.php

@@ -104,8 +104,8 @@ function forumdirectory_content()
 	$total = 0;
 	$cnt = DBA::fetchFirst("SELECT COUNT(*) AS `total` FROM `profile`
 				INNER JOIN `user` ON `user`.`uid` = `profile`.`uid`
-				WHERE $publish AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND `user`.`page-flags` = ? $sql_extra",
-				User::PAGE_FLAGS_COMMUNITY);
+				WHERE $publish AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND `user`.`page-flags` IN (?, ?) $sql_extra",
+				User::PAGE_FLAGS_COMMUNITY, User::PAGE_FLAGS_COMM_MAN);
 	if (DBA::isResult($cnt)) {
 		$total = $cnt['total'];
 	}
@@ -120,8 +120,8 @@ function forumdirectory_content()
 			`contact`.`addr`, `contact`.`url` FROM `profile`
 			INNER JOIN `user` ON `user`.`uid` = `profile`.`uid`
 			INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid`
-			WHERE $publish AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND `user`.`page-flags` = ? AND `contact`.`self`
-			$sql_extra $order LIMIT $limit", User::PAGE_FLAGS_COMMUNITY
+			WHERE $publish AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND `user`.`page-flags` IN (?, ?) AND `contact`.`self`
+			$sql_extra $order LIMIT $limit", User::PAGE_FLAGS_COMMUNITY, User::PAGE_FLAGS_COMM_MAN
 	);
 
 	if (DBA::isResult($r)) {

groupdirectory/groupdirectory.php

@@ -101,8 +101,8 @@ function groupdirectory_content()
 	$total = 0;
 	$cnt   = DBA::fetchFirst("SELECT COUNT(*) AS `total` FROM `profile`
 				INNER JOIN `user` ON `user`.`uid` = `profile`.`uid`
-				WHERE $publish AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND `user`.`page-flags` = ? $sql_extra",
-		User::PAGE_FLAGS_COMMUNITY);
+				WHERE $publish AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND `user`.`page-flags` IN (?, ?) $sql_extra",
+		User::PAGE_FLAGS_COMMUNITY, User::PAGE_FLAGS_COMM_MAN);
 	if (DBA::isResult($cnt)) {
 		$total = $cnt['total'];
 	}
@@ -117,8 +117,8 @@ function groupdirectory_content()
 			`contact`.`addr`, `contact`.`url` FROM `profile`
 			INNER JOIN `user` ON `user`.`uid` = `profile`.`uid`
 			INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid`
-			WHERE $publish AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND `user`.`page-flags` = ? AND `contact`.`self`
-			$sql_extra $order LIMIT $limit", User::PAGE_FLAGS_COMMUNITY
+			WHERE $publish AND NOT `user`.`blocked` AND NOT `user`.`account_removed` AND `user`.`page-flags` IN (?, ?)  AND `contact`.`self`
+			$sql_extra $order LIMIT $limit", User::PAGE_FLAGS_COMMUNITY, User::PAGE_FLAGS_COMM_MAN
 	);
 
 	if (DBA::isResult($r)) {

mastodoncustomemojis/mastodoncustomemojis.php

@@ -15,7 +15,6 @@ use Friendica\Core\Cache\Enum\Duration;
 use Friendica\Core\Hook;
 use Friendica\Core\Protocol;
 use Friendica\DI;
-use Friendica\Util\Proxy as ProxyUtils;
 
 function mastodoncustomemojis_install()
 {
@@ -88,7 +87,7 @@ function mastodoncustomemojis_fetch_custom_emojis_for_url($api_base_url)
 			foreach ($emojis_array as $emoji) {
 				if (!empty($emoji['shortcode']) && !empty($emoji['static_url'])) {
 					$return['texts'][] = ':' . $emoji['shortcode'] . ':';
-					$return['icons'][] = '<img class="emoji mastodon" src="' . ProxyUtils::proxifyUrl($emoji['static_url']) . '" alt=":' . $emoji['shortcode'] . ':" title=":' . $emoji['shortcode'] . ':"/>';
+					$return['icons'][] = '<img class="emoji mastodon" src="' . $emoji['static_url'] . '" alt=":' . $emoji['shortcode'] . ':" title=":' . $emoji['shortcode'] . ':"/>';
 				}
 			}
 		}