Merge pull request 'Blockbot: More agents added' (#1484) from heluecht/friendica-addons:blockbots into 2024.03-rc

Reviewed-on: friendica/friendica-addons#1484

blockbot/blockbot.php

@@ -61,7 +61,7 @@ function blockbot_init_1()
 		'adscanner/', 'HeadlessChrome', 'wpif', 'startmebot/', 'Googlebot/', 'Applebot/',
 		'GoogleImageProxy', 'bingbot/', 'heritrix/', 'ldspider',
 		'AwarioRssBot/', 'TweetmemeBot/', 'dcrawl/', 'PhantomJS/', 'Googlebot-Image/',
-		'CrowdTanglebot/', 'Mediapartners-Google', 'Baiduspider/', 'datagnionbot',
+		'CrowdTanglebot/', 'Mediapartners-Google', 'Baiduspider', 'datagnionbot',
 		'MegaIndex.ru/', 'SMUrlExpander', 'Hatena-Favicon/', 'Wappalyzer', 'FlipboardProxy/',
 		'NetcraftSurveyAgent/', 'Dataprovider.com', 'SMTBot/', 'Nimbostratus-Bot/',
 		'DuckDuckGo-Favicons-Bot/', 'IndieWebCards/', 'proximic', 'netEstate NE Crawler',
@@ -80,7 +80,13 @@ function blockbot_init_1()
 		'YurichevBot/', 'Crawling at Home Project', 'InfoTigerBot/', 'AdIdxBot/',
 		'MicrosoftPreview/', 'masscan/', 'Timpibot/', 'everyfeed-spider/', 'AndroidDownloadManager/',
 		'WebZIP/', 'WDG_Validator/', 'Screaming Frog SEO Spider/', ' Bytespider;', 'ISSCyberRiskCrawler/',
-		'BitSightBot/', 'ev-crawler/',
+		'BitSightBot/', 'ev-crawler/', 'CensysInspect/1.1', 'Protopage/', 'Gaisbot/', 'WellKnownBot/',
+		'SuperBot/', 'Googlebot-Mobile/', 'GPTBot/', 'GenomeCrawlerd/', '2ip bot/', 'Ocarinabot',
+		'Yahoo! Slurp;', 'AdsBot-Google', 'Gregarius/', 'FAST-WebCrawler/', 'Xenu Link Sleuth/',
+		'Ask Jeeves', 'alexa site audit/', 'Yahoo! Slurp China;', 'Microsoft URL Control',
+		'Facebot', 'Googlebot-Video/', 'msnbot/', 'Offline Explorer/', 'YandexNews/', 'msnbot-media/',
+		'EmailWolf', 'Download Demon/', 'FeedFetcher-Google;', 'WebCopier', '+ONB_Bot_Btrix',
+		'scoopit-crawler/'
 	];
 
 	if (DI::config()->get('blockbot', 'block_gab')) {
@@ -92,8 +98,9 @@ function blockbot_init_1()
 		'fediverse.space crawler', 'fediverse.network crawler', 'Active_Pods_CheckBot_3.0',
 		'Social-Relay/', 'Test Certificate Info', 'Uptimebot/', 'GNUSocialBot', 'UptimeRobot/',
 		'PTST/', 'Zabbix', 'Poduptime/', 'FediFetcher', 'lemmy-stats-crawler',
-		'FedditLemmyverseCrawler/', 'kbinBot/', 'lemmy-explorer-crawler/', 'URIports Validator',
-		'rss-is-dead.lol web bot;',
+		'FedditLemmyverseCrawler/', 'lemmy-explorer-crawler/', 'URIports Validator',
+		'rss-is-dead.lol web bot;', 'fedistatsCrawler/', 'W3C_CSS_Validator_JFouffa/',
+		'IABot/', 'Slackbot 1', 'BeeperBot/', 'Matrix-Media-Repo/', 'P3P Validator',
 	];
 
 	if (!DI::config()->get('blockbot', 'good_crawlers')) {
@@ -105,7 +112,9 @@ function blockbot_init_1()
 	// List of agents from social media systems that fetch preview data via opem graph or twitter cards.
 	$socialmedia_agents = ['Twitterbot', 'facebookexternalhit/', 'SkypeUriPreview Preview/',
 		'TelegramBot', 'WhatsApp/', 'github-camo', 'Bluesky Cardyb/', 'XING-contenttabreceiver/', 
-		'LinkedInBot/', 'Instagram ', 'Synapse (bot; ', 'Discordbot/', 'SummalyBot/'];
+		'LinkedInBot/', 'Instagram ', 'Synapse (bot; ', 'Discordbot/', 'SummalyBot/',
+		'Slackbot-LinkExpanding', 'Slack-ImgProxy', 'Iframely/',
+	];
 
 	if (!DI::config()->get('blockbot', 'socialmedia_agents')) {
 		$agents = array_merge($agents, $socialmedia_agents);
@@ -115,7 +124,8 @@ function blockbot_init_1()
 	
 	// HTTP Libraries
 	$http_libraries = ['ReactorNetty/', 'GuzzleHttp/', 'Embed PHP library', 'python-urllib3/',
-		'EventMachine HttpClient'];
+		'EventMachine HttpClient', 'HTMLParser/'
+	];
 
 	if (!DI::config()->get('blockbot', 'http_libraries')) {
 		$agents = array_merge($agents, $http_libraries);
@@ -150,7 +160,9 @@ function blockbot_init_1()
 		'W3C_Validator/', 'NetNewsWire', 'FeedValidator/', 'theoldreader.com', 'axios/',
 		'Paw/', 'PeerTube/', 'fedi.inex.dev', 'FediDB/', 'index.community crawler',
 		'Slackbot-LinkExpanding', 'Firefish/', 'Takahe/', 'Akkoma ', 'Misskey/', 'Lynx/',
-		'camo-rs asset proxy', 'gotosocial/',
+		'camo-rs asset proxy', 'gotosocial/', 'incestoma ', 'SpaceCowboys Android RSS Reader',
+		'NewsBlur Feed Finder', 'Lemmy/', 'enby-town/', 'rss2tg bot;', '; HTTrack ',
+		'MbinBot', 'kbinBot', 'Pixelfed/'
 	];
 
 	if (blockbot_match($agents)) {

bluesky/bluesky.php

@@ -1358,6 +1358,7 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $le
 			break;
 
 		case 'app.bsky.embed.recordWithMedia#view':
+			bluesky_add_media($embed->media, $item, $fetch_uid, $level, $last_poll);
 			$original_uri = $uri = bluesky_get_uri($embed->record->record);
 			$uri = bluesky_fetch_missing_post($uri, $item['uid'], $fetch_uid, Item::PR_FETCHED, $item['contact-id'], $level, $last_poll);
 			if ($uri) {

tumblr/tumblr.php

@@ -677,11 +677,20 @@ function tumblr_send_npf(array $post): bool
 		return true;
 	}
 
-	$post['body'] = Post\Media::addAttachmentsToBody($post['uri-id'], $post['body']);
+	$post['body'] = Post\Media::addAttachmentsToBody($post['uri-id'], $post['body'], [Post\Media::IMAGE, Post\Media::AUDIO, Post\Media::VIDEO, Post\Media::ACTIVITY]);
 	if (!empty($post['title'])) {
 		$post['body'] = '[h1]' . $post['title'] . "[/h1]\n" . $post['body'];
 	}
 
+	if (!empty($post['quote-uri-id'])) {
+		$quote = Post::selectFirstPost(['uri', 'plink'], ['uri-id' => $post['quote-uri-id']]);
+		if (!empty($quote)) {
+			if ((strpos($post['body'], $quote['plink'] ?: $quote['uri']) === false) && (strpos($post['body'], $quote['uri']) === false)) {
+				$post['body'] .= "\n[url]" . ($quote['plink'] ?: $quote['uri']) . "[/url]\n";
+			}
+		}
+	}
+
 	$params = [
 		'content'                => NPF::fromBBCode($post['body'], $post['uri-id']),
 		'state'                  => 'published',

url_replace/url_replace.php

@@ -47,7 +47,7 @@ function url_replace_addon_admin(string &$o)
 {
 	$nitter_server    = DI::config()->get('url_replace', 'nitter_server');
 	$invidious_server = DI::config()->get('url_replace', 'invidious_server');
-	$twelvefeet_sites = implode(PHP_EOL, DI::config()->get('url_replace', 'twelvefeet_sites'));
+	$twelvefeet_sites = implode(PHP_EOL, DI::config()->get('url_replace', 'twelvefeet_sites') ?? [] ?: []);
 
 	$t = Renderer::getMarkupTemplate('admin.tpl', 'addon/url_replace/');
 	$o = Renderer::replaceMacros($t, [
@@ -114,10 +114,7 @@ function url_replace_render(array &$b)
 		}
 	}
 
-	$twelvefeet_sites = DI::config()->get('url_replace', 'twelvefeet_sites');
-	if (empty($twelvefeet_sites)) {
-		$twelvefeet_sites = [];
-	}
+	$twelvefeet_sites = DI::config()->get('url_replace', 'twelvefeet_sites') ?? [] ?: [];
 	foreach ($twelvefeet_sites as $twelvefeet_site) {
 		if (strpos($b['html'], $twelvefeet_site) !== false) {
 			$b['html'] = str_replace($twelvefeet_site, 'https://12ft.io/' . $twelvefeet_site, $b['html']);