Art4/friendica-addons
1
0
Fork 0
forked from friendica/friendica-addons
Code Pull requests Activity Actions

[blockbot] Update Composer dependency ahead of release

Browse source 
- Updating jaybizzle/crawler-detect (v1.2.80 => v1.2.116)
This commit is contained in:
Hypolite Petovan 2024-03-19 22:51:17 -04:00
commit d838fc6421
20 changed files with 686 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

44
blockbot/composer.json
View file

@ -1,24 +1,24 @@
{
"name": "friendica-addons/blockbot",
"description": "Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.",
"type": "friendica-addon",
"authors": [
{
"name": "Philipp Holzer",
"email": "admin@philipp.info",
"homepage": "https://friendica.philipp.info/profile/nupplaphil",
"role": "Developer"
}
],
"require": {
"php": ">=5.6.0",
"jaybizzle/crawler-detect": "1.*"
},
"license": "3-clause BSD license",
"minimum-stability": "stable",
"config": {
"optimize-autoloader": true,
"autoloader-suffix": "BlockBotAddon",
"preferred-install": "dist"
}
"name": "friendica-addons/blockbot",
"description": "Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.",
"type": "friendica-addon",
"authors": [
{
"name": "Philipp Holzer",
"email": "admin@philipp.info",
"homepage": "https://friendica.philipp.info/profile/nupplaphil",
"role": "Developer"
}
],
"require": {
"php": ">=5.6.0",
"jaybizzle/crawler-detect": "1.*"
},
"license": "3-clause BSD license",
"minimum-stability": "stable",
"config": {
"optimize-autoloader": true,
"autoloader-suffix": "BlockBotAddon",
"preferred-install": "dist"
}
}

16
blockbot/composer.lock generated
View file

@ -8,24 +8,23 @@
"packages": [
{
"name": "jaybizzle/crawler-detect",
"version": "v1.2.80",
"version": "v1.2.116",
"source": {
"type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git",
"reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847"
"reference": "97e9fe30219e60092e107651abb379a38b342921"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
"reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921",
"reference": "97e9fe30219e60092e107651abb379a38b342921",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "^4.8|^5.5|^6.5",
"satooshi/php-coveralls": "1.*"
"phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
},
"type": "library",
"autoload": {
@ -53,7 +52,7 @@
"crawlerdetect",
"php crawler detect"
],
"time": "2019-04-05T19:52:02+00:00"
"time": "2023-07-21T15:49:49+00:00"
}
],
"packages-dev": [],
@ -65,5 +64,6 @@
"platform": {
"php": ">=5.6.0"
},
"platform-dev": []
"platform-dev": [],
"plugin-api-version": "1.1.0"
}

4
blockbot/vendor/composer/ClassLoader.php vendored
View file

@ -60,7 +60,7 @@ class ClassLoader
public function getPrefixes()
{
if (!empty($this->prefixesPsr0)) {
return call_user_func_array('array_merge', $this->prefixesPsr0);
return call_user_func_array('array_merge', array_values($this->prefixesPsr0));
}
return array();
@ -279,7 +279,7 @@ class ClassLoader
*/
public function setApcuPrefix($apcuPrefix)
{
$this->apcuPrefix = function_exists('apcu_fetch') && ini_get('apc.enabled') ? $apcuPrefix : null;
$this->apcuPrefix = function_exists('apcu_fetch') && filter_var(ini_get('apc.enabled'), FILTER_VALIDATE_BOOLEAN) ? $apcuPrefix : null;
}
/**

3
blockbot/vendor/composer/autoload_real.php vendored
View file

@ -13,6 +13,9 @@ class ComposerAutoloaderInitBlockBotAddon
}
}
/**
* @return \Composer\Autoload\ClassLoader
*/
public static function getLoader()
{
if (null !== self::$loader) {

15
blockbot/vendor/composer/installed.json vendored
View file

@ -1,27 +1,26 @@
[
{
"name": "jaybizzle/crawler-detect",
"version": "v1.2.80",
"version_normalized": "1.2.80.0",
"version": "v1.2.116",
"version_normalized": "1.2.116.0",
"source": {
"type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git",
"reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847"
"reference": "97e9fe30219e60092e107651abb379a38b342921"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
"reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921",
"reference": "97e9fe30219e60092e107651abb379a38b342921",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "^4.8|^5.5|^6.5",
"satooshi/php-coveralls": "1.*"
"phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
},
"time": "2019-04-05T19:52:02+00:00",
"time": "2023-07-21T15:49:49+00:00",
"type": "library",
"installation-source": "dist",
"autoload": {

23
blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml vendored Normal file
View file

@ -0,0 +1,23 @@
name: Check & fix styling
on: [ push ]
jobs:
php-cs-fixer:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
ref: ${{ github.head_ref }}
- name: Run PHP CS Fixer
uses: docker://oskarstark/php-cs-fixer-ga:2.18.6
with:
args: --config=.php_cs.dist --allow-risky=yes
- name: Commit changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: Fix styling

56
blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml vendored Normal file
View file

@ -0,0 +1,56 @@
name: Test
on:
push:
branches:
- "master"
pull_request:
jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
php: [5.3, 5.4, 5.5, 5.6, 7.0, 7.1, 7.2, 7.3, 7.4, 8.0, 8.1, 8.2]
name: PHP:${{ matrix.php }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup PHP, with composer
uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php }}
tools: composer:v2
coverage: xdebug
- name: Get composer cache directory
id: composer-cache
run: |
echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Cache composer dependencies
uses: actions/cache@v3
with:
path: ${{ steps.composer-cache.outputs.dir }}
key: dependencies-php-${{ matrix.php }}-composer-${{ hashFiles('composer.json') }}
restore-keys: dependencies-php-${{ matrix.php }}-composer-
- name: Install Composer dependencies
run: |
composer install --prefer-dist --no-interaction --no-suggest
- name: Run Unit tests
run: |
vendor/bin/phpunit --coverage-clover=tests/logs/clover.xml
- name: Upload coverage results to Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
composer global require php-coveralls/php-coveralls "^1.0"
coveralls --coverage_clover=tests/logs/clover.xml -v

33
blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist vendored Normal file
View file

@ -0,0 +1,33 @@
<?php
$finder = Symfony\Component\Finder\Finder::create()
->in([
__DIR__.'/src',
__DIR__.'/tests',
])
->name('*.php')
->ignoreDotFiles(true)
->ignoreVCS(true);
return PhpCsFixer\Config::create()
->setRules([
'@PSR2' => true,
'array_syntax' => ['syntax' => 'long'],
'ordered_imports' => ['sortAlgorithm' => 'alpha'],
'no_unused_imports' => true,
'not_operator_with_successor_space' => true,
'trailing_comma_in_multiline_array' => true,
'phpdoc_scalar' => true,
'unary_operator_spaces' => true,
'binary_operator_spaces' => true,
'blank_line_before_statement' => [
'statements' => ['break', 'continue', 'declare', 'return', 'throw', 'try'],
],
'phpdoc_single_line_var_spacing' => true,
'phpdoc_var_without_name' => true,
'method_argument_space' => [
'on_multiline' => 'ensure_fully_multiline',
'keep_multiple_spaces_after_comma' => true,
],
])
->setFinder($finder);

2
blockbot/vendor/jaybizzle/crawler-detect/LICENSE vendored
View file

@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2015-2018 Mark Beech
Copyright (c) 2015-2020 Mark Beech
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

30
blockbot/vendor/jaybizzle/crawler-detect/README.md vendored
View file

@ -1,24 +1,23 @@
<p align="center"><a href="http://crawlerdetect.io/" target="_blank"><img src="https://cloud.githubusercontent.com/assets/340752/23082173/1bd1a396-f550-11e6-8aba-4d3c75edea2f.png" width="321" height="219" /></a><br><br>
<a href="http://crawlerdetect.io/" target="_blank">crawlerdetect.io</a>
<p align="center"><a href="https://crawlerdetect.io/" target="_blank"><img src="https://cloud.githubusercontent.com/assets/340752/23082173/1bd1a396-f550-11e6-8aba-4d3c75edea2f.png" width="321" height="219" /></a><br><br>
<a href="https://crawlerdetect.io/" target="_blank">crawlerdetect.io</a>
<br><br>
</p>
<p align="center">
<a href="https://travis-ci.org/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/travis/JayBizzle/Crawler-Detect/master.svg?style=flat-square" /></a>
<a href="https://github.com/JayBizzle/Crawler-Detect/actions"><img alt="GitHub Workflow Status" src="https://img.shields.io/github/actions/workflow/status/JayBizzle/Crawler-Detect/test.yml?branch=master&style=flat-square"></a>
<a href="https://packagist.org/packages/jaybizzle/crawler-detect"><img src="https://img.shields.io/packagist/dm/JayBizzle/Crawler-Detect.svg?style=flat-square" /></a>
<a href="https://scrutinizer-ci.com/g/JayBizzle/Crawler-Detect/?branch=master"><img src="https://img.shields.io/scrutinizer/g/JayBizzle/Crawler-Detect.svg?style=flat-square" /></a>
<a href="https://github.com/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/badge/license-MIT-ff69b4.svg?style=flat-square" /></a>
<a href="https://packagist.org/packages/jaybizzle/crawler-detect"><img src="https://img.shields.io/packagist/v/jaybizzle/Crawler-Detect.svg?style=flat-square" /></a>
<a href="https://styleci.io/repos/32755917"><img src="https://styleci.io/repos/32755917/shield" /></a>
<a href="https://coveralls.io/github/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/coveralls/JayBizzle/Crawler-Detect/master.svg?style=flat-square" /></a>
</p>
## About CrawlerDetect
CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the user agent and http_from header. Currently able to detect 1,000's of bots/spiders/crawlers.
CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the `user agent` and `http_from` header. Currently able to detect 1,000's of bots/spiders/crawlers.
### Installation
Run `composer require jaybizzle/crawler-detect 1.*` or add `"jaybizzle/crawler-detect" :"1.*"` to your `composer.json`.
```
composer require jaybizzle/crawler-detect
```
### Usage
```PHP
@ -46,7 +45,7 @@ If you find a bot/spider/crawler user agent that CrawlerDetect fails to detect,
Failing that, just create an issue with the user agent you have found, and we'll take it from there :)
### Laravel Package
If you would like to use this with Laravel 4/5, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect)
If you would like to use this with Laravel, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect)
### Symfony Bundle
To use this library with Symfony 2/3/4, check out the [CrawlerDetectBundle](https://github.com/nicolasmure/CrawlerDetectBundle).
@ -57,16 +56,21 @@ To use this library with the YII2 framework, check out [yii2-crawler-detect](htt
### ES6 Library
To use this library with NodeJS or any ES6 application based, check out [es6-crawler-detect](https://github.com/JefferyHus/es6-crawler-detect).
### Python Library
To use this library in a Python project, check out [crawlerdetect](https://github.com/moskrc/CrawlerDetect).
### JVM Library (written in Java)
To use this library in a JVM project (including Java, Scala, Kotlin, etc.), check out [CrawlerDetect](https://github.com/nekosoftllc/crawler-detect).
### .NET Library
To use this library in a .net standard (including .net core) based project, check out [NetCrawlerDetect](https://github.com/gplumb/NetCrawlerDetect).
### Nette Extension
To use this library with the Nette framework, checkout [NetteCrawlerDetect](https://github.com/JanGalek/Crawler-Detect).
### Ruby Gem
To use this library with Ruby on Rails or any Ruby-based application, check out [crawler_detect](https://github.com/loadkpi/crawler_detect) gem.
### Go Module
To use this library with Go, check out the [crawlerdetect](https://github.com/x-way/crawlerdetect) module.
_Parts of this class are based on the brilliant [MobileDetect](https://github.com/serbanghita/Mobile-Detect)_
[![Analytics](https://ga-beacon.appspot.com/UA-72430465-1/Crawler-Detect/readme?pixel)](https://github.com/JayBizzle/Crawler-Detect)

3
blockbot/vendor/jaybizzle/crawler-detect/composer.json vendored
View file

@ -16,8 +16,7 @@
"php": ">=5.3.0"
},
"require-dev": {
"phpunit/phpunit": "^4.8|^5.5|^6.5",
"satooshi/php-coveralls": "1.*"
"phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
},
"autoload": {
"psr-4": {

2
blockbot/vendor/jaybizzle/crawler-detect/export.php vendored
View file

@ -37,5 +37,5 @@ function outputJson($object)
function outputTxt($object)
{
$className = (new ReflectionClass($object))->getShortName();
file_put_contents("raw/$className.txt", implode($object->getAll(), PHP_EOL));
file_put_contents("raw/$className.txt", implode(PHP_EOL, $object->getAll()));
}

2
blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json vendored
View file

File diff suppressed because one or more lines are too long

302
blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt vendored
View file

@ -1,27 +1,46 @@
.*Java.*outbrain
YLT
^Aether
^Amazon Simple Notification Service Agent$
^Amazon-Route53-Health-Check-Service
^b0t$
^bluefish
^Calypso v\/
^COMODO DCV
^Corax
^DangDang
^DavClnt
^DHSH
^docker\/[0-9]
^Expanse
^FDM
^git\/
^Goose\/
^Grabber
^Gradle\/
^HTTPClient\/
^HTTPing
^Java\/
^Jeode\/
^Jetty\/
^Mail\/
^Mget
^Microsoft URL Control
^Mikrotik\/
^Netlab360
^NG\/[0-9\.]
^NING\/
^PHP\/[0-9]
^npm\/
^Nuclei
^PHP-AYMAPI\/
^PHP\/
^pip\/
^pnpm\/
^RMA\/
^Ruby|Ruby\/[0-9]
^Swurl
^TLS tester
^twine\/
^ureq
^VSE\/[0-9]
^WordPress\.com
^XRL\/[0-9]
@ -29,17 +48,18 @@
008\/
13TABS
192\.comAgent
2GDPR\/
2ip\.ru
404enemy
7Siters
80legs
a\.pr-cy\.ru
a3logics\.in
A6-Indexer
Abonti
Aboundex
aboutthedomain
Accoona-AI-Agent
acebookexternalhit\/
acoon
acrylicapps\.com\/pulp
Acunetix
@ -49,8 +69,14 @@ AddThis
ADmantX
AdminLabs
adressendeutschland
adreview\/
adscanner
adstxt-worker
Adstxtaggregator
adstxt\.com
Adyen HttpClient
AffiliateLabz\/
affilimate-puppeteer
agentslug
AHC
aihit
@ -71,20 +97,23 @@ AndroidDownloadManager
Anemone
AngleSharp
annotate_google
Ant\.com
Anthill
Anturis Agent
Ant\.com
AnyEvent-HTTP\/
Apache Ant\/
Apache Droid
Apache OpenOffice
Apache-HttpAsyncClient
Apache-HttpClient
ApacheBench
Apexoo
apimon\.de
APIs-Google
AportWorm\/
AppBeat\/
AppEngine-Google
AppStoreScraperZ
AppleSyndication
Aprc\/[0-9]
Arachmo
arachnode
@ -92,21 +121,28 @@ Arachnophilia
aria2
Arukereso
asafaweb
AskQuickly
Asana\/
Ask Jeeves
AskQuickly
ASPSeek
Asterias
Astute
asynchttp
Attach
attohttpc
autocite
AutomaticWPTester
Autonomy
awin\.com
AWS Security Scanner
axios\/
a\.pr-cy\.ru
B-l-i-t-z-B-O-T
Backlink-Ceck
backlink-check
BacklinkHttpStatus
BackStreet
BackupLand
BackWeb
Bad-Neighborhood
Badass
@ -121,6 +157,7 @@ BBBike
BCKLINKS
BDFetch
BegunAdvertising
Bewica-security-scan
Bidtellect
BigBozz
Bigfoot
@ -130,11 +167,14 @@ BingPreview
binlar
biNu image cacher
Bitacle
Bitrix link preview
biz_Directory
BKCTwitterUnshortener\/
Black Hole
Blackboard Safeassign
BlackWidow
BlockNote\.Net
BlogBridge
Bloglines
Bloglovin
BlogPulseLive
@ -142,6 +182,7 @@ BlogSearch
Blogtrottr
BlowFish
boitho\.com-dc
Boost\.Beast
BPImageWalker
Braintree-Webhooks
Branch Metrics API
@ -169,12 +210,15 @@ catexplorador
CC Metadata Scaper
Cegbfeieh
censys
centuryb.o.t9[at]gmail.com
Cerberian Drtrs
CERT\.at-Statistics-Survey
cf-facebook
cg-eye
changedetection
ChangesMeter
Charlotte
chatterino-api-cache
CheckHost
checkprivacy
CherryPicker
@ -185,6 +229,7 @@ Chlooe
Chromaxa
CirrusExplorer
CISPA Vulnerability Notification
CISPA Web Analyser
Citoid
CJNetworkQuality
Clarsentia
@ -192,27 +237,34 @@ clips\.ua\.ac\.be
Cloud mapping
CloudEndure
CloudFlare-AlwaysOnline
Cloudflare-Healthchecks
Cloudinary
cmcm\.com
coccoc
cognitiveseo
ColdFusion
colly -
CommaFeed
Commons-HttpClient
commonscan
contactbigdatafr
contentkingapp
Contextual Code Sites Explorer
convera
CookieReports
copyright sheriff
CopyRightCheck
Copyscape
cortex\/
Cosmos4j\.feedback
Covario-IDS
Craw\/
Crescent
Crowsnest
Criteo
Crowsnest
CSHttp
CSSCheck
Cula\/
curb
Curious George
curl
@ -222,12 +274,17 @@ DAP\/NetHTTP
DareBoost
DatabaseDriverMysqli
DataCha0s
DatadogSynthetics
Datafeedwatch
Datanyze
DataparkSearch
dataprovider
DataXu
Daum(oa)?[ \/][0-9]
dBpoweramp
ddline
deeris
delve\.ai
Demon
DeuSu
developers\.google\.com\/\+\/web\/snippet\/
@ -237,8 +294,8 @@ Digincore
DigitalPebble
Dirbuster
Discourse Forum Onebox
Disqus\/
Dispatch\/
Disqus\/
DittoSpyder
dlvr
DMBrowser
@ -246,6 +303,8 @@ DNSPod-reporting
docoloc
Dolphin http client
DomainAppender
DomainLabz
Domains Project\/
Donuts Content Explorer
dotMailer content retrieval
dotSemantic
@ -258,6 +317,8 @@ drupact
Drupal \(\+http:\/\/drupal\.org\/\)
DTS Agent
dubaiindex
DuplexWeb-Google
DynatraceSynthetic
EARTHCOM
Easy-Thumb
EasyDL
@ -288,20 +349,22 @@ ExactSearch
ExaleadCloudview
Excel\/
exif
ExoRank
Exploratodo
Express WebPictures
Extreme Picture Finder
EyeNetIE
ezooms
facebookexternalhit
facebookexternalua
facebookplatform
fairshare
Faraday v
fasthttp
Faveeo
Favicon downloader
faviconkit
faviconarchive
faviconkit
FavOrg
Feed Wrangler
Feedable\/
@ -313,7 +376,9 @@ FeedBurner
feeder
Feedly
FeedshowOnline
Feedshow\/
Feedspot
FeedViewer\/
Feedwind\/
FeedZcollector
feeltiptop
@ -321,6 +386,8 @@ Fetch API
Fetch\/[0-9]
Fever\/[0-9]
FHscan
Fiery%20Feeds
Filestack
Fimap
findlink
findthatfile
@ -329,18 +396,24 @@ FlipboardBrowserProxy
FlipboardProxy
FlipboardRSS
Flock\/
Florienzh\/
fluffy
Flunky
flynxapp
forensiq
ForusP
FoundSeoTool
http:\/\/www.neomo.de\/
fragFINN\.de
free thumbnails
Freeuploader
FreshRSS
frontman
Funnelback
Fuzz Faster U Fool
G-i-g-a-b-o-t
g00g1e\.net
ganarvisitas
gdnplus\.com
geek-tools
Genieo
GentleSource
@ -352,18 +425,24 @@ GetRight
getroot
GetURLInfo\/
GetWeb
Geziyor
Ghost Inspector
GigablastOpenSource
GIS-LABS
github-camo
GitHub-Hookshot
github\.com
Go [\d\.]* package http
Go http package
Go [\d\.]* package http
Go!Zilla
Go-Ahead-Got-It
Go-http-client
Go!Zilla
go-mtasts\/
gobuster
gobyus
Gofeed
gofetch
Goldfire Server
GomezAgent
gooblog
Goodzer\/
@ -376,24 +455,31 @@ Google Page Speed Insights
Google PP Default
Google Search Console
Google Web Preview
Google-Ads-Creatives-Assistant
Google-Ads-Overview
Google-Adwords
Google-Apps-Script
Google-Calendar-Importer
Google-HotelAdsVerifier
Google-HTTP-Java-Client
Google-InspectionTool
Google-Podcast
Google-Publisher-Plugin
Google-Read-Aloud
Google-SearchByImage
Google-Site-Verification
Google-SMTP-STS
Google-speakr
Google-Structured-Data-Testing-Tool
Google-Youtube-Links
Google-Transparency-Report
google-xrawler
Google-Youtube-Links
GoogleDocs
GoogleHC\/
GoogleProber
GoogleProducer
GoogleSites
Google-Transparency-Report
Gookey
GoScraper
GoSpotCheck
gosquared-thumbnailer
Gotit
@ -419,13 +505,16 @@ Haansoft
hackney\/
Hadi Agent
HappyApps-WebCheck
Hardenize
Hatena
Havij
HaxerMen
HeadlessChrome
HEADMasterSEO
HeartRails_Capture
help@dataminr\.com
heritrix
Hexometer
historious
hkedcity
hledejLevne\.cz
@ -442,28 +531,31 @@ htdig
HTMLparser
htmlyse
HTTP Banner Detection
HTTP_Compression_Test
http_request2
http_requester
http-get
HTTP-Header-Abfrage
http-kit
http-request\/
HTTP-Tiny
HTTP::Lite
http\.rb\/
http_get
http:\/\/www.neomo.de\/
HttpComponents
httphr
HTTPie
HTTPMon
httpRequest
httpscheck
httpssites_power
httpunit
HttpUrlConnection
http\.rb\/
HTTP_Compression_Test
http_get
http_request2
http_requester
httrack
huaweisymantec
HubSpot
HubSpot-Link-Resolver
Humanlinks
i2kconnect\/
Iblog
@ -474,6 +566,7 @@ IDG Twitter Links Resolver
IDwhois\/
Iframely
igdeSpyder
iGooglePortal
IlTrovatore
Image Fetch
Image Sucker
@ -498,13 +591,15 @@ Integrity
integromedb
Intelliseek
InterGET
internet_archive
Internet Ninja
InternetSeer
internetVista monitor
internetwache
internet_archive
intraVnews
IODC
IOI
Inboxb0t
iplabel
ips-agent
IPS\/[0-9]
@ -516,10 +611,13 @@ isitup\.org
iskanie
isUp\.li
iThemes Sync\/
IZaBEE
iZSearch
JAHHO
janforman
Jaunt\/
Java.*outbrain
javelin\.io
Jbrofuzz
Jersey\/
JetCar
@ -535,6 +633,7 @@ JolokiaPwn
Joomla
Jorgee
JS-Kit
JungleKeyThumbnail
JustView
Kaspersky Lab CFR link resolver
Kelny\/
@ -542,6 +641,7 @@ Kerrigan\/
KeyCDN
Keyword Density
Keywords Research
khttp\/
KickFire
KimonoLabs\/
Kml-Google
@ -549,11 +649,12 @@ knows\.is
KOCMOHABT
kouio
kube-probe
kubectl
kulturarw3
KumKie
L\.webis
Larbin
Lavf\/
leakix\.net
LeechFTP
LeechGet
letsencrypt
@ -564,41 +665,50 @@ Libwhisker
libwww
Licorne
Liferea\/
Lightspeedsystems
Lighthouse
Lightspeedsystems
Likse
limber\.io
Link Valet
link_thumbnailer
LinkAlarm\/
LinkAnalyser
linkCheck
linkdex
LinkExaminer
linkfluence
linkpeek
LinkPreviewGenerator
LinkPreview
LinkScan
LinksManager
LinkTiger
LinkWalker
link_thumbnailer
Lipperhey
Litemage_walker
livedoor ScreenShot
LoadImpactRload
localsearch-web
LongURL API
longurl-r-package
looid\.com
looksystems\.net
ltx71
lua-resty-http
Lucee \(CFML Engine\)
Lush Http Client
lwp-request
lwp-trivial
LWP::Simple
lycos
LYT\.SR
L\.webis
mabontland
MacOutlook\/
Mag-Net
MagpieRSS
Mail\.Ru
Mail::STS
MailChimp
Mail\.Ru
Majestic12
makecontact\/
Mandrill
@ -609,6 +719,8 @@ MarkWatch
Mass Downloader
masscan\/
Mata Hari
mattermost
Mediametric
Mediapartners-Google
mediawords
MegaIndex\.ru
@ -619,28 +731,31 @@ Metaspinner
MetaURI
MFC_Tear_Sample
Microsearch
Microsoft Office
Microsoft Data Access
Microsoft Office
Microsoft Outlook
Microsoft Windows Network Diagnostics
Microsoft-WebDAV-MiniRedir
Microsoft Data Access
Microsoft\.Data\.Mashup
MIDown tool
MIIxpc
Mindjet
Miniature\.io
Miniflux
mio_httpc
Miro-HttpClient
Mister PiX
mixdata dot com
mixed-content-scan
Mixmax-LinkPreview
mixnode
Mnogosearch
mogimogi
Mojeek
Mojolicious \(Perl\)
Monit\/
Mollie
monitis
Monitority\/
Monit\/
montastic
MonTools
Moreover
@ -649,6 +764,7 @@ Morning Paper
MovableType
mowser
Mrcgiguy
Mr\.4x3 Powered
MS Web Services Client Protocol
MSFrontPage
mShots
@ -656,10 +772,13 @@ MuckRack\/
muhstik-scan
MVAClient
MxToolbox\/
myseosnapshot
nagios
Najdi\.si
Name Intelligence
NameFo\.com
Nameprotect
nationalarchives
Navroad
NearSite
Needle
@ -674,6 +793,8 @@ Netpursual
netresearch
NetShelter ContentScan
Netsparker
NetSystemsResearch
nettle
NetTrack
Netvibes
NetZIP
@ -686,22 +807,26 @@ newsme
newspaper\/
Nexgate Ruby Client
NG-Search
nghttp2
Nibbler
NICErsPRO
NihilScio
Nikto
nineconnections
NLNZ_IAHarvester
Nmap Scripting Engine
node-fetch
node-superagent
node-urllib
node\.io
Nodemeter
NodePing
node\.io
nominet\.org\.uk
nominet\.uk
Norton-Safeweb
Notifixious
notifyninja
NotionEmbedder
nuhk
nutch
Nuzzel
@ -709,39 +834,45 @@ nWormFeedFinder
nyawc\/
Nymesis
NYU
Observatory\/
Ocelli\/
Octopus
oegp
Offline Explorer
Offline Navigator
og-scraper
OgScrper
okhttp
omgili
OMSC
Online Domain Tools
Open Source RSS
OpenCalaisSemanticProxy
Openfind
OpenLinkProfiler
Openstat\/
OpenVAS
OPPO A33
Optimizer
Orbiter
OrgProbe\/
orion-semantics
Outlook-Express
Outlook-iOS
ow\.ly
Owler
Owlin
ownCloud News
ow\.ly
OxfordCloudService
Page Valet
page_verifier
page scorer
Page Valet
page2rss
PageFreezer
PageGrabber
PagePeeker
PageScorer
Pagespeed\/
PageThing
page_verifier
Panopta
panscient
Papa Foto
@ -750,6 +881,7 @@ Pavuk
PayPal IPN
pcBrowser
Pcore-HTTP
PDF24 URL To PDF
Pearltrees
PECL::HTTP
peerindex
@ -759,44 +891,52 @@ Perlu -
PhantomJS Screenshoter
PhantomJS\/
Photon\/
php-requests
phpservermon
Pi-Monster
Picscout
Picsearch
PictureFinder
Pimonster
ping\.blo\.gs
Pingability
PingAdmin\.Ru
Pingdom
Pingoscope
PingSpot
ping\.blo\.gs
pinterest\.com
Pixray
Pizilla
Plagger\/
Pleroma
Ploetz \+ Zeller
Plukkie
plumanalytics
PocketImageCache
PocketParser
Pockey
PodcastAddict\/
POE-Component-Client-HTTP
Polymail\/
Pompos
Porkbun
Port Monitor
postano
postfix-mta-sts-resolver
PostmanRuntime
postplanner\.com
PostPost
postrank
PowerPoint\/
Prebid
Prerender
Priceonomics Analysis Engine
PrintFriendly
PritTorrent
Prlog
probethenet
Project 25499
Project ?25499
Project-Resonance
prospectb2b
Protopage
ProWebWalker
@ -805,9 +945,9 @@ PRTG Network Monitor
pshtt, https scanning
PTST
PTST\/[0-9]+
Pulsepoint XT3 web scraper
Pump
Python-httplib2
python-httpx
python-requests
Python-urllib
Qirina Hurdler
@ -817,12 +957,17 @@ Qseero
Qualidator
QueryN Metasearch
queuedriver
quic-go-HTTP\/
QuiteRSS
Quora Link Preview
Qwantify
Radian6
RadioPublicImageResizer
Railgun\/
RankActive
RankFlex
RankSonicSiteAuditor
RapidLoad\/
Re-re Studio
ReactorNetty
Readability
@ -833,6 +978,7 @@ Recorder
RecurPost\/
redback\/
ReederForMac
Reeder\/
ReGet
RepoMonkey
request\.js
@ -845,38 +991,44 @@ Robosourcer
Robozilla
ROI Hunter
RPT-HTTPClient
RSSMix\/
RSSOwl
RyowlEngine
safe-agent-scanner
SalesIntelligent
Saleslift
Sendsay\.Ru
SAP NetWeaver Application Server
SauceNAO
SBIder
sc-downloader
scalaj-http
scan\.lol
Scamadviser-Frontend
ScanAlert
scan\.lol
Scoop
scooter
ScopeContentAG-HTTP-Client
ScoutJet
ScoutURLMonitor
ScrapeBox Page Scanner
SimpleScraper
Scrapy
Screaming
ScreenShotService
Scrubby
Scrutiny\/
search\.thunderstone
Search37
searchenginepromotionhelp
Searchestate
SearchExpress
SearchSight
SearchWP
search\.thunderstone
Seeker
semanticdiscovery
semanticjuice
Semiocast HTTP client
Semrush
Sendsay\.Ru
sentry\/
SEO Browser
Seo Servis
@ -885,11 +1037,13 @@ seo4ajax
Seobility
SEOCentro
SeoCheck
seocompany
SEOkicks
SEOlizer
Seomoz
SEOprofiler
SEOsearch
seoscanners
SEOsearch
seositecheckup
SEOstats
servernfo
@ -900,14 +1054,16 @@ Shodan
Shoppimon
ShopWiki
ShortLinkTranslate
shortURL lengthener
shrinktheweb
Sideqik
Siege
SimplePie
SimplyFast
Siphon
SISTRIX
Site-Shot\/
Site Sucker
Site-Shot\/
Site24x7
SiteBar
Sitebeam
@ -928,6 +1084,7 @@ Sitevigil
sitexy\.com
SkypeUriPreview
Slack\/
sli-systems\.com
slider\.com
slurp
SlySearch
@ -950,9 +1107,11 @@ SpaceBison
SpamExperts
Spammen
Spanner
Spawning-AI
spaziodati
SPDYCheck
Specificfeeds
SpeedKit
speedy
SPEng
Spinn3r
@ -966,9 +1125,11 @@ SSL Labs
ssl-tools
StackRambler
Statastico\/
Statically-
StatusCake
Steeler
Stratagems Kumo
Stripe\/
Stroke\.cz
StudioFACA
StumbleUpon
@ -978,9 +1139,11 @@ summify
SuperHTTP
Surphace Scout
Suzuran
SwiteScraper
swcd
Symfony BrowserKit
Symfony2 BrowserKit
Synapse\/
Syndirella\/
SynHttpClient-Built
Sysomos
sysscan
@ -990,6 +1153,8 @@ tAkeOut
Tarantula\/
Taringa UGC
TarmotGezgin
tchelebi\.io
techiaith\.cymru
Teleport
Telesoft
Telesphoreo
@ -1000,13 +1165,15 @@ terrainformatica
Test Certificate Info
testuri
Tetrahedron
TextRazor Downloader
The Drop Reaper
The Expert HTML Source Viewer
The Knowledge AI
The Intraformant
The Knowledge AI
theinternetrules
TheNomad
Thinklab
Thumbor
Thumbshots
ThumbSniper
timewe\.net
@ -1040,9 +1207,11 @@ Typhoeus
ubermetrics-technologies
uclassify
UdmSearch
ultimate_sitemap_parser
unchaos
unirest-java
UniversalFeedParser
unshortenit
Unshorten\.It
Untiny
UnwindFetchor
@ -1050,8 +1219,8 @@ updated
updown\.io daemon
Upflow
Uptimia
Urlcheckr
URL Verifier
Urlcheckr
URLitor
urlresolver
Urlstat
@ -1059,15 +1228,16 @@ URLTester
UrlTrends Ranking Updater
URLy Warning
URLy\.Warning
URL\/Emacs
Vacuum
Vagabondo
VB Project
vBSEO
VCI
via ggpht\.com GoogleImageProxy
VidibleScraper
Virusdie
visionutils
Visual Rights Group
vkShare
VoidEYE
Voil
@ -1078,19 +1248,18 @@ VSB-TUO\/
Vulnbusters Meter
VYU2
w3af\.org
W3C_Unicorn
W3C-checklink
W3C-mobileOK
W3C_Unicorn
WAC-OFU
Wallpapers\/[0-9]+
WakeletLinkExpander
WallpapersHD
Wallpapers\/[0-9]+
wangling
Wappalyzer
WatchMouse
WbSrch\/
WDT\.io
web-capture\.net
Web-sniffer
Web Auto
Web Collage
Web Enhancer
@ -1098,7 +1267,10 @@ Web Fetch
Web Fuck
Web Pix
Web Sauger
Web spyder
Web Sucker
web-capture\.net
Web-sniffer
Webalta
Webauskunft
WebAuto
@ -1136,8 +1308,9 @@ WebSniffer
Webster
WebStripper
WebSucker
Webthumb\/
webtech\/
WebThumbnail
Webthumb\/
WebWhacker
WebZIP
WeLikeLinks
@ -1146,24 +1319,31 @@ WeSEE
wf84
Wfuzz\/
wget
WhatCMS
WhatsApp
WhatsMyIP
WhatWeb
WhereGoes\?
Whibse
WhoAPI\/
WhoRunsCoinHive
Whynder Magnet
Windows-RSS-Platform
WinHttp-Autoproxy-Service
WinHTTP\/
WinPodder
wkhtmlto
wmtips
Woko
Wolfram HTTPClient
woorankreview
Word\/
WordPress\/
WordupinfoSearch
Word\/
worldping-api
wotbox
WP Engine Install Performance API
WP Rocket
wpif
wprecon\.com survey
WPScan
@ -1173,20 +1353,21 @@ WWW-Collector-E
WWW-Mechanize
WWW::Document
WWW::Mechanize
www\.monitor\.us
WWWOFFLE
www\.monitor\.us
x09Mozilla
x22Mozilla
XaxisSemanticsClassifier
XenForo\/
Xenu Link Sleuth
XING-contenttabreceiver
xpymep([0-9]?)\.exe
Y!J-(ASR|BSC)
Y\!J-BRW
Y!J-[A-Z][A-Z][A-Z]
Yaanb
yacy
Yahoo Link Preview
YahooCacheSystem
YahooMailProxy
YahooYSMcm
YandeG
Yandex(?!Search)
@ -1194,6 +1375,7 @@ yanga
yeti
Yo-yo
Yoleo Consumer
yomins\.com
yoogliFetchAgent
YottaaMonitor
Your-Website-Sucks
@ -1205,13 +1387,15 @@ Zade
Zao
Zauba
Zemanta Aggregator
Zend_Http_Client
Zend\\Http\\Client
Zend_Http_Client
Zermelo
Zeus
zgrab
ZnajdzFoto
ZnHTTP
Zombie\.js
Zoom\.Mac
ZoteroTranslationServer
ZyBorg
[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)
[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)

2
blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json vendored
View file

@ -1 +1 @@
["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT",";"]
["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"]

4
blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt vendored
View file

@ -45,4 +45,6 @@ cubot
; IDbot
; ID bot
; POWER BOT
;
OCTOPUS-CORE
htc_botdugls
super\/\d+\/Android\/\d+

29
blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php vendored
View file

@ -20,9 +20,9 @@ class CrawlerDetect
/**
* The user agent.
*
* @var null
* @var string|null
*/
protected $userAgent = null;
protected $userAgent;
/**
* Headers that contain a user agent.
@ -93,7 +93,7 @@ class CrawlerDetect
* Compile the regex patterns into one regex string.
*
* @param array
*
*
* @return string
*/
public function compileRegex($patterns)
@ -138,7 +138,7 @@ class CrawlerDetect
/**
* Set the user agent.
*
* @param string $userAgent
* @param string|null $userAgent
*/
public function setUserAgent($userAgent)
{
@ -165,20 +165,14 @@ class CrawlerDetect
$agent = trim(preg_replace(
"/{$this->compiledExclusions}/i",
'',
$userAgent ?: $this->userAgent
$userAgent ?: $this->userAgent ?: ''
));
if ($agent == '') {
if ($agent === '') {
return false;
}
$result = preg_match("/{$this->compiledRegex}/i", $agent, $matches);
if ($matches) {
$this->matches = $matches;
}
return (bool) $result;
return (bool) preg_match("/{$this->compiledRegex}/i", $agent, $this->matches);
}
/**
@ -190,4 +184,13 @@ class CrawlerDetect
{
return isset($this->matches[0]) ? $this->matches[0] : null;
}
/**
* @return string|null
*/
public function getUserAgent()
{
return $this->userAgent;
}
}

4
blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php vendored
View file

@ -15,14 +15,14 @@ abstract class AbstractProvider
{
/**
* The data set.
*
*
* @var array
*/
protected $data;
/**
* Return the data set.
*
*
* @return array
*/
public function getAll()

302
blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php vendored
View file

@ -19,30 +19,49 @@ class Crawlers extends AbstractProvider
* @var array
*/
protected $data = array(
'.*Java.*outbrain',
' YLT',
'^Aether',
'^Amazon Simple Notification Service Agent$',
'^Amazon-Route53-Health-Check-Service',
'^b0t$',
'^bluefish ',
'^Calypso v\/',
'^COMODO DCV',
'^Corax',
'^DangDang',
'^DavClnt',
'^DHSH',
'^docker\/[0-9]',
'^Expanse',
'^FDM ',
'^git\/',
'^Goose\/',
'^Grabber',
'^Gradle\/',
'^HTTPClient\/',
'^HTTPing',
'^Java\/',
'^Jeode\/',
'^Jetty\/',
'^Mail\/',
'^Mget',
'^Microsoft URL Control',
'^Mikrotik\/',
'^Netlab360',
'^NG\/[0-9\.]',
'^NING\/',
'^PHP\/[0-9]',
'^npm\/',
'^Nuclei',
'^PHP-AYMAPI\/',
'^PHP\/',
'^pip\/',
'^pnpm\/',
'^RMA\/',
'^Ruby|Ruby\/[0-9]',
'^Swurl ',
'^TLS tester ',
'^twine\/',
'^ureq',
'^VSE\/[0-9]',
'^WordPress\.com',
'^XRL\/[0-9]',
@ -50,17 +69,18 @@ class Crawlers extends AbstractProvider
'008\/',
'13TABS',
'192\.comAgent',
'2GDPR\/',
'2ip\.ru',
'404enemy',
'7Siters',
'80legs',
'a\.pr-cy\.ru',
'a3logics\.in',
'A6-Indexer',
'Abonti',
'Aboundex',
'aboutthedomain',
'Accoona-AI-Agent',
'acebookexternalhit\/',
'acoon',
'acrylicapps\.com\/pulp',
'Acunetix',
@ -70,8 +90,14 @@ class Crawlers extends AbstractProvider
'ADmantX',
'AdminLabs',
'adressendeutschland',
'adreview\/',
'adscanner',
'adstxt-worker',
'Adstxtaggregator',
'adstxt\.com',
'Adyen HttpClient',
'AffiliateLabz\/',
'affilimate-puppeteer',
'agentslug',
'AHC',
'aihit',
@ -92,20 +118,23 @@ class Crawlers extends AbstractProvider
'Anemone',
'AngleSharp',
'annotate_google',
'Ant\.com',
'Anthill',
'Anturis Agent',
'Ant\.com',
'AnyEvent-HTTP\/',
'Apache Ant\/',
'Apache Droid',
'Apache OpenOffice',
'Apache-HttpAsyncClient',
'Apache-HttpClient',
'ApacheBench',
'Apexoo',
'apimon\.de',
'APIs-Google',
'AportWorm\/',
'AppBeat\/',
'AppEngine-Google',
'AppStoreScraperZ',
'AppleSyndication',
'Aprc\/[0-9]',
'Arachmo',
'arachnode',
@ -113,21 +142,28 @@ class Crawlers extends AbstractProvider
'aria2',
'Arukereso',
'asafaweb',
'AskQuickly',
'Asana\/',
'Ask Jeeves',
'AskQuickly',
'ASPSeek',
'Asterias',
'Astute',
'asynchttp',
'Attach',
'attohttpc',
'autocite',
'AutomaticWPTester',
'Autonomy',
'awin\.com',
'AWS Security Scanner',
'axios\/',
'a\.pr-cy\.ru',
'B-l-i-t-z-B-O-T',
'Backlink-Ceck',
'backlink-check',
'BacklinkHttpStatus',
'BackStreet',
'BackupLand',
'BackWeb',
'Bad-Neighborhood',
'Badass',
@ -142,6 +178,7 @@ class Crawlers extends AbstractProvider
'BCKLINKS',
'BDFetch',
'BegunAdvertising',
'Bewica-security-scan',
'Bidtellect',
'BigBozz',
'Bigfoot',
@ -151,11 +188,14 @@ class Crawlers extends AbstractProvider
'binlar',
'biNu image cacher',
'Bitacle',
'Bitrix link preview',
'biz_Directory',
'BKCTwitterUnshortener\/',
'Black Hole',
'Blackboard Safeassign',
'BlackWidow',
'BlockNote\.Net',
'BlogBridge',
'Bloglines',
'Bloglovin',
'BlogPulseLive',
@ -163,6 +203,7 @@ class Crawlers extends AbstractProvider
'Blogtrottr',
'BlowFish',
'boitho\.com-dc',
'Boost\.Beast',
'BPImageWalker',
'Braintree-Webhooks',
'Branch Metrics API',
@ -190,12 +231,15 @@ class Crawlers extends AbstractProvider
'CC Metadata Scaper',
'Cegbfeieh',
'censys',
'centuryb.o.t9[at]gmail.com',
'Cerberian Drtrs',
'CERT\.at-Statistics-Survey',
'cf-facebook',
'cg-eye',
'changedetection',
'ChangesMeter',
'Charlotte',
'chatterino-api-cache',
'CheckHost',
'checkprivacy',
'CherryPicker',
@ -206,6 +250,7 @@ class Crawlers extends AbstractProvider
'Chromaxa',
'CirrusExplorer',
'CISPA Vulnerability Notification',
'CISPA Web Analyser',
'Citoid',
'CJNetworkQuality',
'Clarsentia',
@ -213,27 +258,34 @@ class Crawlers extends AbstractProvider
'Cloud mapping',
'CloudEndure',
'CloudFlare-AlwaysOnline',
'Cloudflare-Healthchecks',
'Cloudinary',
'cmcm\.com',
'coccoc',
'cognitiveseo',
'ColdFusion',
'colly -',
'CommaFeed',
'Commons-HttpClient',
'commonscan',
'contactbigdatafr',
'contentkingapp',
'Contextual Code Sites Explorer',
'convera',
'CookieReports',
'copyright sheriff',
'CopyRightCheck',
'Copyscape',
'cortex\/',
'Cosmos4j\.feedback',
'Covario-IDS',
'Craw\/',
'Crescent',
'Crowsnest',
'Criteo',
'Crowsnest',
'CSHttp',
'CSSCheck',
'Cula\/',
'curb',
'Curious George',
'curl',
@ -243,12 +295,17 @@ class Crawlers extends AbstractProvider
'DareBoost',
'DatabaseDriverMysqli',
'DataCha0s',
'DatadogSynthetics',
'Datafeedwatch',
'Datanyze',
'DataparkSearch',
'dataprovider',
'DataXu',
'Daum(oa)?[ \/][0-9]',
'dBpoweramp',
'ddline',
'deeris',
'delve\.ai',
'Demon',
'DeuSu',
'developers\.google\.com\/\+\/web\/snippet\/',
@ -258,8 +315,8 @@ class Crawlers extends AbstractProvider
'DigitalPebble',
'Dirbuster',
'Discourse Forum Onebox',
'Disqus\/',
'Dispatch\/',
'Disqus\/',
'DittoSpyder',
'dlvr',
'DMBrowser',
@ -267,6 +324,8 @@ class Crawlers extends AbstractProvider
'docoloc',
'Dolphin http client',
'DomainAppender',
'DomainLabz',
'Domains Project\/',
'Donuts Content Explorer',
'dotMailer content retrieval',
'dotSemantic',
@ -279,6 +338,8 @@ class Crawlers extends AbstractProvider
'Drupal \(\+http:\/\/drupal\.org\/\)',
'DTS Agent',
'dubaiindex',
'DuplexWeb-Google',
'DynatraceSynthetic',
'EARTHCOM',
'Easy-Thumb',
'EasyDL',
@ -309,20 +370,22 @@ class Crawlers extends AbstractProvider
'ExaleadCloudview',
'Excel\/',
'exif',
'ExoRank',
'Exploratodo',
'Express WebPictures',
'Extreme Picture Finder',
'EyeNetIE',
'ezooms',
'facebookexternalhit',
'facebookexternalua',
'facebookplatform',
'fairshare',
'Faraday v',
'fasthttp',
'Faveeo',
'Favicon downloader',
'faviconkit',
'faviconarchive',
'faviconkit',
'FavOrg',
'Feed Wrangler',
'Feedable\/',
@ -334,7 +397,9 @@ class Crawlers extends AbstractProvider
'feeder',
'Feedly',
'FeedshowOnline',
'Feedshow\/',
'Feedspot',
'FeedViewer\/',
'Feedwind\/',
'FeedZcollector',
'feeltiptop',
@ -342,6 +407,8 @@ class Crawlers extends AbstractProvider
'Fetch\/[0-9]',
'Fever\/[0-9]',
'FHscan',
'Fiery%20Feeds',
'Filestack',
'Fimap',
'findlink',
'findthatfile',
@ -350,18 +417,24 @@ class Crawlers extends AbstractProvider
'FlipboardProxy',
'FlipboardRSS',
'Flock\/',
'Florienzh\/',
'fluffy',
'Flunky',
'flynxapp',
'forensiq',
'ForusP',
'FoundSeoTool',
'http:\/\/www.neomo.de\/', //'Francis [Bot]'
'fragFINN\.de',
'free thumbnails',
'Freeuploader',
'FreshRSS',
'frontman',
'Funnelback',
'Fuzz Faster U Fool',
'G-i-g-a-b-o-t',
'g00g1e\.net',
'ganarvisitas',
'gdnplus\.com',
'geek-tools',
'Genieo',
'GentleSource',
@ -373,18 +446,24 @@ class Crawlers extends AbstractProvider
'getroot',
'GetURLInfo\/',
'GetWeb',
'Geziyor',
'Ghost Inspector',
'GigablastOpenSource',
'GIS-LABS',
'github-camo',
'GitHub-Hookshot',
'github\.com',
'Go [\d\.]* package http',
'Go http package',
'Go [\d\.]* package http',
'Go!Zilla',
'Go-Ahead-Got-It',
'Go-http-client',
'Go!Zilla',
'go-mtasts\/',
'gobuster',
'gobyus',
'Gofeed',
'gofetch',
'Goldfire Server',
'GomezAgent',
'gooblog',
'Goodzer\/',
@ -397,24 +476,31 @@ class Crawlers extends AbstractProvider
'Google PP Default',
'Google Search Console',
'Google Web Preview',
'Google-Ads-Creatives-Assistant',
'Google-Ads-Overview',
'Google-Adwords',
'Google-Apps-Script',
'Google-Calendar-Importer',
'Google-HotelAdsVerifier',
'Google-HTTP-Java-Client',
'Google-InspectionTool',
'Google-Podcast',
'Google-Publisher-Plugin',
'Google-Read-Aloud',
'Google-SearchByImage',
'Google-Site-Verification',
'Google-SMTP-STS',
'Google-speakr',
'Google-Structured-Data-Testing-Tool',
'Google-Youtube-Links',
'Google-Transparency-Report',
'google-xrawler',
'Google-Youtube-Links',
'GoogleDocs',
'GoogleHC\/',
'GoogleProber',
'GoogleProducer',
'GoogleSites',
'Google-Transparency-Report',
'Gookey',
'GoScraper',
'GoSpotCheck',
'gosquared-thumbnailer',
'Gotit',
@ -440,13 +526,16 @@ class Crawlers extends AbstractProvider
'hackney\/',
'Hadi Agent',
'HappyApps-WebCheck',
'Hardenize',
'Hatena',
'Havij',
'HaxerMen',
'HeadlessChrome',
'HEADMasterSEO',
'HeartRails_Capture',
'help@dataminr\.com',
'heritrix',
'Hexometer',
'historious',
'hkedcity',
'hledejLevne\.cz',
@ -463,28 +552,31 @@ class Crawlers extends AbstractProvider
'HTMLparser',
'htmlyse',
'HTTP Banner Detection',
'HTTP_Compression_Test',
'http_request2',
'http_requester',
'http-get',
'HTTP-Header-Abfrage',
'http-kit',
'http-request\/',
'HTTP-Tiny',
'HTTP::Lite',
'http\.rb\/',
'http_get',
'http:\/\/www.neomo.de\/', //'Francis [Bot]'
'HttpComponents',
'httphr',
'HTTPie',
'HTTPMon',
'httpRequest',
'httpscheck',
'httpssites_power',
'httpunit',
'HttpUrlConnection',
'http\.rb\/',
'HTTP_Compression_Test',
'http_get',
'http_request2',
'http_requester',
'httrack',
'huaweisymantec',
'HubSpot ',
'HubSpot-Link-Resolver',
'Humanlinks',
'i2kconnect\/',
'Iblog',
@ -495,6 +587,7 @@ class Crawlers extends AbstractProvider
'IDwhois\/',
'Iframely',
'igdeSpyder',
'iGooglePortal',
'IlTrovatore',
'Image Fetch',
'Image Sucker',
@ -519,13 +612,15 @@ class Crawlers extends AbstractProvider
'integromedb',
'Intelliseek',
'InterGET',
'internet_archive',
'Internet Ninja',
'InternetSeer',
'internetVista monitor',
'internetwache',
'internet_archive',
'intraVnews',
'IODC',
'IOI',
'Inboxb0t',
'iplabel',
'ips-agent',
'IPS\/[0-9]',
@ -537,10 +632,13 @@ class Crawlers extends AbstractProvider
'iskanie',
'isUp\.li',
'iThemes Sync\/',
'IZaBEE',
'iZSearch',
'JAHHO',
'janforman',
'Jaunt\/',
'Java.*outbrain',
'javelin\.io',
'Jbrofuzz',
'Jersey\/',
'JetCar',
@ -556,6 +654,7 @@ class Crawlers extends AbstractProvider
'Joomla',
'Jorgee',
'JS-Kit',
'JungleKeyThumbnail',
'JustView',
'Kaspersky Lab CFR link resolver',
'Kelny\/',
@ -563,6 +662,7 @@ class Crawlers extends AbstractProvider
'KeyCDN',
'Keyword Density',
'Keywords Research',
'khttp\/',
'KickFire',
'KimonoLabs\/',
'Kml-Google',
@ -570,11 +670,12 @@ class Crawlers extends AbstractProvider
'KOCMOHABT',
'kouio',
'kube-probe',
'kubectl',
'kulturarw3',
'KumKie',
'L\.webis',
'Larbin',
'Lavf\/',
'leakix\.net',
'LeechFTP',
'LeechGet',
'letsencrypt',
@ -585,41 +686,50 @@ class Crawlers extends AbstractProvider
'libwww',
'Licorne',
'Liferea\/',
'Lightspeedsystems',
'Lighthouse',
'Lightspeedsystems',
'Likse',
'limber\.io',
'Link Valet',
'link_thumbnailer',
'LinkAlarm\/',
'LinkAnalyser',
'linkCheck',
'linkdex',
'LinkExaminer',
'linkfluence',
'linkpeek',
'LinkPreviewGenerator',
'LinkPreview',
'LinkScan',
'LinksManager',
'LinkTiger',
'LinkWalker',
'link_thumbnailer',
'Lipperhey',
'Litemage_walker',
'livedoor ScreenShot',
'LoadImpactRload',
'localsearch-web',
'LongURL API',
'longurl-r-package',
'looid\.com',
'looksystems\.net',
'ltx71',
'lua-resty-http',
'Lucee \(CFML Engine\)',
'Lush Http Client',
'lwp-request',
'lwp-trivial',
'LWP::Simple',
'lycos',
'LYT\.SR',
'L\.webis',
'mabontland',
'MacOutlook\/',
'Mag-Net',
'MagpieRSS',
'Mail\.Ru',
'Mail::STS',
'MailChimp',
'Mail\.Ru',
'Majestic12',
'makecontact\/',
'Mandrill',
@ -630,6 +740,8 @@ class Crawlers extends AbstractProvider
'Mass Downloader',
'masscan\/',
'Mata Hari',
'mattermost',
'Mediametric',
'Mediapartners-Google',
'mediawords',
'MegaIndex\.ru',
@ -640,28 +752,31 @@ class Crawlers extends AbstractProvider
'MetaURI',
'MFC_Tear_Sample',
'Microsearch',
'Microsoft Office ',
'Microsoft Data Access',
'Microsoft Office',
'Microsoft Outlook',
'Microsoft Windows Network Diagnostics',
'Microsoft-WebDAV-MiniRedir',
'Microsoft Data Access',
'Microsoft\.Data\.Mashup',
'MIDown tool',
'MIIxpc',
'Mindjet',
'Miniature\.io',
'Miniflux',
'mio_httpc',
'Miro-HttpClient',
'Mister PiX',
'mixdata dot com',
'mixed-content-scan',
'Mixmax-LinkPreview',
'mixnode',
'Mnogosearch',
'mogimogi',
'Mojeek',
'Mojolicious \(Perl\)',
'Monit\/',
'Mollie',
'monitis',
'Monitority\/',
'Monit\/',
'montastic',
'MonTools',
'Moreover',
@ -670,6 +785,7 @@ class Crawlers extends AbstractProvider
'MovableType',
'mowser',
'Mrcgiguy',
'Mr\.4x3 Powered',
'MS Web Services Client Protocol',
'MSFrontPage',
'mShots',
@ -677,10 +793,13 @@ class Crawlers extends AbstractProvider
'muhstik-scan',
'MVAClient',
'MxToolbox\/',
'myseosnapshot',
'nagios',
'Najdi\.si',
'Name Intelligence',
'NameFo\.com',
'Nameprotect',
'nationalarchives',
'Navroad',
'NearSite',
'Needle',
@ -695,6 +814,8 @@ class Crawlers extends AbstractProvider
'netresearch',
'NetShelter ContentScan',
'Netsparker',
'NetSystemsResearch',
'nettle',
'NetTrack',
'Netvibes',
'NetZIP',
@ -707,22 +828,26 @@ class Crawlers extends AbstractProvider
'newspaper\/',
'Nexgate Ruby Client',
'NG-Search',
'nghttp2',
'Nibbler',
'NICErsPRO',
'NihilScio',
'Nikto',
'nineconnections',
'NLNZ_IAHarvester',
'Nmap Scripting Engine',
'node-fetch',
'node-superagent',
'node-urllib',
'node\.io',
'Nodemeter',
'NodePing',
'node\.io',
'nominet\.org\.uk',
'nominet\.uk',
'Norton-Safeweb',
'Notifixious',
'notifyninja',
'NotionEmbedder',
'nuhk',
'nutch',
'Nuzzel',
@ -730,39 +855,45 @@ class Crawlers extends AbstractProvider
'nyawc\/',
'Nymesis',
'NYU',
'Observatory\/',
'Ocelli\/',
'Octopus',
'oegp',
'Offline Explorer',
'Offline Navigator',
'og-scraper',
'OgScrper',
'okhttp',
'omgili',
'OMSC',
'Online Domain Tools',
'Open Source RSS',
'OpenCalaisSemanticProxy',
'Openfind',
'OpenLinkProfiler',
'Openstat\/',
'OpenVAS',
'OPPO A33',
'Optimizer',
'Orbiter',
'OrgProbe\/',
'orion-semantics',
'Outlook-Express',
'Outlook-iOS',
'ow\.ly',
'Owler',
'Owlin',
'ownCloud News',
'ow\.ly',
'OxfordCloudService',
'Page Valet',
'page_verifier',
'page scorer',
'Page Valet',
'page2rss',
'PageFreezer',
'PageGrabber',
'PagePeeker',
'PageScorer',
'Pagespeed\/',
'PageThing',
'page_verifier',
'Panopta',
'panscient',
'Papa Foto',
@ -771,6 +902,7 @@ class Crawlers extends AbstractProvider
'PayPal IPN',
'pcBrowser',
'Pcore-HTTP',
'PDF24 URL To PDF',
'Pearltrees',
'PECL::HTTP',
'peerindex',
@ -780,44 +912,52 @@ class Crawlers extends AbstractProvider
'PhantomJS Screenshoter',
'PhantomJS\/',
'Photon\/',
'php-requests',
'phpservermon',
'Pi-Monster',
'Picscout',
'Picsearch',
'PictureFinder',
'Pimonster',
'ping\.blo\.gs',
'Pingability',
'PingAdmin\.Ru',
'Pingdom',
'Pingoscope',
'PingSpot',
'ping\.blo\.gs',
'pinterest\.com',
'Pixray',
'Pizilla',
'Plagger\/',
'Pleroma ',
'Ploetz \+ Zeller',
'Plukkie',
'plumanalytics',
'PocketImageCache',
'PocketParser',
'Pockey',
'PodcastAddict\/',
'POE-Component-Client-HTTP',
'Polymail\/',
'Pompos',
'Porkbun',
'Port Monitor',
'postano',
'postfix-mta-sts-resolver',
'PostmanRuntime',
'postplanner\.com',
'PostPost',
'postrank',
'PowerPoint\/',
'Prebid',
'Prerender',
'Priceonomics Analysis Engine',
'PrintFriendly',
'PritTorrent',
'Prlog',
'probethenet',
'Project 25499',
'Project ?25499',
'Project-Resonance',
'prospectb2b',
'Protopage',
'ProWebWalker',
@ -826,9 +966,9 @@ class Crawlers extends AbstractProvider
'pshtt, https scanning',
'PTST ',
'PTST\/[0-9]+',
'Pulsepoint XT3 web scraper',
'Pump',
'Python-httplib2',
'python-httpx',
'python-requests',
'Python-urllib',
'Qirina Hurdler',
@ -838,12 +978,17 @@ class Crawlers extends AbstractProvider
'Qualidator',
'QueryN Metasearch',
'queuedriver',
'quic-go-HTTP\/',
'QuiteRSS',
'Quora Link Preview',
'Qwantify',
'Radian6',
'RadioPublicImageResizer',
'Railgun\/',
'RankActive',
'RankFlex',
'RankSonicSiteAuditor',
'RapidLoad\/',
'Re-re Studio',
'ReactorNetty',
'Readability',
@ -854,6 +999,7 @@ class Crawlers extends AbstractProvider
'RecurPost\/',
'redback\/',
'ReederForMac',
'Reeder\/',
'ReGet',
'RepoMonkey',
'request\.js',
@ -866,38 +1012,44 @@ class Crawlers extends AbstractProvider
'Robozilla',
'ROI Hunter',
'RPT-HTTPClient',
'RSSMix\/',
'RSSOwl',
'RyowlEngine',
'safe-agent-scanner',
'SalesIntelligent',
'Saleslift',
'Sendsay\.Ru',
'SAP NetWeaver Application Server',
'SauceNAO',
'SBIder',
'sc-downloader',
'scalaj-http',
'scan\.lol',
'Scamadviser-Frontend',
'ScanAlert',
'scan\.lol',
'Scoop',
'scooter',
'ScopeContentAG-HTTP-Client',
'ScoutJet',
'ScoutURLMonitor',
'ScrapeBox Page Scanner',
'SimpleScraper',
'Scrapy',
'Screaming',
'ScreenShotService',
'Scrubby',
'Scrutiny\/',
'search\.thunderstone',
'Search37',
'searchenginepromotionhelp',
'Searchestate',
'SearchExpress',
'SearchSight',
'SearchWP',
'search\.thunderstone',
'Seeker',
'semanticdiscovery',
'semanticjuice',
'Semiocast HTTP client',
'Semrush',
'Sendsay\.Ru',
'sentry\/',
'SEO Browser',
'Seo Servis',
@ -906,11 +1058,13 @@ class Crawlers extends AbstractProvider
'Seobility',
'SEOCentro',
'SeoCheck',
'seocompany',
'SEOkicks',
'SEOlizer',
'Seomoz',
'SEOprofiler',
'SEOsearch',
'seoscanners',
'SEOsearch',
'seositecheckup',
'SEOstats',
'servernfo',
@ -921,14 +1075,16 @@ class Crawlers extends AbstractProvider
'Shoppimon',
'ShopWiki',
'ShortLinkTranslate',
'shortURL lengthener',
'shrinktheweb',
'Sideqik',
'Siege',
'SimplePie',
'SimplyFast',
'Siphon',
'SISTRIX',
'Site-Shot\/',
'Site Sucker',
'Site-Shot\/',
'Site24x7',
'SiteBar',
'Sitebeam',
@ -949,6 +1105,7 @@ class Crawlers extends AbstractProvider
'sitexy\.com',
'SkypeUriPreview',
'Slack\/',
'sli-systems\.com',
'slider\.com',
'slurp',
'SlySearch',
@ -971,9 +1128,11 @@ class Crawlers extends AbstractProvider
'SpamExperts',
'Spammen',
'Spanner',
'Spawning-AI',
'spaziodati',
'SPDYCheck',
'Specificfeeds',
'SpeedKit',
'speedy',
'SPEng',
'Spinn3r',
@ -987,9 +1146,11 @@ class Crawlers extends AbstractProvider
'ssl-tools',
'StackRambler',
'Statastico\/',
'Statically-',
'StatusCake',
'Steeler',
'Stratagems Kumo',
'Stripe\/',
'Stroke\.cz',
'StudioFACA',
'StumbleUpon',
@ -999,9 +1160,11 @@ class Crawlers extends AbstractProvider
'SuperHTTP',
'Surphace Scout',
'Suzuran',
'SwiteScraper',
'swcd ',
'Symfony BrowserKit',
'Symfony2 BrowserKit',
'Synapse\/',
'Syndirella\/',
'SynHttpClient-Built',
'Sysomos',
'sysscan',
@ -1011,6 +1174,8 @@ class Crawlers extends AbstractProvider
'Tarantula\/',
'Taringa UGC',
'TarmotGezgin',
'tchelebi\.io',
'techiaith\.cymru',
'Teleport',
'Telesoft',
'Telesphoreo',
@ -1021,13 +1186,15 @@ class Crawlers extends AbstractProvider
'Test Certificate Info',
'testuri',
'Tetrahedron',
'TextRazor Downloader',
'The Drop Reaper',
'The Expert HTML Source Viewer',
'The Knowledge AI',
'The Intraformant',
'The Knowledge AI',
'theinternetrules',
'TheNomad',
'Thinklab',
'Thumbor',
'Thumbshots',
'ThumbSniper',
'timewe\.net',
@ -1061,9 +1228,11 @@ class Crawlers extends AbstractProvider
'ubermetrics-technologies',
'uclassify',
'UdmSearch',
'ultimate_sitemap_parser',
'unchaos',
'unirest-java',
'UniversalFeedParser',
'unshortenit',
'Unshorten\.It',
'Untiny',
'UnwindFetchor',
@ -1071,8 +1240,8 @@ class Crawlers extends AbstractProvider
'updown\.io daemon',
'Upflow',
'Uptimia',
'Urlcheckr',
'URL Verifier',
'Urlcheckr',
'URLitor',
'urlresolver',
'Urlstat',
@ -1080,15 +1249,16 @@ class Crawlers extends AbstractProvider
'UrlTrends Ranking Updater',
'URLy Warning',
'URLy\.Warning',
'URL\/Emacs',
'Vacuum',
'Vagabondo',
'VB Project',
'vBSEO',
'VCI',
'via ggpht\.com GoogleImageProxy',
'VidibleScraper',
'Virusdie',
'visionutils',
'Visual Rights Group',
'vkShare',
'VoidEYE',
'Voil',
@ -1099,19 +1269,18 @@ class Crawlers extends AbstractProvider
'Vulnbusters Meter',
'VYU2',
'w3af\.org',
'W3C_Unicorn',
'W3C-checklink',
'W3C-mobileOK',
'W3C_Unicorn',
'WAC-OFU',
'Wallpapers\/[0-9]+',
'WakeletLinkExpander',
'WallpapersHD',
'Wallpapers\/[0-9]+',
'wangling',
'Wappalyzer',
'WatchMouse',
'WbSrch\/',
'WDT\.io',
'web-capture\.net',
'Web-sniffer',
'Web Auto',
'Web Collage',
'Web Enhancer',
@ -1119,7 +1288,10 @@ class Crawlers extends AbstractProvider
'Web Fuck',
'Web Pix',
'Web Sauger',
'Web spyder',
'Web Sucker',
'web-capture\.net',
'Web-sniffer',
'Webalta',
'Webauskunft',
'WebAuto',
@ -1157,8 +1329,9 @@ class Crawlers extends AbstractProvider
'Webster',
'WebStripper',
'WebSucker',
'Webthumb\/',
'webtech\/',
'WebThumbnail',
'Webthumb\/',
'WebWhacker',
'WebZIP',
'WeLikeLinks',
@ -1167,24 +1340,31 @@ class Crawlers extends AbstractProvider
'wf84',
'Wfuzz\/',
'wget',
'WhatCMS',
'WhatsApp',
'WhatsMyIP',
'WhatWeb',
'WhereGoes\?',
'Whibse',
'WhoAPI\/',
'WhoRunsCoinHive',
'Whynder Magnet',
'Windows-RSS-Platform',
'WinHttp-Autoproxy-Service',
'WinHTTP\/',
'WinPodder',
'wkhtmlto',
'wmtips',
'Woko',
'Wolfram HTTPClient',
'woorankreview',
'Word\/',
'WordPress\/',
'WordupinfoSearch',
'Word\/',
'worldping-api',
'wotbox',
'WP Engine Install Performance API',
'WP Rocket',
'wpif',
'wprecon\.com survey',
'WPScan',
@ -1194,20 +1374,21 @@ class Crawlers extends AbstractProvider
'WWW-Mechanize',
'WWW::Document',
'WWW::Mechanize',
'www\.monitor\.us',
'WWWOFFLE',
'www\.monitor\.us',
'x09Mozilla',
'x22Mozilla',
'XaxisSemanticsClassifier',
'XenForo\/',
'Xenu Link Sleuth',
'XING-contenttabreceiver',
'xpymep([0-9]?)\.exe',
'Y!J-(ASR|BSC)',
'Y\!J-BRW',
'Y!J-[A-Z][A-Z][A-Z]',
'Yaanb',
'yacy',
'Yahoo Link Preview',
'YahooCacheSystem',
'YahooMailProxy',
'YahooYSMcm',
'YandeG',
'Yandex(?!Search)',
@ -1215,6 +1396,7 @@ class Crawlers extends AbstractProvider
'yeti',
'Yo-yo',
'Yoleo Consumer',
'yomins\.com',
'yoogliFetchAgent',
'YottaaMonitor',
'Your-Website-Sucks',
@ -1226,15 +1408,17 @@ class Crawlers extends AbstractProvider
'Zao',
'Zauba',
'Zemanta Aggregator',
'Zend_Http_Client',
'Zend\\\\Http\\\\Client',
'Zend_Http_Client',
'Zermelo',
'Zeus ',
'zgrab',
'ZnajdzFoto',
'ZnHTTP',
'Zombie\.js',
'Zoom\.Mac',
'ZoteroTranslationServer',
'ZyBorg',
'[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)',
'[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)',
);
}

4
blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php vendored
View file

@ -67,6 +67,8 @@ class Exclusions extends AbstractProvider
'; IDbot',
'; ID bot',
'; POWER BOT',
';', // Remove the following characters ;
'OCTOPUS-CORE',
'htc_botdugls',
'super\/\d+\/Android\/\d+',
);
}