jakob/friendica-addons
1
0
Fork 0
forked from friendica/friendica-addons
Code Pull requests Activity

[blockbot] Update Composer dependency ahead of release

Browse source 
- Updating jaybizzle/crawler-detect (v1.2.80 => v1.2.116)
This commit is contained in:
Hypolite Petovan 2024-03-19 22:51:17 -04:00
commit d838fc6421
20 changed files with 686 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

16
blockbot/composer.lock generated
View file

@ -8,24 +8,23 @@
"packages": [ "packages": [
{ {
"name": "jaybizzle/crawler-detect", "name": "jaybizzle/crawler-detect",
"version": "v1.2.80", "version": "v1.2.116",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git", "url": "https://github.com/JayBizzle/Crawler-Detect.git",
"reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847" "reference": "97e9fe30219e60092e107651abb379a38b342921"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847", "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921",
"reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847", "reference": "97e9fe30219e60092e107651abb379a38b342921",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"php": ">=5.3.0" "php": ">=5.3.0"
}, },
"require-dev": { "require-dev": {
"phpunit/phpunit": "^4.8|^5.5|^6.5", "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
"satooshi/php-coveralls": "1.*"
}, },
"type": "library", "type": "library",
"autoload": { "autoload": {
@ -53,7 +52,7 @@
"crawlerdetect", "crawlerdetect",
"php crawler detect" "php crawler detect"
], ],
"time": "2019-04-05T19:52:02+00:00" "time": "2023-07-21T15:49:49+00:00"
} }
], ],
"packages-dev": [], "packages-dev": [],
@ -65,5 +64,6 @@
"platform": { "platform": {
"php": ">=5.6.0" "php": ">=5.6.0"
}, },
"platform-dev": [] "platform-dev": [],
"plugin-api-version": "1.1.0"
} }

4
blockbot/vendor/composer/ClassLoader.php vendored
View file

@ -60,7 +60,7 @@ class ClassLoader
public function getPrefixes() public function getPrefixes()
{ {
if (!empty($this->prefixesPsr0)) { if (!empty($this->prefixesPsr0)) {
return call_user_func_array('array_merge', $this->prefixesPsr0); return call_user_func_array('array_merge', array_values($this->prefixesPsr0));
} }
return array(); return array();
@ -279,7 +279,7 @@ class ClassLoader
*/ */
public function setApcuPrefix($apcuPrefix) public function setApcuPrefix($apcuPrefix)
{ {
$this->apcuPrefix = function_exists('apcu_fetch') && ini_get('apc.enabled') ? $apcuPrefix : null; $this->apcuPrefix = function_exists('apcu_fetch') && filter_var(ini_get('apc.enabled'), FILTER_VALIDATE_BOOLEAN) ? $apcuPrefix : null;
} }
/** /**

3
blockbot/vendor/composer/autoload_real.php vendored
View file

@ -13,6 +13,9 @@ class ComposerAutoloaderInitBlockBotAddon
} }
} }
/**
* @return \Composer\Autoload\ClassLoader
*/
public static function getLoader() public static function getLoader()
{ {
if (null !== self::$loader) { if (null !== self::$loader) {

15
blockbot/vendor/composer/installed.json vendored
View file

@ -1,27 +1,26 @@
[ [
{ {
"name": "jaybizzle/crawler-detect", "name": "jaybizzle/crawler-detect",
"version": "v1.2.80", "version": "v1.2.116",
"version_normalized": "1.2.80.0", "version_normalized": "1.2.116.0",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/JayBizzle/Crawler-Detect.git", "url": "https://github.com/JayBizzle/Crawler-Detect.git",
"reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847" "reference": "97e9fe30219e60092e107651abb379a38b342921"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847", "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921",
"reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847", "reference": "97e9fe30219e60092e107651abb379a38b342921",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"php": ">=5.3.0" "php": ">=5.3.0"
}, },
"require-dev": { "require-dev": {
"phpunit/phpunit": "^4.8|^5.5|^6.5", "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
"satooshi/php-coveralls": "1.*"
}, },
"time": "2019-04-05T19:52:02+00:00", "time": "2023-07-21T15:49:49+00:00",
"type": "library", "type": "library",
"installation-source": "dist", "installation-source": "dist",
"autoload": { "autoload": {

23
blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml vendored Normal file
View file

@ -0,0 +1,23 @@
name: Check & fix styling
on: [ push ]
jobs:
php-cs-fixer:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
ref: ${{ github.head_ref }}
- name: Run PHP CS Fixer
uses: docker://oskarstark/php-cs-fixer-ga:2.18.6
with:
args: --config=.php_cs.dist --allow-risky=yes
- name: Commit changes
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: Fix styling

56
blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml vendored Normal file
View file

@ -0,0 +1,56 @@
name: Test
on:
push:
branches:
- "master"
pull_request:
jobs:
build:
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
php: [5.3, 5.4, 5.5, 5.6, 7.0, 7.1, 7.2, 7.3, 7.4, 8.0, 8.1, 8.2]
name: PHP:${{ matrix.php }}
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Setup PHP, with composer
uses: shivammathur/setup-php@v2
with:
php-version: ${{ matrix.php }}
tools: composer:v2
coverage: xdebug
- name: Get composer cache directory
id: composer-cache
run: |
echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT
shell: bash
- name: Cache composer dependencies
uses: actions/cache@v3
with:
path: ${{ steps.composer-cache.outputs.dir }}
key: dependencies-php-${{ matrix.php }}-composer-${{ hashFiles('composer.json') }}
restore-keys: dependencies-php-${{ matrix.php }}-composer-
- name: Install Composer dependencies
run: |
composer install --prefer-dist --no-interaction --no-suggest
- name: Run Unit tests
run: |
vendor/bin/phpunit --coverage-clover=tests/logs/clover.xml
- name: Upload coverage results to Coveralls
env:
COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
composer global require php-coveralls/php-coveralls "^1.0"
coveralls --coverage_clover=tests/logs/clover.xml -v

33
blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist vendored Normal file
View file

@ -0,0 +1,33 @@
<?php
$finder = Symfony\Component\Finder\Finder::create()
->in([
__DIR__.'/src',
__DIR__.'/tests',
])
->name('*.php')
->ignoreDotFiles(true)
->ignoreVCS(true);
return PhpCsFixer\Config::create()
->setRules([
'@PSR2' => true,
'array_syntax' => ['syntax' => 'long'],
'ordered_imports' => ['sortAlgorithm' => 'alpha'],
'no_unused_imports' => true,
'not_operator_with_successor_space' => true,
'trailing_comma_in_multiline_array' => true,
'phpdoc_scalar' => true,
'unary_operator_spaces' => true,
'binary_operator_spaces' => true,
'blank_line_before_statement' => [
'statements' => ['break', 'continue', 'declare', 'return', 'throw', 'try'],
],
'phpdoc_single_line_var_spacing' => true,
'phpdoc_var_without_name' => true,
'method_argument_space' => [
'on_multiline' => 'ensure_fully_multiline',
'keep_multiple_spaces_after_comma' => true,
],
])
->setFinder($finder);

2
blockbot/vendor/jaybizzle/crawler-detect/LICENSE vendored
View file

@ -1,6 +1,6 @@
The MIT License (MIT) The MIT License (MIT)
Copyright (c) 2015-2018 Mark Beech Copyright (c) 2015-2020 Mark Beech
Permission is hereby granted, free of charge, to any person obtaining a copy Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal of this software and associated documentation files (the "Software"), to deal

30
blockbot/vendor/jaybizzle/crawler-detect/README.md vendored
View file

@ -1,24 +1,23 @@
<p align="center"><a href="http://crawlerdetect.io/" target="_blank"><img src="https://cloud.githubusercontent.com/assets/340752/23082173/1bd1a396-f550-11e6-8aba-4d3c75edea2f.png" width="321" height="219" /></a><br><br> <p align="center"><a href="https://crawlerdetect.io/" target="_blank"><img src="https://cloud.githubusercontent.com/assets/340752/23082173/1bd1a396-f550-11e6-8aba-4d3c75edea2f.png" width="321" height="219" /></a><br><br>
<a href="http://crawlerdetect.io/" target="_blank">crawlerdetect.io</a> <a href="https://crawlerdetect.io/" target="_blank">crawlerdetect.io</a>
<br><br> <br><br>
</p> </p>
<p align="center"> <p align="center">
<a href="https://travis-ci.org/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/travis/JayBizzle/Crawler-Detect/master.svg?style=flat-square" /></a> <a href="https://github.com/JayBizzle/Crawler-Detect/actions"><img alt="GitHub Workflow Status" src="https://img.shields.io/github/actions/workflow/status/JayBizzle/Crawler-Detect/test.yml?branch=master&style=flat-square"></a>
<a href="https://packagist.org/packages/jaybizzle/crawler-detect"><img src="https://img.shields.io/packagist/dm/JayBizzle/Crawler-Detect.svg?style=flat-square" /></a> <a href="https://packagist.org/packages/jaybizzle/crawler-detect"><img src="https://img.shields.io/packagist/dm/JayBizzle/Crawler-Detect.svg?style=flat-square" /></a>
<a href="https://scrutinizer-ci.com/g/JayBizzle/Crawler-Detect/?branch=master"><img src="https://img.shields.io/scrutinizer/g/JayBizzle/Crawler-Detect.svg?style=flat-square" /></a>
<a href="https://github.com/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/badge/license-MIT-ff69b4.svg?style=flat-square" /></a> <a href="https://github.com/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/badge/license-MIT-ff69b4.svg?style=flat-square" /></a>
<a href="https://packagist.org/packages/jaybizzle/crawler-detect"><img src="https://img.shields.io/packagist/v/jaybizzle/Crawler-Detect.svg?style=flat-square" /></a> <a href="https://packagist.org/packages/jaybizzle/crawler-detect"><img src="https://img.shields.io/packagist/v/jaybizzle/Crawler-Detect.svg?style=flat-square" /></a>
<a href="https://styleci.io/repos/32755917"><img src="https://styleci.io/repos/32755917/shield" /></a>
<a href="https://coveralls.io/github/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/coveralls/JayBizzle/Crawler-Detect/master.svg?style=flat-square" /></a> <a href="https://coveralls.io/github/JayBizzle/Crawler-Detect"><img src="https://img.shields.io/coveralls/JayBizzle/Crawler-Detect/master.svg?style=flat-square" /></a>
</p> </p>
## About CrawlerDetect ## About CrawlerDetect
CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the user agent and http_from header. Currently able to detect 1,000's of bots/spiders/crawlers. CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the `user agent` and `http_from` header. Currently able to detect 1,000's of bots/spiders/crawlers.
### Installation ### Installation
Run `composer require jaybizzle/crawler-detect 1.*` or add `"jaybizzle/crawler-detect" :"1.*"` to your `composer.json`. ```
composer require jaybizzle/crawler-detect
```
### Usage ### Usage
```PHP ```PHP
@ -46,7 +45,7 @@ If you find a bot/spider/crawler user agent that CrawlerDetect fails to detect,
Failing that, just create an issue with the user agent you have found, and we'll take it from there :) Failing that, just create an issue with the user agent you have found, and we'll take it from there :)
### Laravel Package ### Laravel Package
If you would like to use this with Laravel 4/5, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect) If you would like to use this with Laravel, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect)
### Symfony Bundle ### Symfony Bundle
To use this library with Symfony 2/3/4, check out the [CrawlerDetectBundle](https://github.com/nicolasmure/CrawlerDetectBundle). To use this library with Symfony 2/3/4, check out the [CrawlerDetectBundle](https://github.com/nicolasmure/CrawlerDetectBundle).
@ -57,16 +56,21 @@ To use this library with the YII2 framework, check out [yii2-crawler-detect](htt
### ES6 Library ### ES6 Library
To use this library with NodeJS or any ES6 application based, check out [es6-crawler-detect](https://github.com/JefferyHus/es6-crawler-detect). To use this library with NodeJS or any ES6 application based, check out [es6-crawler-detect](https://github.com/JefferyHus/es6-crawler-detect).
### Python Library
To use this library in a Python project, check out [crawlerdetect](https://github.com/moskrc/CrawlerDetect).
### JVM Library (written in Java)
To use this library in a JVM project (including Java, Scala, Kotlin, etc.), check out [CrawlerDetect](https://github.com/nekosoftllc/crawler-detect).
### .NET Library ### .NET Library
To use this library in a .net standard (including .net core) based project, check out [NetCrawlerDetect](https://github.com/gplumb/NetCrawlerDetect). To use this library in a .net standard (including .net core) based project, check out [NetCrawlerDetect](https://github.com/gplumb/NetCrawlerDetect).
### Nette Extension
To use this library with the Nette framework, checkout [NetteCrawlerDetect](https://github.com/JanGalek/Crawler-Detect).
### Ruby Gem ### Ruby Gem
To use this library with Ruby on Rails or any Ruby-based application, check out [crawler_detect](https://github.com/loadkpi/crawler_detect) gem. To use this library with Ruby on Rails or any Ruby-based application, check out [crawler_detect](https://github.com/loadkpi/crawler_detect) gem.
### Go Module
To use this library with Go, check out the [crawlerdetect](https://github.com/x-way/crawlerdetect) module.
_Parts of this class are based on the brilliant [MobileDetect](https://github.com/serbanghita/Mobile-Detect)_ _Parts of this class are based on the brilliant [MobileDetect](https://github.com/serbanghita/Mobile-Detect)_
[![Analytics](https://ga-beacon.appspot.com/UA-72430465-1/Crawler-Detect/readme?pixel)](https://github.com/JayBizzle/Crawler-Detect) [![Analytics](https://ga-beacon.appspot.com/UA-72430465-1/Crawler-Detect/readme?pixel)](https://github.com/JayBizzle/Crawler-Detect)

3
blockbot/vendor/jaybizzle/crawler-detect/composer.json vendored
View file

@ -16,8 +16,7 @@
"php": ">=5.3.0" "php": ">=5.3.0"
}, },
"require-dev": { "require-dev": {
"phpunit/phpunit": "^4.8|^5.5|^6.5", "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4"
"satooshi/php-coveralls": "1.*"
}, },
"autoload": { "autoload": {
"psr-4": { "psr-4": {

2
blockbot/vendor/jaybizzle/crawler-detect/export.php vendored
View file

@ -37,5 +37,5 @@ function outputJson($object)
function outputTxt($object) function outputTxt($object)
{ {
$className = (new ReflectionClass($object))->getShortName(); $className = (new ReflectionClass($object))->getShortName();
file_put_contents("raw/$className.txt", implode($object->getAll(), PHP_EOL)); file_put_contents("raw/$className.txt", implode(PHP_EOL, $object->getAll()));
} }

2
blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json vendored
View file

File diff suppressed because one or more lines are too long

300
blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt vendored
View file

@ -1,27 +1,46 @@
.*Java.*outbrain
YLT YLT
^Aether
^Amazon Simple Notification Service Agent$
^Amazon-Route53-Health-Check-Service
^b0t$ ^b0t$
^bluefish ^bluefish
^Calypso v\/ ^Calypso v\/
^COMODO DCV ^COMODO DCV
^Corax
^DangDang ^DangDang
^DavClnt ^DavClnt
^DHSH
^docker\/[0-9]
^Expanse
^FDM ^FDM
^git\/ ^git\/
^Goose\/ ^Goose\/
^Grabber ^Grabber
^Gradle\/
^HTTPClient\/ ^HTTPClient\/
^HTTPing
^Java\/ ^Java\/
^Jeode\/ ^Jeode\/
^Jetty\/ ^Jetty\/
^Mail\/ ^Mail\/
^Mget ^Mget
^Microsoft URL Control ^Microsoft URL Control
^Mikrotik\/
^Netlab360
^NG\/[0-9\.] ^NG\/[0-9\.]
^NING\/ ^NING\/
^PHP\/[0-9] ^npm\/
^Nuclei
^PHP-AYMAPI\/
^PHP\/
^pip\/
^pnpm\/
^RMA\/ ^RMA\/
^Ruby|Ruby\/[0-9] ^Ruby|Ruby\/[0-9]
^Swurl
^TLS tester
^twine\/
^ureq
^VSE\/[0-9] ^VSE\/[0-9]
^WordPress\.com ^WordPress\.com
^XRL\/[0-9] ^XRL\/[0-9]
@ -29,17 +48,18 @@
008\/ 008\/
13TABS 13TABS
192\.comAgent 192\.comAgent
2GDPR\/
2ip\.ru 2ip\.ru
404enemy 404enemy
7Siters 7Siters
80legs 80legs
a\.pr-cy\.ru
a3logics\.in a3logics\.in
A6-Indexer A6-Indexer
Abonti Abonti
Aboundex Aboundex
aboutthedomain aboutthedomain
Accoona-AI-Agent Accoona-AI-Agent
acebookexternalhit\/
acoon acoon
acrylicapps\.com\/pulp acrylicapps\.com\/pulp
Acunetix Acunetix
@ -49,8 +69,14 @@ AddThis
ADmantX ADmantX
AdminLabs AdminLabs
adressendeutschland adressendeutschland
adreview\/
adscanner adscanner
adstxt-worker
Adstxtaggregator Adstxtaggregator
adstxt\.com
Adyen HttpClient
AffiliateLabz\/
affilimate-puppeteer
agentslug agentslug
AHC AHC
aihit aihit
@ -71,20 +97,23 @@ AndroidDownloadManager
Anemone Anemone
AngleSharp AngleSharp
annotate_google annotate_google
Ant\.com Anthill
Anturis Agent Anturis Agent
Ant\.com
AnyEvent-HTTP\/ AnyEvent-HTTP\/
Apache Ant\/
Apache Droid Apache Droid
Apache OpenOffice Apache OpenOffice
Apache-HttpAsyncClient Apache-HttpAsyncClient
Apache-HttpClient Apache-HttpClient
ApacheBench ApacheBench
Apexoo Apexoo
apimon\.de
APIs-Google APIs-Google
AportWorm\/ AportWorm\/
AppBeat\/ AppBeat\/
AppEngine-Google AppEngine-Google
AppStoreScraperZ AppleSyndication
Aprc\/[0-9] Aprc\/[0-9]
Arachmo Arachmo
arachnode arachnode
@ -92,21 +121,28 @@ Arachnophilia
aria2 aria2
Arukereso Arukereso
asafaweb asafaweb
AskQuickly Asana\/
Ask Jeeves Ask Jeeves
AskQuickly
ASPSeek ASPSeek
Asterias Asterias
Astute Astute
asynchttp asynchttp
Attach Attach
attohttpc
autocite autocite
AutomaticWPTester
Autonomy Autonomy
awin\.com
AWS Security Scanner
axios\/ axios\/
a\.pr-cy\.ru
B-l-i-t-z-B-O-T B-l-i-t-z-B-O-T
Backlink-Ceck Backlink-Ceck
backlink-check backlink-check
BacklinkHttpStatus BacklinkHttpStatus
BackStreet BackStreet
BackupLand
BackWeb BackWeb
Bad-Neighborhood Bad-Neighborhood
Badass Badass
@ -121,6 +157,7 @@ BBBike
BCKLINKS BCKLINKS
BDFetch BDFetch
BegunAdvertising BegunAdvertising
Bewica-security-scan
Bidtellect Bidtellect
BigBozz BigBozz
Bigfoot Bigfoot
@ -130,11 +167,14 @@ BingPreview
binlar binlar
biNu image cacher biNu image cacher
Bitacle Bitacle
Bitrix link preview
biz_Directory biz_Directory
BKCTwitterUnshortener\/
Black Hole Black Hole
Blackboard Safeassign Blackboard Safeassign
BlackWidow BlackWidow
BlockNote\.Net BlockNote\.Net
BlogBridge
Bloglines Bloglines
Bloglovin Bloglovin
BlogPulseLive BlogPulseLive
@ -142,6 +182,7 @@ BlogSearch
Blogtrottr Blogtrottr
BlowFish BlowFish
boitho\.com-dc boitho\.com-dc
Boost\.Beast
BPImageWalker BPImageWalker
Braintree-Webhooks Braintree-Webhooks
Branch Metrics API Branch Metrics API
@ -169,12 +210,15 @@ catexplorador
CC Metadata Scaper CC Metadata Scaper
Cegbfeieh Cegbfeieh
censys censys
centuryb.o.t9[at]gmail.com
Cerberian Drtrs Cerberian Drtrs
CERT\.at-Statistics-Survey CERT\.at-Statistics-Survey
cf-facebook
cg-eye cg-eye
changedetection changedetection
ChangesMeter ChangesMeter
Charlotte Charlotte
chatterino-api-cache
CheckHost CheckHost
checkprivacy checkprivacy
CherryPicker CherryPicker
@ -185,6 +229,7 @@ Chlooe
Chromaxa Chromaxa
CirrusExplorer CirrusExplorer
CISPA Vulnerability Notification CISPA Vulnerability Notification
CISPA Web Analyser
Citoid Citoid
CJNetworkQuality CJNetworkQuality
Clarsentia Clarsentia
@ -192,27 +237,34 @@ clips\.ua\.ac\.be
Cloud mapping Cloud mapping
CloudEndure CloudEndure
CloudFlare-AlwaysOnline CloudFlare-AlwaysOnline
Cloudflare-Healthchecks
Cloudinary Cloudinary
cmcm\.com cmcm\.com
coccoc coccoc
cognitiveseo cognitiveseo
ColdFusion
colly - colly -
CommaFeed CommaFeed
Commons-HttpClient Commons-HttpClient
commonscan commonscan
contactbigdatafr contactbigdatafr
contentkingapp contentkingapp
Contextual Code Sites Explorer
convera convera
CookieReports CookieReports
copyright sheriff copyright sheriff
CopyRightCheck CopyRightCheck
Copyscape Copyscape
cortex\/
Cosmos4j\.feedback Cosmos4j\.feedback
Covario-IDS Covario-IDS
Craw\/
Crescent Crescent
Crowsnest
Criteo Criteo
Crowsnest
CSHttp CSHttp
CSSCheck
Cula\/
curb curb
Curious George Curious George
curl curl
@ -222,12 +274,17 @@ DAP\/NetHTTP
DareBoost DareBoost
DatabaseDriverMysqli DatabaseDriverMysqli
DataCha0s DataCha0s
DatadogSynthetics
Datafeedwatch Datafeedwatch
Datanyze Datanyze
DataparkSearch DataparkSearch
dataprovider dataprovider
DataXu DataXu
Daum(oa)?[ \/][0-9] Daum(oa)?[ \/][0-9]
dBpoweramp
ddline
deeris
delve\.ai
Demon Demon
DeuSu DeuSu
developers\.google\.com\/\+\/web\/snippet\/ developers\.google\.com\/\+\/web\/snippet\/
@ -237,8 +294,8 @@ Digincore
DigitalPebble DigitalPebble
Dirbuster Dirbuster
Discourse Forum Onebox Discourse Forum Onebox
Disqus\/
Dispatch\/ Dispatch\/
Disqus\/
DittoSpyder DittoSpyder
dlvr dlvr
DMBrowser DMBrowser
@ -246,6 +303,8 @@ DNSPod-reporting
docoloc docoloc
Dolphin http client Dolphin http client
DomainAppender DomainAppender
DomainLabz
Domains Project\/
Donuts Content Explorer Donuts Content Explorer
dotMailer content retrieval dotMailer content retrieval
dotSemantic dotSemantic
@ -258,6 +317,8 @@ drupact
Drupal \(\+http:\/\/drupal\.org\/\) Drupal \(\+http:\/\/drupal\.org\/\)
DTS Agent DTS Agent
dubaiindex dubaiindex
DuplexWeb-Google
DynatraceSynthetic
EARTHCOM EARTHCOM
Easy-Thumb Easy-Thumb
EasyDL EasyDL
@ -288,20 +349,22 @@ ExactSearch
ExaleadCloudview ExaleadCloudview
Excel\/ Excel\/
exif exif
ExoRank
Exploratodo Exploratodo
Express WebPictures Express WebPictures
Extreme Picture Finder Extreme Picture Finder
EyeNetIE EyeNetIE
ezooms ezooms
facebookexternalhit facebookexternalhit
facebookexternalua
facebookplatform facebookplatform
fairshare fairshare
Faraday v Faraday v
fasthttp fasthttp
Faveeo Faveeo
Favicon downloader Favicon downloader
faviconkit
faviconarchive faviconarchive
faviconkit
FavOrg FavOrg
Feed Wrangler Feed Wrangler
Feedable\/ Feedable\/
@ -313,7 +376,9 @@ FeedBurner
feeder feeder
Feedly Feedly
FeedshowOnline FeedshowOnline
Feedshow\/
Feedspot Feedspot
FeedViewer\/
Feedwind\/ Feedwind\/
FeedZcollector FeedZcollector
feeltiptop feeltiptop
@ -321,6 +386,8 @@ Fetch API
Fetch\/[0-9] Fetch\/[0-9]
Fever\/[0-9] Fever\/[0-9]
FHscan FHscan
Fiery%20Feeds
Filestack
Fimap Fimap
findlink findlink
findthatfile findthatfile
@ -329,18 +396,24 @@ FlipboardBrowserProxy
FlipboardProxy FlipboardProxy
FlipboardRSS FlipboardRSS
Flock\/ Flock\/
Florienzh\/
fluffy fluffy
Flunky Flunky
flynxapp flynxapp
forensiq forensiq
ForusP
FoundSeoTool FoundSeoTool
http:\/\/www.neomo.de\/ fragFINN\.de
free thumbnails free thumbnails
Freeuploader Freeuploader
FreshRSS
frontman
Funnelback Funnelback
Fuzz Faster U Fool
G-i-g-a-b-o-t G-i-g-a-b-o-t
g00g1e\.net g00g1e\.net
ganarvisitas ganarvisitas
gdnplus\.com
geek-tools geek-tools
Genieo Genieo
GentleSource GentleSource
@ -352,18 +425,24 @@ GetRight
getroot getroot
GetURLInfo\/ GetURLInfo\/
GetWeb GetWeb
Geziyor
Ghost Inspector Ghost Inspector
GigablastOpenSource GigablastOpenSource
GIS-LABS GIS-LABS
github-camo github-camo
GitHub-Hookshot
github\.com github\.com
Go [\d\.]* package http
Go http package Go http package
Go [\d\.]* package http
Go!Zilla
Go-Ahead-Got-It Go-Ahead-Got-It
Go-http-client Go-http-client
Go!Zilla go-mtasts\/
gobuster
gobyus gobyus
Gofeed
gofetch gofetch
Goldfire Server
GomezAgent GomezAgent
gooblog gooblog
Goodzer\/ Goodzer\/
@ -376,24 +455,31 @@ Google Page Speed Insights
Google PP Default Google PP Default
Google Search Console Google Search Console
Google Web Preview Google Web Preview
Google-Ads-Creatives-Assistant
Google-Ads-Overview
Google-Adwords Google-Adwords
Google-Apps-Script Google-Apps-Script
Google-Calendar-Importer Google-Calendar-Importer
Google-HotelAdsVerifier Google-HotelAdsVerifier
Google-HTTP-Java-Client Google-HTTP-Java-Client
Google-InspectionTool
Google-Podcast
Google-Publisher-Plugin Google-Publisher-Plugin
Google-Read-Aloud
Google-SearchByImage Google-SearchByImage
Google-Site-Verification Google-Site-Verification
Google-SMTP-STS
Google-speakr
Google-Structured-Data-Testing-Tool Google-Structured-Data-Testing-Tool
Google-Youtube-Links Google-Transparency-Report
google-xrawler google-xrawler
Google-Youtube-Links
GoogleDocs GoogleDocs
GoogleHC\/ GoogleHC\/
GoogleProber
GoogleProducer GoogleProducer
GoogleSites GoogleSites
Google-Transparency-Report
Gookey Gookey
GoScraper
GoSpotCheck GoSpotCheck
gosquared-thumbnailer gosquared-thumbnailer
Gotit Gotit
@ -419,13 +505,16 @@ Haansoft
hackney\/ hackney\/
Hadi Agent Hadi Agent
HappyApps-WebCheck HappyApps-WebCheck
Hardenize
Hatena Hatena
Havij Havij
HaxerMen
HeadlessChrome HeadlessChrome
HEADMasterSEO HEADMasterSEO
HeartRails_Capture HeartRails_Capture
help@dataminr\.com help@dataminr\.com
heritrix heritrix
Hexometer
historious historious
hkedcity hkedcity
hledejLevne\.cz hledejLevne\.cz
@ -442,28 +531,31 @@ htdig
HTMLparser HTMLparser
htmlyse htmlyse
HTTP Banner Detection HTTP Banner Detection
HTTP_Compression_Test
http_request2
http_requester
http-get http-get
HTTP-Header-Abfrage HTTP-Header-Abfrage
http-kit http-kit
http-request\/ http-request\/
HTTP-Tiny HTTP-Tiny
HTTP::Lite HTTP::Lite
http\.rb\/ http:\/\/www.neomo.de\/
http_get
HttpComponents HttpComponents
httphr httphr
HTTPie
HTTPMon HTTPMon
httpRequest httpRequest
httpscheck httpscheck
httpssites_power httpssites_power
httpunit httpunit
HttpUrlConnection HttpUrlConnection
http\.rb\/
HTTP_Compression_Test
http_get
http_request2
http_requester
httrack httrack
huaweisymantec huaweisymantec
HubSpot HubSpot
HubSpot-Link-Resolver
Humanlinks Humanlinks
i2kconnect\/ i2kconnect\/
Iblog Iblog
@ -474,6 +566,7 @@ IDG Twitter Links Resolver
IDwhois\/ IDwhois\/
Iframely Iframely
igdeSpyder igdeSpyder
iGooglePortal
IlTrovatore IlTrovatore
Image Fetch Image Fetch
Image Sucker Image Sucker
@ -498,13 +591,15 @@ Integrity
integromedb integromedb
Intelliseek Intelliseek
InterGET InterGET
internet_archive
Internet Ninja Internet Ninja
InternetSeer InternetSeer
internetVista monitor internetVista monitor
internetwache
internet_archive
intraVnews intraVnews
IODC IODC
IOI IOI
Inboxb0t
iplabel iplabel
ips-agent ips-agent
IPS\/[0-9] IPS\/[0-9]
@ -516,10 +611,13 @@ isitup\.org
iskanie iskanie
isUp\.li isUp\.li
iThemes Sync\/ iThemes Sync\/
IZaBEE
iZSearch iZSearch
JAHHO JAHHO
janforman janforman
Jaunt\/ Jaunt\/
Java.*outbrain
javelin\.io
Jbrofuzz Jbrofuzz
Jersey\/ Jersey\/
JetCar JetCar
@ -535,6 +633,7 @@ JolokiaPwn
Joomla Joomla
Jorgee Jorgee
JS-Kit JS-Kit
JungleKeyThumbnail
JustView JustView
Kaspersky Lab CFR link resolver Kaspersky Lab CFR link resolver
Kelny\/ Kelny\/
@ -542,6 +641,7 @@ Kerrigan\/
KeyCDN KeyCDN
Keyword Density Keyword Density
Keywords Research Keywords Research
khttp\/
KickFire KickFire
KimonoLabs\/ KimonoLabs\/
Kml-Google Kml-Google
@ -549,11 +649,12 @@ knows\.is
KOCMOHABT KOCMOHABT
kouio kouio
kube-probe kube-probe
kubectl
kulturarw3 kulturarw3
KumKie KumKie
L\.webis
Larbin Larbin
Lavf\/ Lavf\/
leakix\.net
LeechFTP LeechFTP
LeechGet LeechGet
letsencrypt letsencrypt
@ -564,41 +665,50 @@ Libwhisker
libwww libwww
Licorne Licorne
Liferea\/ Liferea\/
Lightspeedsystems
Lighthouse Lighthouse
Lightspeedsystems
Likse Likse
limber\.io
Link Valet Link Valet
link_thumbnailer
LinkAlarm\/ LinkAlarm\/
LinkAnalyser
linkCheck linkCheck
linkdex linkdex
LinkExaminer LinkExaminer
linkfluence linkfluence
linkpeek linkpeek
LinkPreviewGenerator LinkPreview
LinkScan LinkScan
LinksManager LinksManager
LinkTiger LinkTiger
LinkWalker LinkWalker
link_thumbnailer
Lipperhey Lipperhey
Litemage_walker Litemage_walker
livedoor ScreenShot livedoor ScreenShot
LoadImpactRload LoadImpactRload
localsearch-web localsearch-web
LongURL API LongURL API
longurl-r-package
looid\.com
looksystems\.net looksystems\.net
ltx71 ltx71
lua-resty-http lua-resty-http
Lucee \(CFML Engine\)
Lush Http Client
lwp-request lwp-request
lwp-trivial lwp-trivial
LWP::Simple LWP::Simple
lycos lycos
LYT\.SR LYT\.SR
L\.webis
mabontland mabontland
MacOutlook\/
Mag-Net Mag-Net
MagpieRSS MagpieRSS
Mail\.Ru Mail::STS
MailChimp MailChimp
Mail\.Ru
Majestic12 Majestic12
makecontact\/ makecontact\/
Mandrill Mandrill
@ -609,6 +719,8 @@ MarkWatch
Mass Downloader Mass Downloader
masscan\/ masscan\/
Mata Hari Mata Hari
mattermost
Mediametric
Mediapartners-Google Mediapartners-Google
mediawords mediawords
MegaIndex\.ru MegaIndex\.ru
@ -619,28 +731,31 @@ Metaspinner
MetaURI MetaURI
MFC_Tear_Sample MFC_Tear_Sample
Microsearch Microsearch
Microsoft Data Access
Microsoft Office Microsoft Office
Microsoft Outlook Microsoft Outlook
Microsoft Windows Network Diagnostics Microsoft Windows Network Diagnostics
Microsoft-WebDAV-MiniRedir Microsoft-WebDAV-MiniRedir
Microsoft Data Access Microsoft\.Data\.Mashup
MIDown tool MIDown tool
MIIxpc MIIxpc
Mindjet Mindjet
Miniature\.io Miniature\.io
Miniflux Miniflux
mio_httpc
Miro-HttpClient
Mister PiX Mister PiX
mixdata dot com mixdata dot com
mixed-content-scan mixed-content-scan
Mixmax-LinkPreview
mixnode mixnode
Mnogosearch Mnogosearch
mogimogi mogimogi
Mojeek Mojeek
Mojolicious \(Perl\) Mojolicious \(Perl\)
Monit\/ Mollie
monitis monitis
Monitority\/ Monitority\/
Monit\/
montastic montastic
MonTools MonTools
Moreover Moreover
@ -649,6 +764,7 @@ Morning Paper
MovableType MovableType
mowser mowser
Mrcgiguy Mrcgiguy
Mr\.4x3 Powered
MS Web Services Client Protocol MS Web Services Client Protocol
MSFrontPage MSFrontPage
mShots mShots
@ -656,10 +772,13 @@ MuckRack\/
muhstik-scan muhstik-scan
MVAClient MVAClient
MxToolbox\/ MxToolbox\/
myseosnapshot
nagios nagios
Najdi\.si Najdi\.si
Name Intelligence Name Intelligence
NameFo\.com
Nameprotect Nameprotect
nationalarchives
Navroad Navroad
NearSite NearSite
Needle Needle
@ -674,6 +793,8 @@ Netpursual
netresearch netresearch
NetShelter ContentScan NetShelter ContentScan
Netsparker Netsparker
NetSystemsResearch
nettle
NetTrack NetTrack
Netvibes Netvibes
NetZIP NetZIP
@ -686,22 +807,26 @@ newsme
newspaper\/ newspaper\/
Nexgate Ruby Client Nexgate Ruby Client
NG-Search NG-Search
nghttp2
Nibbler Nibbler
NICErsPRO NICErsPRO
NihilScio
Nikto Nikto
nineconnections nineconnections
NLNZ_IAHarvester NLNZ_IAHarvester
Nmap Scripting Engine Nmap Scripting Engine
node-fetch
node-superagent node-superagent
node-urllib node-urllib
node\.io
Nodemeter Nodemeter
NodePing NodePing
node\.io
nominet\.org\.uk nominet\.org\.uk
nominet\.uk nominet\.uk
Norton-Safeweb Norton-Safeweb
Notifixious Notifixious
notifyninja notifyninja
NotionEmbedder
nuhk nuhk
nutch nutch
Nuzzel Nuzzel
@ -709,39 +834,45 @@ nWormFeedFinder
nyawc\/ nyawc\/
Nymesis Nymesis
NYU NYU
Observatory\/
Ocelli\/ Ocelli\/
Octopus Octopus
oegp oegp
Offline Explorer Offline Explorer
Offline Navigator Offline Navigator
og-scraper OgScrper
okhttp okhttp
omgili omgili
OMSC OMSC
Online Domain Tools Online Domain Tools
Open Source RSS
OpenCalaisSemanticProxy OpenCalaisSemanticProxy
Openfind Openfind
OpenLinkProfiler OpenLinkProfiler
Openstat\/ Openstat\/
OpenVAS OpenVAS
OPPO A33
Optimizer Optimizer
Orbiter Orbiter
OrgProbe\/ OrgProbe\/
orion-semantics orion-semantics
Outlook-Express Outlook-Express
Outlook-iOS Outlook-iOS
ow\.ly
Owler Owler
Owlin
ownCloud News ownCloud News
ow\.ly
OxfordCloudService OxfordCloudService
Page Valet
page_verifier
page scorer page scorer
Page Valet
page2rss page2rss
PageFreezer
PageGrabber PageGrabber
PagePeeker PagePeeker
PageScorer PageScorer
Pagespeed\/ Pagespeed\/
PageThing
page_verifier
Panopta Panopta
panscient panscient
Papa Foto Papa Foto
@ -750,6 +881,7 @@ Pavuk
PayPal IPN PayPal IPN
pcBrowser pcBrowser
Pcore-HTTP Pcore-HTTP
PDF24 URL To PDF
Pearltrees Pearltrees
PECL::HTTP PECL::HTTP
peerindex peerindex
@ -759,44 +891,52 @@ Perlu -
PhantomJS Screenshoter PhantomJS Screenshoter
PhantomJS\/ PhantomJS\/
Photon\/ Photon\/
php-requests
phpservermon phpservermon
Pi-Monster Pi-Monster
Picscout Picscout
Picsearch Picsearch
PictureFinder PictureFinder
Pimonster Pimonster
ping\.blo\.gs
Pingability Pingability
PingAdmin\.Ru PingAdmin\.Ru
Pingdom Pingdom
Pingoscope Pingoscope
PingSpot PingSpot
ping\.blo\.gs
pinterest\.com pinterest\.com
Pixray Pixray
Pizilla Pizilla
Plagger\/ Plagger\/
Pleroma
Ploetz \+ Zeller Ploetz \+ Zeller
Plukkie Plukkie
plumanalytics plumanalytics
PocketImageCache PocketImageCache
PocketParser PocketParser
Pockey Pockey
PodcastAddict\/
POE-Component-Client-HTTP POE-Component-Client-HTTP
Polymail\/ Polymail\/
Pompos Pompos
Porkbun Porkbun
Port Monitor Port Monitor
postano postano
postfix-mta-sts-resolver
PostmanRuntime PostmanRuntime
postplanner\.com
PostPost PostPost
postrank postrank
PowerPoint\/ PowerPoint\/
Prebid
Prerender
Priceonomics Analysis Engine Priceonomics Analysis Engine
PrintFriendly PrintFriendly
PritTorrent PritTorrent
Prlog Prlog
probethenet probethenet
Project 25499 Project ?25499
Project-Resonance
prospectb2b prospectb2b
Protopage Protopage
ProWebWalker ProWebWalker
@ -805,9 +945,9 @@ PRTG Network Monitor
pshtt, https scanning pshtt, https scanning
PTST PTST
PTST\/[0-9]+ PTST\/[0-9]+
Pulsepoint XT3 web scraper
Pump Pump
Python-httplib2 Python-httplib2
python-httpx
python-requests python-requests
Python-urllib Python-urllib
Qirina Hurdler Qirina Hurdler
@ -817,12 +957,17 @@ Qseero
Qualidator Qualidator
QueryN Metasearch QueryN Metasearch
queuedriver queuedriver
quic-go-HTTP\/
QuiteRSS
Quora Link Preview Quora Link Preview
Qwantify Qwantify
Radian6 Radian6
RadioPublicImageResizer
Railgun\/
RankActive RankActive
RankFlex RankFlex
RankSonicSiteAuditor RankSonicSiteAuditor
RapidLoad\/
Re-re Studio Re-re Studio
ReactorNetty ReactorNetty
Readability Readability
@ -833,6 +978,7 @@ Recorder
RecurPost\/ RecurPost\/
redback\/ redback\/
ReederForMac ReederForMac
Reeder\/
ReGet ReGet
RepoMonkey RepoMonkey
request\.js request\.js
@ -845,38 +991,44 @@ Robosourcer
Robozilla Robozilla
ROI Hunter ROI Hunter
RPT-HTTPClient RPT-HTTPClient
RSSMix\/
RSSOwl RSSOwl
RyowlEngine
safe-agent-scanner safe-agent-scanner
SalesIntelligent SalesIntelligent
Saleslift Saleslift
Sendsay\.Ru SAP NetWeaver Application Server
SauceNAO SauceNAO
SBIder SBIder
sc-downloader
scalaj-http scalaj-http
scan\.lol Scamadviser-Frontend
ScanAlert ScanAlert
scan\.lol
Scoop Scoop
scooter scooter
ScopeContentAG-HTTP-Client
ScoutJet ScoutJet
ScoutURLMonitor ScoutURLMonitor
ScrapeBox Page Scanner ScrapeBox Page Scanner
SimpleScraper
Scrapy Scrapy
Screaming Screaming
ScreenShotService ScreenShotService
Scrubby Scrubby
Scrutiny\/ Scrutiny\/
search\.thunderstone
Search37 Search37
searchenginepromotionhelp searchenginepromotionhelp
Searchestate Searchestate
SearchExpress SearchExpress
SearchSight SearchSight
SearchWP
search\.thunderstone
Seeker Seeker
semanticdiscovery semanticdiscovery
semanticjuice semanticjuice
Semiocast HTTP client Semiocast HTTP client
Semrush Semrush
Sendsay\.Ru
sentry\/ sentry\/
SEO Browser SEO Browser
Seo Servis Seo Servis
@ -885,11 +1037,13 @@ seo4ajax
Seobility Seobility
SEOCentro SEOCentro
SeoCheck SeoCheck
seocompany
SEOkicks SEOkicks
SEOlizer
Seomoz Seomoz
SEOprofiler SEOprofiler
SEOsearch
seoscanners seoscanners
SEOsearch
seositecheckup seositecheckup
SEOstats SEOstats
servernfo servernfo
@ -900,14 +1054,16 @@ Shodan
Shoppimon Shoppimon
ShopWiki ShopWiki
ShortLinkTranslate ShortLinkTranslate
shortURL lengthener
shrinktheweb shrinktheweb
Sideqik Sideqik
Siege
SimplePie SimplePie
SimplyFast SimplyFast
Siphon Siphon
SISTRIX SISTRIX
Site-Shot\/
Site Sucker Site Sucker
Site-Shot\/
Site24x7 Site24x7
SiteBar SiteBar
Sitebeam Sitebeam
@ -928,6 +1084,7 @@ Sitevigil
sitexy\.com sitexy\.com
SkypeUriPreview SkypeUriPreview
Slack\/ Slack\/
sli-systems\.com
slider\.com slider\.com
slurp slurp
SlySearch SlySearch
@ -950,9 +1107,11 @@ SpaceBison
SpamExperts SpamExperts
Spammen Spammen
Spanner Spanner
Spawning-AI
spaziodati spaziodati
SPDYCheck SPDYCheck
Specificfeeds Specificfeeds
SpeedKit
speedy speedy
SPEng SPEng
Spinn3r Spinn3r
@ -966,9 +1125,11 @@ SSL Labs
ssl-tools ssl-tools
StackRambler StackRambler
Statastico\/ Statastico\/
Statically-
StatusCake StatusCake
Steeler Steeler
Stratagems Kumo Stratagems Kumo
Stripe\/
Stroke\.cz Stroke\.cz
StudioFACA StudioFACA
StumbleUpon StumbleUpon
@ -978,9 +1139,11 @@ summify
SuperHTTP SuperHTTP
Surphace Scout Surphace Scout
Suzuran Suzuran
SwiteScraper swcd
Symfony BrowserKit Symfony BrowserKit
Symfony2 BrowserKit Symfony2 BrowserKit
Synapse\/
Syndirella\/
SynHttpClient-Built SynHttpClient-Built
Sysomos Sysomos
sysscan sysscan
@ -990,6 +1153,8 @@ tAkeOut
Tarantula\/ Tarantula\/
Taringa UGC Taringa UGC
TarmotGezgin TarmotGezgin
tchelebi\.io
techiaith\.cymru
Teleport Teleport
Telesoft Telesoft
Telesphoreo Telesphoreo
@ -1000,13 +1165,15 @@ terrainformatica
Test Certificate Info Test Certificate Info
testuri testuri
Tetrahedron Tetrahedron
TextRazor Downloader
The Drop Reaper The Drop Reaper
The Expert HTML Source Viewer The Expert HTML Source Viewer
The Knowledge AI
The Intraformant The Intraformant
The Knowledge AI
theinternetrules theinternetrules
TheNomad TheNomad
Thinklab Thinklab
Thumbor
Thumbshots Thumbshots
ThumbSniper ThumbSniper
timewe\.net timewe\.net
@ -1040,9 +1207,11 @@ Typhoeus
ubermetrics-technologies ubermetrics-technologies
uclassify uclassify
UdmSearch UdmSearch
ultimate_sitemap_parser
unchaos unchaos
unirest-java unirest-java
UniversalFeedParser UniversalFeedParser
unshortenit
Unshorten\.It Unshorten\.It
Untiny Untiny
UnwindFetchor UnwindFetchor
@ -1050,8 +1219,8 @@ updated
updown\.io daemon updown\.io daemon
Upflow Upflow
Uptimia Uptimia
Urlcheckr
URL Verifier URL Verifier
Urlcheckr
URLitor URLitor
urlresolver urlresolver
Urlstat Urlstat
@ -1059,15 +1228,16 @@ URLTester
UrlTrends Ranking Updater UrlTrends Ranking Updater
URLy Warning URLy Warning
URLy\.Warning URLy\.Warning
URL\/Emacs
Vacuum Vacuum
Vagabondo Vagabondo
VB Project VB Project
vBSEO vBSEO
VCI VCI
via ggpht\.com GoogleImageProxy via ggpht\.com GoogleImageProxy
VidibleScraper
Virusdie Virusdie
visionutils visionutils
Visual Rights Group
vkShare vkShare
VoidEYE VoidEYE
Voil Voil
@ -1078,19 +1248,18 @@ VSB-TUO\/
Vulnbusters Meter Vulnbusters Meter
VYU2 VYU2
w3af\.org w3af\.org
W3C_Unicorn
W3C-checklink W3C-checklink
W3C-mobileOK W3C-mobileOK
W3C_Unicorn
WAC-OFU WAC-OFU
Wallpapers\/[0-9]+ WakeletLinkExpander
WallpapersHD WallpapersHD
Wallpapers\/[0-9]+
wangling wangling
Wappalyzer Wappalyzer
WatchMouse WatchMouse
WbSrch\/ WbSrch\/
WDT\.io WDT\.io
web-capture\.net
Web-sniffer
Web Auto Web Auto
Web Collage Web Collage
Web Enhancer Web Enhancer
@ -1098,7 +1267,10 @@ Web Fetch
Web Fuck Web Fuck
Web Pix Web Pix
Web Sauger Web Sauger
Web spyder
Web Sucker Web Sucker
web-capture\.net
Web-sniffer
Webalta Webalta
Webauskunft Webauskunft
WebAuto WebAuto
@ -1136,8 +1308,9 @@ WebSniffer
Webster Webster
WebStripper WebStripper
WebSucker WebSucker
Webthumb\/ webtech\/
WebThumbnail WebThumbnail
Webthumb\/
WebWhacker WebWhacker
WebZIP WebZIP
WeLikeLinks WeLikeLinks
@ -1146,24 +1319,31 @@ WeSEE
wf84 wf84
Wfuzz\/ Wfuzz\/
wget wget
WhatCMS
WhatsApp WhatsApp
WhatsMyIP WhatsMyIP
WhatWeb WhatWeb
WhereGoes\? WhereGoes\?
Whibse Whibse
WhoAPI\/
WhoRunsCoinHive WhoRunsCoinHive
Whynder Magnet Whynder Magnet
Windows-RSS-Platform Windows-RSS-Platform
WinHttp-Autoproxy-Service
WinHTTP\/
WinPodder WinPodder
wkhtmlto wkhtmlto
wmtips wmtips
Woko Woko
Wolfram HTTPClient
woorankreview woorankreview
Word\/
WordPress\/ WordPress\/
WordupinfoSearch WordupinfoSearch
Word\/
worldping-api
wotbox wotbox
WP Engine Install Performance API WP Engine Install Performance API
WP Rocket
wpif wpif
wprecon\.com survey wprecon\.com survey
WPScan WPScan
@ -1173,20 +1353,21 @@ WWW-Collector-E
WWW-Mechanize WWW-Mechanize
WWW::Document WWW::Document
WWW::Mechanize WWW::Mechanize
www\.monitor\.us
WWWOFFLE WWWOFFLE
www\.monitor\.us
x09Mozilla x09Mozilla
x22Mozilla x22Mozilla
XaxisSemanticsClassifier XaxisSemanticsClassifier
XenForo\/
Xenu Link Sleuth Xenu Link Sleuth
XING-contenttabreceiver XING-contenttabreceiver
xpymep([0-9]?)\.exe xpymep([0-9]?)\.exe
Y!J-(ASR|BSC) Y!J-[A-Z][A-Z][A-Z]
Y\!J-BRW
Yaanb Yaanb
yacy yacy
Yahoo Link Preview Yahoo Link Preview
YahooCacheSystem YahooCacheSystem
YahooMailProxy
YahooYSMcm YahooYSMcm
YandeG YandeG
Yandex(?!Search) Yandex(?!Search)
@ -1194,6 +1375,7 @@ yanga
yeti yeti
Yo-yo Yo-yo
Yoleo Consumer Yoleo Consumer
yomins\.com
yoogliFetchAgent yoogliFetchAgent
YottaaMonitor YottaaMonitor
Your-Website-Sucks Your-Website-Sucks
@ -1205,13 +1387,15 @@ Zade
Zao Zao
Zauba Zauba
Zemanta Aggregator Zemanta Aggregator
Zend_Http_Client
Zend\\Http\\Client Zend\\Http\\Client
Zend_Http_Client
Zermelo Zermelo
Zeus Zeus
zgrab zgrab
ZnajdzFoto ZnajdzFoto
ZnHTTP
Zombie\.js Zombie\.js
Zoom\.Mac Zoom\.Mac
ZoteroTranslationServer
ZyBorg ZyBorg
[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer) [a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)

2
blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json vendored
View file

@ -1 +1 @@
["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT",";"] ["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"]

4
blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt vendored
View file

@ -45,4 +45,6 @@ cubot
; IDbot ; IDbot
; ID bot ; ID bot
; POWER BOT ; POWER BOT
; OCTOPUS-CORE
htc_botdugls
super\/\d+\/Android\/\d+

27
blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php vendored
View file

@ -20,9 +20,9 @@ class CrawlerDetect
/** /**
* The user agent. * The user agent.
* *
* @var null * @var string|null
*/ */
protected $userAgent = null; protected $userAgent;
/** /**
* Headers that contain a user agent. * Headers that contain a user agent.
@ -138,7 +138,7 @@ class CrawlerDetect
/** /**
* Set the user agent. * Set the user agent.
* *
* @param string $userAgent * @param string|null $userAgent
*/ */
public function setUserAgent($userAgent) public function setUserAgent($userAgent)
{ {
@ -165,20 +165,14 @@ class CrawlerDetect
$agent = trim(preg_replace( $agent = trim(preg_replace(
"/{$this->compiledExclusions}/i", "/{$this->compiledExclusions}/i",
'', '',
$userAgent ?: $this->userAgent $userAgent ?: $this->userAgent ?: ''
)); ));
if ($agent == '') { if ($agent === '') {
return false; return false;
} }
$result = preg_match("/{$this->compiledRegex}/i", $agent, $matches); return (bool) preg_match("/{$this->compiledRegex}/i", $agent, $this->matches);
if ($matches) {
$this->matches = $matches;
}
return (bool) $result;
} }
/** /**
@ -190,4 +184,13 @@ class CrawlerDetect
{ {
return isset($this->matches[0]) ? $this->matches[0] : null; return isset($this->matches[0]) ? $this->matches[0] : null;
} }
/**
* @return string|null
*/
public function getUserAgent()
{
return $this->userAgent;
}
} }

302
blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php vendored
View file

@ -19,30 +19,49 @@ class Crawlers extends AbstractProvider
* @var array * @var array
*/ */
protected $data = array( protected $data = array(
'.*Java.*outbrain',
' YLT', ' YLT',
'^Aether',
'^Amazon Simple Notification Service Agent$',
'^Amazon-Route53-Health-Check-Service',
'^b0t$', '^b0t$',
'^bluefish ', '^bluefish ',
'^Calypso v\/', '^Calypso v\/',
'^COMODO DCV', '^COMODO DCV',
'^Corax',
'^DangDang', '^DangDang',
'^DavClnt', '^DavClnt',
'^DHSH',
'^docker\/[0-9]',
'^Expanse',
'^FDM ', '^FDM ',
'^git\/', '^git\/',
'^Goose\/', '^Goose\/',
'^Grabber', '^Grabber',
'^Gradle\/',
'^HTTPClient\/', '^HTTPClient\/',
'^HTTPing',
'^Java\/', '^Java\/',
'^Jeode\/', '^Jeode\/',
'^Jetty\/', '^Jetty\/',
'^Mail\/', '^Mail\/',
'^Mget', '^Mget',
'^Microsoft URL Control', '^Microsoft URL Control',
'^Mikrotik\/',
'^Netlab360',
'^NG\/[0-9\.]', '^NG\/[0-9\.]',
'^NING\/', '^NING\/',
'^PHP\/[0-9]', '^npm\/',
'^Nuclei',
'^PHP-AYMAPI\/',
'^PHP\/',
'^pip\/',
'^pnpm\/',
'^RMA\/', '^RMA\/',
'^Ruby|Ruby\/[0-9]', '^Ruby|Ruby\/[0-9]',
'^Swurl ',
'^TLS tester ',
'^twine\/',
'^ureq',
'^VSE\/[0-9]', '^VSE\/[0-9]',
'^WordPress\.com', '^WordPress\.com',
'^XRL\/[0-9]', '^XRL\/[0-9]',
@ -50,17 +69,18 @@ class Crawlers extends AbstractProvider
'008\/', '008\/',
'13TABS', '13TABS',
'192\.comAgent', '192\.comAgent',
'2GDPR\/',
'2ip\.ru', '2ip\.ru',
'404enemy', '404enemy',
'7Siters', '7Siters',
'80legs', '80legs',
'a\.pr-cy\.ru',
'a3logics\.in', 'a3logics\.in',
'A6-Indexer', 'A6-Indexer',
'Abonti', 'Abonti',
'Aboundex', 'Aboundex',
'aboutthedomain', 'aboutthedomain',
'Accoona-AI-Agent', 'Accoona-AI-Agent',
'acebookexternalhit\/',
'acoon', 'acoon',
'acrylicapps\.com\/pulp', 'acrylicapps\.com\/pulp',
'Acunetix', 'Acunetix',
@ -70,8 +90,14 @@ class Crawlers extends AbstractProvider
'ADmantX', 'ADmantX',
'AdminLabs', 'AdminLabs',
'adressendeutschland', 'adressendeutschland',
'adreview\/',
'adscanner', 'adscanner',
'adstxt-worker',
'Adstxtaggregator', 'Adstxtaggregator',
'adstxt\.com',
'Adyen HttpClient',
'AffiliateLabz\/',
'affilimate-puppeteer',
'agentslug', 'agentslug',
'AHC', 'AHC',
'aihit', 'aihit',
@ -92,20 +118,23 @@ class Crawlers extends AbstractProvider
'Anemone', 'Anemone',
'AngleSharp', 'AngleSharp',
'annotate_google', 'annotate_google',
'Ant\.com', 'Anthill',
'Anturis Agent', 'Anturis Agent',
'Ant\.com',
'AnyEvent-HTTP\/', 'AnyEvent-HTTP\/',
'Apache Ant\/',
'Apache Droid', 'Apache Droid',
'Apache OpenOffice', 'Apache OpenOffice',
'Apache-HttpAsyncClient', 'Apache-HttpAsyncClient',
'Apache-HttpClient', 'Apache-HttpClient',
'ApacheBench', 'ApacheBench',
'Apexoo', 'Apexoo',
'apimon\.de',
'APIs-Google', 'APIs-Google',
'AportWorm\/', 'AportWorm\/',
'AppBeat\/', 'AppBeat\/',
'AppEngine-Google', 'AppEngine-Google',
'AppStoreScraperZ', 'AppleSyndication',
'Aprc\/[0-9]', 'Aprc\/[0-9]',
'Arachmo', 'Arachmo',
'arachnode', 'arachnode',
@ -113,21 +142,28 @@ class Crawlers extends AbstractProvider
'aria2', 'aria2',
'Arukereso', 'Arukereso',
'asafaweb', 'asafaweb',
'AskQuickly', 'Asana\/',
'Ask Jeeves', 'Ask Jeeves',
'AskQuickly',
'ASPSeek', 'ASPSeek',
'Asterias', 'Asterias',
'Astute', 'Astute',
'asynchttp', 'asynchttp',
'Attach', 'Attach',
'attohttpc',
'autocite', 'autocite',
'AutomaticWPTester',
'Autonomy', 'Autonomy',
'awin\.com',
'AWS Security Scanner',
'axios\/', 'axios\/',
'a\.pr-cy\.ru',
'B-l-i-t-z-B-O-T', 'B-l-i-t-z-B-O-T',
'Backlink-Ceck', 'Backlink-Ceck',
'backlink-check', 'backlink-check',
'BacklinkHttpStatus', 'BacklinkHttpStatus',
'BackStreet', 'BackStreet',
'BackupLand',
'BackWeb', 'BackWeb',
'Bad-Neighborhood', 'Bad-Neighborhood',
'Badass', 'Badass',
@ -142,6 +178,7 @@ class Crawlers extends AbstractProvider
'BCKLINKS', 'BCKLINKS',
'BDFetch', 'BDFetch',
'BegunAdvertising', 'BegunAdvertising',
'Bewica-security-scan',
'Bidtellect', 'Bidtellect',
'BigBozz', 'BigBozz',
'Bigfoot', 'Bigfoot',
@ -151,11 +188,14 @@ class Crawlers extends AbstractProvider
'binlar', 'binlar',
'biNu image cacher', 'biNu image cacher',
'Bitacle', 'Bitacle',
'Bitrix link preview',
'biz_Directory', 'biz_Directory',
'BKCTwitterUnshortener\/',
'Black Hole', 'Black Hole',
'Blackboard Safeassign', 'Blackboard Safeassign',
'BlackWidow', 'BlackWidow',
'BlockNote\.Net', 'BlockNote\.Net',
'BlogBridge',
'Bloglines', 'Bloglines',
'Bloglovin', 'Bloglovin',
'BlogPulseLive', 'BlogPulseLive',
@ -163,6 +203,7 @@ class Crawlers extends AbstractProvider
'Blogtrottr', 'Blogtrottr',
'BlowFish', 'BlowFish',
'boitho\.com-dc', 'boitho\.com-dc',
'Boost\.Beast',
'BPImageWalker', 'BPImageWalker',
'Braintree-Webhooks', 'Braintree-Webhooks',
'Branch Metrics API', 'Branch Metrics API',
@ -190,12 +231,15 @@ class Crawlers extends AbstractProvider
'CC Metadata Scaper', 'CC Metadata Scaper',
'Cegbfeieh', 'Cegbfeieh',
'censys', 'censys',
'centuryb.o.t9[at]gmail.com',
'Cerberian Drtrs', 'Cerberian Drtrs',
'CERT\.at-Statistics-Survey', 'CERT\.at-Statistics-Survey',
'cf-facebook',
'cg-eye', 'cg-eye',
'changedetection', 'changedetection',
'ChangesMeter', 'ChangesMeter',
'Charlotte', 'Charlotte',
'chatterino-api-cache',
'CheckHost', 'CheckHost',
'checkprivacy', 'checkprivacy',
'CherryPicker', 'CherryPicker',
@ -206,6 +250,7 @@ class Crawlers extends AbstractProvider
'Chromaxa', 'Chromaxa',
'CirrusExplorer', 'CirrusExplorer',
'CISPA Vulnerability Notification', 'CISPA Vulnerability Notification',
'CISPA Web Analyser',
'Citoid', 'Citoid',
'CJNetworkQuality', 'CJNetworkQuality',
'Clarsentia', 'Clarsentia',
@ -213,27 +258,34 @@ class Crawlers extends AbstractProvider
'Cloud mapping', 'Cloud mapping',
'CloudEndure', 'CloudEndure',
'CloudFlare-AlwaysOnline', 'CloudFlare-AlwaysOnline',
'Cloudflare-Healthchecks',
'Cloudinary', 'Cloudinary',
'cmcm\.com', 'cmcm\.com',
'coccoc', 'coccoc',
'cognitiveseo', 'cognitiveseo',
'ColdFusion',
'colly -', 'colly -',
'CommaFeed', 'CommaFeed',
'Commons-HttpClient', 'Commons-HttpClient',
'commonscan', 'commonscan',
'contactbigdatafr', 'contactbigdatafr',
'contentkingapp', 'contentkingapp',
'Contextual Code Sites Explorer',
'convera', 'convera',
'CookieReports', 'CookieReports',
'copyright sheriff', 'copyright sheriff',
'CopyRightCheck', 'CopyRightCheck',
'Copyscape', 'Copyscape',
'cortex\/',
'Cosmos4j\.feedback', 'Cosmos4j\.feedback',
'Covario-IDS', 'Covario-IDS',
'Craw\/',
'Crescent', 'Crescent',
'Crowsnest',
'Criteo', 'Criteo',
'Crowsnest',
'CSHttp', 'CSHttp',
'CSSCheck',
'Cula\/',
'curb', 'curb',
'Curious George', 'Curious George',
'curl', 'curl',
@ -243,12 +295,17 @@ class Crawlers extends AbstractProvider
'DareBoost', 'DareBoost',
'DatabaseDriverMysqli', 'DatabaseDriverMysqli',
'DataCha0s', 'DataCha0s',
'DatadogSynthetics',
'Datafeedwatch', 'Datafeedwatch',
'Datanyze', 'Datanyze',
'DataparkSearch', 'DataparkSearch',
'dataprovider', 'dataprovider',
'DataXu', 'DataXu',
'Daum(oa)?[ \/][0-9]', 'Daum(oa)?[ \/][0-9]',
'dBpoweramp',
'ddline',
'deeris',
'delve\.ai',
'Demon', 'Demon',
'DeuSu', 'DeuSu',
'developers\.google\.com\/\+\/web\/snippet\/', 'developers\.google\.com\/\+\/web\/snippet\/',
@ -258,8 +315,8 @@ class Crawlers extends AbstractProvider
'DigitalPebble', 'DigitalPebble',
'Dirbuster', 'Dirbuster',
'Discourse Forum Onebox', 'Discourse Forum Onebox',
'Disqus\/',
'Dispatch\/', 'Dispatch\/',
'Disqus\/',
'DittoSpyder', 'DittoSpyder',
'dlvr', 'dlvr',
'DMBrowser', 'DMBrowser',
@ -267,6 +324,8 @@ class Crawlers extends AbstractProvider
'docoloc', 'docoloc',
'Dolphin http client', 'Dolphin http client',
'DomainAppender', 'DomainAppender',
'DomainLabz',
'Domains Project\/',
'Donuts Content Explorer', 'Donuts Content Explorer',
'dotMailer content retrieval', 'dotMailer content retrieval',
'dotSemantic', 'dotSemantic',
@ -279,6 +338,8 @@ class Crawlers extends AbstractProvider
'Drupal \(\+http:\/\/drupal\.org\/\)', 'Drupal \(\+http:\/\/drupal\.org\/\)',
'DTS Agent', 'DTS Agent',
'dubaiindex', 'dubaiindex',
'DuplexWeb-Google',
'DynatraceSynthetic',
'EARTHCOM', 'EARTHCOM',
'Easy-Thumb', 'Easy-Thumb',
'EasyDL', 'EasyDL',
@ -309,20 +370,22 @@ class Crawlers extends AbstractProvider
'ExaleadCloudview', 'ExaleadCloudview',
'Excel\/', 'Excel\/',
'exif', 'exif',
'ExoRank',
'Exploratodo', 'Exploratodo',
'Express WebPictures', 'Express WebPictures',
'Extreme Picture Finder', 'Extreme Picture Finder',
'EyeNetIE', 'EyeNetIE',
'ezooms', 'ezooms',
'facebookexternalhit', 'facebookexternalhit',
'facebookexternalua',
'facebookplatform', 'facebookplatform',
'fairshare', 'fairshare',
'Faraday v', 'Faraday v',
'fasthttp', 'fasthttp',
'Faveeo', 'Faveeo',
'Favicon downloader', 'Favicon downloader',
'faviconkit',
'faviconarchive', 'faviconarchive',
'faviconkit',
'FavOrg', 'FavOrg',
'Feed Wrangler', 'Feed Wrangler',
'Feedable\/', 'Feedable\/',
@ -334,7 +397,9 @@ class Crawlers extends AbstractProvider
'feeder', 'feeder',
'Feedly', 'Feedly',
'FeedshowOnline', 'FeedshowOnline',
'Feedshow\/',
'Feedspot', 'Feedspot',
'FeedViewer\/',
'Feedwind\/', 'Feedwind\/',
'FeedZcollector', 'FeedZcollector',
'feeltiptop', 'feeltiptop',
@ -342,6 +407,8 @@ class Crawlers extends AbstractProvider
'Fetch\/[0-9]', 'Fetch\/[0-9]',
'Fever\/[0-9]', 'Fever\/[0-9]',
'FHscan', 'FHscan',
'Fiery%20Feeds',
'Filestack',
'Fimap', 'Fimap',
'findlink', 'findlink',
'findthatfile', 'findthatfile',
@ -350,18 +417,24 @@ class Crawlers extends AbstractProvider
'FlipboardProxy', 'FlipboardProxy',
'FlipboardRSS', 'FlipboardRSS',
'Flock\/', 'Flock\/',
'Florienzh\/',
'fluffy', 'fluffy',
'Flunky', 'Flunky',
'flynxapp', 'flynxapp',
'forensiq', 'forensiq',
'ForusP',
'FoundSeoTool', 'FoundSeoTool',
'http:\/\/www.neomo.de\/', //'Francis [Bot]' 'fragFINN\.de',
'free thumbnails', 'free thumbnails',
'Freeuploader', 'Freeuploader',
'FreshRSS',
'frontman',
'Funnelback', 'Funnelback',
'Fuzz Faster U Fool',
'G-i-g-a-b-o-t', 'G-i-g-a-b-o-t',
'g00g1e\.net', 'g00g1e\.net',
'ganarvisitas', 'ganarvisitas',
'gdnplus\.com',
'geek-tools', 'geek-tools',
'Genieo', 'Genieo',
'GentleSource', 'GentleSource',
@ -373,18 +446,24 @@ class Crawlers extends AbstractProvider
'getroot', 'getroot',
'GetURLInfo\/', 'GetURLInfo\/',
'GetWeb', 'GetWeb',
'Geziyor',
'Ghost Inspector', 'Ghost Inspector',
'GigablastOpenSource', 'GigablastOpenSource',
'GIS-LABS', 'GIS-LABS',
'github-camo', 'github-camo',
'GitHub-Hookshot',
'github\.com', 'github\.com',
'Go [\d\.]* package http',
'Go http package', 'Go http package',
'Go [\d\.]* package http',
'Go!Zilla',
'Go-Ahead-Got-It', 'Go-Ahead-Got-It',
'Go-http-client', 'Go-http-client',
'Go!Zilla', 'go-mtasts\/',
'gobuster',
'gobyus', 'gobyus',
'Gofeed',
'gofetch', 'gofetch',
'Goldfire Server',
'GomezAgent', 'GomezAgent',
'gooblog', 'gooblog',
'Goodzer\/', 'Goodzer\/',
@ -397,24 +476,31 @@ class Crawlers extends AbstractProvider
'Google PP Default', 'Google PP Default',
'Google Search Console', 'Google Search Console',
'Google Web Preview', 'Google Web Preview',
'Google-Ads-Creatives-Assistant',
'Google-Ads-Overview',
'Google-Adwords', 'Google-Adwords',
'Google-Apps-Script', 'Google-Apps-Script',
'Google-Calendar-Importer', 'Google-Calendar-Importer',
'Google-HotelAdsVerifier', 'Google-HotelAdsVerifier',
'Google-HTTP-Java-Client', 'Google-HTTP-Java-Client',
'Google-InspectionTool',
'Google-Podcast',
'Google-Publisher-Plugin', 'Google-Publisher-Plugin',
'Google-Read-Aloud',
'Google-SearchByImage', 'Google-SearchByImage',
'Google-Site-Verification', 'Google-Site-Verification',
'Google-SMTP-STS',
'Google-speakr',
'Google-Structured-Data-Testing-Tool', 'Google-Structured-Data-Testing-Tool',
'Google-Youtube-Links', 'Google-Transparency-Report',
'google-xrawler', 'google-xrawler',
'Google-Youtube-Links',
'GoogleDocs', 'GoogleDocs',
'GoogleHC\/', 'GoogleHC\/',
'GoogleProber',
'GoogleProducer', 'GoogleProducer',
'GoogleSites', 'GoogleSites',
'Google-Transparency-Report',
'Gookey', 'Gookey',
'GoScraper',
'GoSpotCheck', 'GoSpotCheck',
'gosquared-thumbnailer', 'gosquared-thumbnailer',
'Gotit', 'Gotit',
@ -440,13 +526,16 @@ class Crawlers extends AbstractProvider
'hackney\/', 'hackney\/',
'Hadi Agent', 'Hadi Agent',
'HappyApps-WebCheck', 'HappyApps-WebCheck',
'Hardenize',
'Hatena', 'Hatena',
'Havij', 'Havij',
'HaxerMen',
'HeadlessChrome', 'HeadlessChrome',
'HEADMasterSEO', 'HEADMasterSEO',
'HeartRails_Capture', 'HeartRails_Capture',
'help@dataminr\.com', 'help@dataminr\.com',
'heritrix', 'heritrix',
'Hexometer',
'historious', 'historious',
'hkedcity', 'hkedcity',
'hledejLevne\.cz', 'hledejLevne\.cz',
@ -463,28 +552,31 @@ class Crawlers extends AbstractProvider
'HTMLparser', 'HTMLparser',
'htmlyse', 'htmlyse',
'HTTP Banner Detection', 'HTTP Banner Detection',
'HTTP_Compression_Test',
'http_request2',
'http_requester',
'http-get', 'http-get',
'HTTP-Header-Abfrage', 'HTTP-Header-Abfrage',
'http-kit', 'http-kit',
'http-request\/', 'http-request\/',
'HTTP-Tiny', 'HTTP-Tiny',
'HTTP::Lite', 'HTTP::Lite',
'http\.rb\/', 'http:\/\/www.neomo.de\/', //'Francis [Bot]'
'http_get',
'HttpComponents', 'HttpComponents',
'httphr', 'httphr',
'HTTPie',
'HTTPMon', 'HTTPMon',
'httpRequest', 'httpRequest',
'httpscheck', 'httpscheck',
'httpssites_power', 'httpssites_power',
'httpunit', 'httpunit',
'HttpUrlConnection', 'HttpUrlConnection',
'http\.rb\/',
'HTTP_Compression_Test',
'http_get',
'http_request2',
'http_requester',
'httrack', 'httrack',
'huaweisymantec', 'huaweisymantec',
'HubSpot ', 'HubSpot ',
'HubSpot-Link-Resolver',
'Humanlinks', 'Humanlinks',
'i2kconnect\/', 'i2kconnect\/',
'Iblog', 'Iblog',
@ -495,6 +587,7 @@ class Crawlers extends AbstractProvider
'IDwhois\/', 'IDwhois\/',
'Iframely', 'Iframely',
'igdeSpyder', 'igdeSpyder',
'iGooglePortal',
'IlTrovatore', 'IlTrovatore',
'Image Fetch', 'Image Fetch',
'Image Sucker', 'Image Sucker',
@ -519,13 +612,15 @@ class Crawlers extends AbstractProvider
'integromedb', 'integromedb',
'Intelliseek', 'Intelliseek',
'InterGET', 'InterGET',
'internet_archive',
'Internet Ninja', 'Internet Ninja',
'InternetSeer', 'InternetSeer',
'internetVista monitor', 'internetVista monitor',
'internetwache',
'internet_archive',
'intraVnews', 'intraVnews',
'IODC', 'IODC',
'IOI', 'IOI',
'Inboxb0t',
'iplabel', 'iplabel',
'ips-agent', 'ips-agent',
'IPS\/[0-9]', 'IPS\/[0-9]',
@ -537,10 +632,13 @@ class Crawlers extends AbstractProvider
'iskanie', 'iskanie',
'isUp\.li', 'isUp\.li',
'iThemes Sync\/', 'iThemes Sync\/',
'IZaBEE',
'iZSearch', 'iZSearch',
'JAHHO', 'JAHHO',
'janforman', 'janforman',
'Jaunt\/', 'Jaunt\/',
'Java.*outbrain',
'javelin\.io',
'Jbrofuzz', 'Jbrofuzz',
'Jersey\/', 'Jersey\/',
'JetCar', 'JetCar',
@ -556,6 +654,7 @@ class Crawlers extends AbstractProvider
'Joomla', 'Joomla',
'Jorgee', 'Jorgee',
'JS-Kit', 'JS-Kit',
'JungleKeyThumbnail',
'JustView', 'JustView',
'Kaspersky Lab CFR link resolver', 'Kaspersky Lab CFR link resolver',
'Kelny\/', 'Kelny\/',
@ -563,6 +662,7 @@ class Crawlers extends AbstractProvider
'KeyCDN', 'KeyCDN',
'Keyword Density', 'Keyword Density',
'Keywords Research', 'Keywords Research',
'khttp\/',
'KickFire', 'KickFire',
'KimonoLabs\/', 'KimonoLabs\/',
'Kml-Google', 'Kml-Google',
@ -570,11 +670,12 @@ class Crawlers extends AbstractProvider
'KOCMOHABT', 'KOCMOHABT',
'kouio', 'kouio',
'kube-probe', 'kube-probe',
'kubectl',
'kulturarw3', 'kulturarw3',
'KumKie', 'KumKie',
'L\.webis',
'Larbin', 'Larbin',
'Lavf\/', 'Lavf\/',
'leakix\.net',
'LeechFTP', 'LeechFTP',
'LeechGet', 'LeechGet',
'letsencrypt', 'letsencrypt',
@ -585,41 +686,50 @@ class Crawlers extends AbstractProvider
'libwww', 'libwww',
'Licorne', 'Licorne',
'Liferea\/', 'Liferea\/',
'Lightspeedsystems',
'Lighthouse', 'Lighthouse',
'Lightspeedsystems',
'Likse', 'Likse',
'limber\.io',
'Link Valet', 'Link Valet',
'link_thumbnailer',
'LinkAlarm\/', 'LinkAlarm\/',
'LinkAnalyser',
'linkCheck', 'linkCheck',
'linkdex', 'linkdex',
'LinkExaminer', 'LinkExaminer',
'linkfluence', 'linkfluence',
'linkpeek', 'linkpeek',
'LinkPreviewGenerator', 'LinkPreview',
'LinkScan', 'LinkScan',
'LinksManager', 'LinksManager',
'LinkTiger', 'LinkTiger',
'LinkWalker', 'LinkWalker',
'link_thumbnailer',
'Lipperhey', 'Lipperhey',
'Litemage_walker', 'Litemage_walker',
'livedoor ScreenShot', 'livedoor ScreenShot',
'LoadImpactRload', 'LoadImpactRload',
'localsearch-web', 'localsearch-web',
'LongURL API', 'LongURL API',
'longurl-r-package',
'looid\.com',
'looksystems\.net', 'looksystems\.net',
'ltx71', 'ltx71',
'lua-resty-http', 'lua-resty-http',
'Lucee \(CFML Engine\)',
'Lush Http Client',
'lwp-request', 'lwp-request',
'lwp-trivial', 'lwp-trivial',
'LWP::Simple', 'LWP::Simple',
'lycos', 'lycos',
'LYT\.SR', 'LYT\.SR',
'L\.webis',
'mabontland', 'mabontland',
'MacOutlook\/',
'Mag-Net', 'Mag-Net',
'MagpieRSS', 'MagpieRSS',
'Mail\.Ru', 'Mail::STS',
'MailChimp', 'MailChimp',
'Mail\.Ru',
'Majestic12', 'Majestic12',
'makecontact\/', 'makecontact\/',
'Mandrill', 'Mandrill',
@ -630,6 +740,8 @@ class Crawlers extends AbstractProvider
'Mass Downloader', 'Mass Downloader',
'masscan\/', 'masscan\/',
'Mata Hari', 'Mata Hari',
'mattermost',
'Mediametric',
'Mediapartners-Google', 'Mediapartners-Google',
'mediawords', 'mediawords',
'MegaIndex\.ru', 'MegaIndex\.ru',
@ -640,28 +752,31 @@ class Crawlers extends AbstractProvider
'MetaURI', 'MetaURI',
'MFC_Tear_Sample', 'MFC_Tear_Sample',
'Microsearch', 'Microsearch',
'Microsoft Office ', 'Microsoft Data Access',
'Microsoft Office',
'Microsoft Outlook', 'Microsoft Outlook',
'Microsoft Windows Network Diagnostics', 'Microsoft Windows Network Diagnostics',
'Microsoft-WebDAV-MiniRedir', 'Microsoft-WebDAV-MiniRedir',
'Microsoft Data Access', 'Microsoft\.Data\.Mashup',
'MIDown tool', 'MIDown tool',
'MIIxpc', 'MIIxpc',
'Mindjet', 'Mindjet',
'Miniature\.io', 'Miniature\.io',
'Miniflux', 'Miniflux',
'mio_httpc',
'Miro-HttpClient',
'Mister PiX', 'Mister PiX',
'mixdata dot com', 'mixdata dot com',
'mixed-content-scan', 'mixed-content-scan',
'Mixmax-LinkPreview',
'mixnode', 'mixnode',
'Mnogosearch', 'Mnogosearch',
'mogimogi', 'mogimogi',
'Mojeek', 'Mojeek',
'Mojolicious \(Perl\)', 'Mojolicious \(Perl\)',
'Monit\/', 'Mollie',
'monitis', 'monitis',
'Monitority\/', 'Monitority\/',
'Monit\/',
'montastic', 'montastic',
'MonTools', 'MonTools',
'Moreover', 'Moreover',
@ -670,6 +785,7 @@ class Crawlers extends AbstractProvider
'MovableType', 'MovableType',
'mowser', 'mowser',
'Mrcgiguy', 'Mrcgiguy',
'Mr\.4x3 Powered',
'MS Web Services Client Protocol', 'MS Web Services Client Protocol',
'MSFrontPage', 'MSFrontPage',
'mShots', 'mShots',
@ -677,10 +793,13 @@ class Crawlers extends AbstractProvider
'muhstik-scan', 'muhstik-scan',
'MVAClient', 'MVAClient',
'MxToolbox\/', 'MxToolbox\/',
'myseosnapshot',
'nagios', 'nagios',
'Najdi\.si', 'Najdi\.si',
'Name Intelligence', 'Name Intelligence',
'NameFo\.com',
'Nameprotect', 'Nameprotect',
'nationalarchives',
'Navroad', 'Navroad',
'NearSite', 'NearSite',
'Needle', 'Needle',
@ -695,6 +814,8 @@ class Crawlers extends AbstractProvider
'netresearch', 'netresearch',
'NetShelter ContentScan', 'NetShelter ContentScan',
'Netsparker', 'Netsparker',
'NetSystemsResearch',
'nettle',
'NetTrack', 'NetTrack',
'Netvibes', 'Netvibes',
'NetZIP', 'NetZIP',
@ -707,22 +828,26 @@ class Crawlers extends AbstractProvider
'newspaper\/', 'newspaper\/',
'Nexgate Ruby Client', 'Nexgate Ruby Client',
'NG-Search', 'NG-Search',
'nghttp2',
'Nibbler', 'Nibbler',
'NICErsPRO', 'NICErsPRO',
'NihilScio',
'Nikto', 'Nikto',
'nineconnections', 'nineconnections',
'NLNZ_IAHarvester', 'NLNZ_IAHarvester',
'Nmap Scripting Engine', 'Nmap Scripting Engine',
'node-fetch',
'node-superagent', 'node-superagent',
'node-urllib', 'node-urllib',
'node\.io',
'Nodemeter', 'Nodemeter',
'NodePing', 'NodePing',
'node\.io',
'nominet\.org\.uk', 'nominet\.org\.uk',
'nominet\.uk', 'nominet\.uk',
'Norton-Safeweb', 'Norton-Safeweb',
'Notifixious', 'Notifixious',
'notifyninja', 'notifyninja',
'NotionEmbedder',
'nuhk', 'nuhk',
'nutch', 'nutch',
'Nuzzel', 'Nuzzel',
@ -730,39 +855,45 @@ class Crawlers extends AbstractProvider
'nyawc\/', 'nyawc\/',
'Nymesis', 'Nymesis',
'NYU', 'NYU',
'Observatory\/',
'Ocelli\/', 'Ocelli\/',
'Octopus', 'Octopus',
'oegp', 'oegp',
'Offline Explorer', 'Offline Explorer',
'Offline Navigator', 'Offline Navigator',
'og-scraper', 'OgScrper',
'okhttp', 'okhttp',
'omgili', 'omgili',
'OMSC', 'OMSC',
'Online Domain Tools', 'Online Domain Tools',
'Open Source RSS',
'OpenCalaisSemanticProxy', 'OpenCalaisSemanticProxy',
'Openfind', 'Openfind',
'OpenLinkProfiler', 'OpenLinkProfiler',
'Openstat\/', 'Openstat\/',
'OpenVAS', 'OpenVAS',
'OPPO A33',
'Optimizer', 'Optimizer',
'Orbiter', 'Orbiter',
'OrgProbe\/', 'OrgProbe\/',
'orion-semantics', 'orion-semantics',
'Outlook-Express', 'Outlook-Express',
'Outlook-iOS', 'Outlook-iOS',
'ow\.ly',
'Owler', 'Owler',
'Owlin',
'ownCloud News', 'ownCloud News',
'ow\.ly',
'OxfordCloudService', 'OxfordCloudService',
'Page Valet',
'page_verifier',
'page scorer', 'page scorer',
'Page Valet',
'page2rss', 'page2rss',
'PageFreezer',
'PageGrabber', 'PageGrabber',
'PagePeeker', 'PagePeeker',
'PageScorer', 'PageScorer',
'Pagespeed\/', 'Pagespeed\/',
'PageThing',
'page_verifier',
'Panopta', 'Panopta',
'panscient', 'panscient',
'Papa Foto', 'Papa Foto',
@ -771,6 +902,7 @@ class Crawlers extends AbstractProvider
'PayPal IPN', 'PayPal IPN',
'pcBrowser', 'pcBrowser',
'Pcore-HTTP', 'Pcore-HTTP',
'PDF24 URL To PDF',
'Pearltrees', 'Pearltrees',
'PECL::HTTP', 'PECL::HTTP',
'peerindex', 'peerindex',
@ -780,44 +912,52 @@ class Crawlers extends AbstractProvider
'PhantomJS Screenshoter', 'PhantomJS Screenshoter',
'PhantomJS\/', 'PhantomJS\/',
'Photon\/', 'Photon\/',
'php-requests',
'phpservermon', 'phpservermon',
'Pi-Monster', 'Pi-Monster',
'Picscout', 'Picscout',
'Picsearch', 'Picsearch',
'PictureFinder', 'PictureFinder',
'Pimonster', 'Pimonster',
'ping\.blo\.gs',
'Pingability', 'Pingability',
'PingAdmin\.Ru', 'PingAdmin\.Ru',
'Pingdom', 'Pingdom',
'Pingoscope', 'Pingoscope',
'PingSpot', 'PingSpot',
'ping\.blo\.gs',
'pinterest\.com', 'pinterest\.com',
'Pixray', 'Pixray',
'Pizilla', 'Pizilla',
'Plagger\/', 'Plagger\/',
'Pleroma ',
'Ploetz \+ Zeller', 'Ploetz \+ Zeller',
'Plukkie', 'Plukkie',
'plumanalytics', 'plumanalytics',
'PocketImageCache', 'PocketImageCache',
'PocketParser', 'PocketParser',
'Pockey', 'Pockey',
'PodcastAddict\/',
'POE-Component-Client-HTTP', 'POE-Component-Client-HTTP',
'Polymail\/', 'Polymail\/',
'Pompos', 'Pompos',
'Porkbun', 'Porkbun',
'Port Monitor', 'Port Monitor',
'postano', 'postano',
'postfix-mta-sts-resolver',
'PostmanRuntime', 'PostmanRuntime',
'postplanner\.com',
'PostPost', 'PostPost',
'postrank', 'postrank',
'PowerPoint\/', 'PowerPoint\/',
'Prebid',
'Prerender',
'Priceonomics Analysis Engine', 'Priceonomics Analysis Engine',
'PrintFriendly', 'PrintFriendly',
'PritTorrent', 'PritTorrent',
'Prlog', 'Prlog',
'probethenet', 'probethenet',
'Project 25499', 'Project ?25499',
'Project-Resonance',
'prospectb2b', 'prospectb2b',
'Protopage', 'Protopage',
'ProWebWalker', 'ProWebWalker',
@ -826,9 +966,9 @@ class Crawlers extends AbstractProvider
'pshtt, https scanning', 'pshtt, https scanning',
'PTST ', 'PTST ',
'PTST\/[0-9]+', 'PTST\/[0-9]+',
'Pulsepoint XT3 web scraper',
'Pump', 'Pump',
'Python-httplib2', 'Python-httplib2',
'python-httpx',
'python-requests', 'python-requests',
'Python-urllib', 'Python-urllib',
'Qirina Hurdler', 'Qirina Hurdler',
@ -838,12 +978,17 @@ class Crawlers extends AbstractProvider
'Qualidator', 'Qualidator',
'QueryN Metasearch', 'QueryN Metasearch',
'queuedriver', 'queuedriver',
'quic-go-HTTP\/',
'QuiteRSS',
'Quora Link Preview', 'Quora Link Preview',
'Qwantify', 'Qwantify',
'Radian6', 'Radian6',
'RadioPublicImageResizer',
'Railgun\/',
'RankActive', 'RankActive',
'RankFlex', 'RankFlex',
'RankSonicSiteAuditor', 'RankSonicSiteAuditor',
'RapidLoad\/',
'Re-re Studio', 'Re-re Studio',
'ReactorNetty', 'ReactorNetty',
'Readability', 'Readability',
@ -854,6 +999,7 @@ class Crawlers extends AbstractProvider
'RecurPost\/', 'RecurPost\/',
'redback\/', 'redback\/',
'ReederForMac', 'ReederForMac',
'Reeder\/',
'ReGet', 'ReGet',
'RepoMonkey', 'RepoMonkey',
'request\.js', 'request\.js',
@ -866,38 +1012,44 @@ class Crawlers extends AbstractProvider
'Robozilla', 'Robozilla',
'ROI Hunter', 'ROI Hunter',
'RPT-HTTPClient', 'RPT-HTTPClient',
'RSSMix\/',
'RSSOwl', 'RSSOwl',
'RyowlEngine',
'safe-agent-scanner', 'safe-agent-scanner',
'SalesIntelligent', 'SalesIntelligent',
'Saleslift', 'Saleslift',
'Sendsay\.Ru', 'SAP NetWeaver Application Server',
'SauceNAO', 'SauceNAO',
'SBIder', 'SBIder',
'sc-downloader',
'scalaj-http', 'scalaj-http',
'scan\.lol', 'Scamadviser-Frontend',
'ScanAlert', 'ScanAlert',
'scan\.lol',
'Scoop', 'Scoop',
'scooter', 'scooter',
'ScopeContentAG-HTTP-Client',
'ScoutJet', 'ScoutJet',
'ScoutURLMonitor', 'ScoutURLMonitor',
'ScrapeBox Page Scanner', 'ScrapeBox Page Scanner',
'SimpleScraper',
'Scrapy', 'Scrapy',
'Screaming', 'Screaming',
'ScreenShotService', 'ScreenShotService',
'Scrubby', 'Scrubby',
'Scrutiny\/', 'Scrutiny\/',
'search\.thunderstone',
'Search37', 'Search37',
'searchenginepromotionhelp', 'searchenginepromotionhelp',
'Searchestate', 'Searchestate',
'SearchExpress', 'SearchExpress',
'SearchSight', 'SearchSight',
'SearchWP',
'search\.thunderstone',
'Seeker', 'Seeker',
'semanticdiscovery', 'semanticdiscovery',
'semanticjuice', 'semanticjuice',
'Semiocast HTTP client', 'Semiocast HTTP client',
'Semrush', 'Semrush',
'Sendsay\.Ru',
'sentry\/', 'sentry\/',
'SEO Browser', 'SEO Browser',
'Seo Servis', 'Seo Servis',
@ -906,11 +1058,13 @@ class Crawlers extends AbstractProvider
'Seobility', 'Seobility',
'SEOCentro', 'SEOCentro',
'SeoCheck', 'SeoCheck',
'seocompany',
'SEOkicks', 'SEOkicks',
'SEOlizer',
'Seomoz', 'Seomoz',
'SEOprofiler', 'SEOprofiler',
'SEOsearch',
'seoscanners', 'seoscanners',
'SEOsearch',
'seositecheckup', 'seositecheckup',
'SEOstats', 'SEOstats',
'servernfo', 'servernfo',
@ -921,14 +1075,16 @@ class Crawlers extends AbstractProvider
'Shoppimon', 'Shoppimon',
'ShopWiki', 'ShopWiki',
'ShortLinkTranslate', 'ShortLinkTranslate',
'shortURL lengthener',
'shrinktheweb', 'shrinktheweb',
'Sideqik', 'Sideqik',
'Siege',
'SimplePie', 'SimplePie',
'SimplyFast', 'SimplyFast',
'Siphon', 'Siphon',
'SISTRIX', 'SISTRIX',
'Site-Shot\/',
'Site Sucker', 'Site Sucker',
'Site-Shot\/',
'Site24x7', 'Site24x7',
'SiteBar', 'SiteBar',
'Sitebeam', 'Sitebeam',
@ -949,6 +1105,7 @@ class Crawlers extends AbstractProvider
'sitexy\.com', 'sitexy\.com',
'SkypeUriPreview', 'SkypeUriPreview',
'Slack\/', 'Slack\/',
'sli-systems\.com',
'slider\.com', 'slider\.com',
'slurp', 'slurp',
'SlySearch', 'SlySearch',
@ -971,9 +1128,11 @@ class Crawlers extends AbstractProvider
'SpamExperts', 'SpamExperts',
'Spammen', 'Spammen',
'Spanner', 'Spanner',
'Spawning-AI',
'spaziodati', 'spaziodati',
'SPDYCheck', 'SPDYCheck',
'Specificfeeds', 'Specificfeeds',
'SpeedKit',
'speedy', 'speedy',
'SPEng', 'SPEng',
'Spinn3r', 'Spinn3r',
@ -987,9 +1146,11 @@ class Crawlers extends AbstractProvider
'ssl-tools', 'ssl-tools',
'StackRambler', 'StackRambler',
'Statastico\/', 'Statastico\/',
'Statically-',
'StatusCake', 'StatusCake',
'Steeler', 'Steeler',
'Stratagems Kumo', 'Stratagems Kumo',
'Stripe\/',
'Stroke\.cz', 'Stroke\.cz',
'StudioFACA', 'StudioFACA',
'StumbleUpon', 'StumbleUpon',
@ -999,9 +1160,11 @@ class Crawlers extends AbstractProvider
'SuperHTTP', 'SuperHTTP',
'Surphace Scout', 'Surphace Scout',
'Suzuran', 'Suzuran',
'SwiteScraper', 'swcd ',
'Symfony BrowserKit', 'Symfony BrowserKit',
'Symfony2 BrowserKit', 'Symfony2 BrowserKit',
'Synapse\/',
'Syndirella\/',
'SynHttpClient-Built', 'SynHttpClient-Built',
'Sysomos', 'Sysomos',
'sysscan', 'sysscan',
@ -1011,6 +1174,8 @@ class Crawlers extends AbstractProvider
'Tarantula\/', 'Tarantula\/',
'Taringa UGC', 'Taringa UGC',
'TarmotGezgin', 'TarmotGezgin',
'tchelebi\.io',
'techiaith\.cymru',
'Teleport', 'Teleport',
'Telesoft', 'Telesoft',
'Telesphoreo', 'Telesphoreo',
@ -1021,13 +1186,15 @@ class Crawlers extends AbstractProvider
'Test Certificate Info', 'Test Certificate Info',
'testuri', 'testuri',
'Tetrahedron', 'Tetrahedron',
'TextRazor Downloader',
'The Drop Reaper', 'The Drop Reaper',
'The Expert HTML Source Viewer', 'The Expert HTML Source Viewer',
'The Knowledge AI',
'The Intraformant', 'The Intraformant',
'The Knowledge AI',
'theinternetrules', 'theinternetrules',
'TheNomad', 'TheNomad',
'Thinklab', 'Thinklab',
'Thumbor',
'Thumbshots', 'Thumbshots',
'ThumbSniper', 'ThumbSniper',
'timewe\.net', 'timewe\.net',
@ -1061,9 +1228,11 @@ class Crawlers extends AbstractProvider
'ubermetrics-technologies', 'ubermetrics-technologies',
'uclassify', 'uclassify',
'UdmSearch', 'UdmSearch',
'ultimate_sitemap_parser',
'unchaos', 'unchaos',
'unirest-java', 'unirest-java',
'UniversalFeedParser', 'UniversalFeedParser',
'unshortenit',
'Unshorten\.It', 'Unshorten\.It',
'Untiny', 'Untiny',
'UnwindFetchor', 'UnwindFetchor',
@ -1071,8 +1240,8 @@ class Crawlers extends AbstractProvider
'updown\.io daemon', 'updown\.io daemon',
'Upflow', 'Upflow',
'Uptimia', 'Uptimia',
'Urlcheckr',
'URL Verifier', 'URL Verifier',
'Urlcheckr',
'URLitor', 'URLitor',
'urlresolver', 'urlresolver',
'Urlstat', 'Urlstat',
@ -1080,15 +1249,16 @@ class Crawlers extends AbstractProvider
'UrlTrends Ranking Updater', 'UrlTrends Ranking Updater',
'URLy Warning', 'URLy Warning',
'URLy\.Warning', 'URLy\.Warning',
'URL\/Emacs',
'Vacuum', 'Vacuum',
'Vagabondo', 'Vagabondo',
'VB Project', 'VB Project',
'vBSEO', 'vBSEO',
'VCI', 'VCI',
'via ggpht\.com GoogleImageProxy', 'via ggpht\.com GoogleImageProxy',
'VidibleScraper',
'Virusdie', 'Virusdie',
'visionutils', 'visionutils',
'Visual Rights Group',
'vkShare', 'vkShare',
'VoidEYE', 'VoidEYE',
'Voil', 'Voil',
@ -1099,19 +1269,18 @@ class Crawlers extends AbstractProvider
'Vulnbusters Meter', 'Vulnbusters Meter',
'VYU2', 'VYU2',
'w3af\.org', 'w3af\.org',
'W3C_Unicorn',
'W3C-checklink', 'W3C-checklink',
'W3C-mobileOK', 'W3C-mobileOK',
'W3C_Unicorn',
'WAC-OFU', 'WAC-OFU',
'Wallpapers\/[0-9]+', 'WakeletLinkExpander',
'WallpapersHD', 'WallpapersHD',
'Wallpapers\/[0-9]+',
'wangling', 'wangling',
'Wappalyzer', 'Wappalyzer',
'WatchMouse', 'WatchMouse',
'WbSrch\/', 'WbSrch\/',
'WDT\.io', 'WDT\.io',
'web-capture\.net',
'Web-sniffer',
'Web Auto', 'Web Auto',
'Web Collage', 'Web Collage',
'Web Enhancer', 'Web Enhancer',
@ -1119,7 +1288,10 @@ class Crawlers extends AbstractProvider
'Web Fuck', 'Web Fuck',
'Web Pix', 'Web Pix',
'Web Sauger', 'Web Sauger',
'Web spyder',
'Web Sucker', 'Web Sucker',
'web-capture\.net',
'Web-sniffer',
'Webalta', 'Webalta',
'Webauskunft', 'Webauskunft',
'WebAuto', 'WebAuto',
@ -1157,8 +1329,9 @@ class Crawlers extends AbstractProvider
'Webster', 'Webster',
'WebStripper', 'WebStripper',
'WebSucker', 'WebSucker',
'Webthumb\/', 'webtech\/',
'WebThumbnail', 'WebThumbnail',
'Webthumb\/',
'WebWhacker', 'WebWhacker',
'WebZIP', 'WebZIP',
'WeLikeLinks', 'WeLikeLinks',
@ -1167,24 +1340,31 @@ class Crawlers extends AbstractProvider
'wf84', 'wf84',
'Wfuzz\/', 'Wfuzz\/',
'wget', 'wget',
'WhatCMS',
'WhatsApp', 'WhatsApp',
'WhatsMyIP', 'WhatsMyIP',
'WhatWeb', 'WhatWeb',
'WhereGoes\?', 'WhereGoes\?',
'Whibse', 'Whibse',
'WhoAPI\/',
'WhoRunsCoinHive', 'WhoRunsCoinHive',
'Whynder Magnet', 'Whynder Magnet',
'Windows-RSS-Platform', 'Windows-RSS-Platform',
'WinHttp-Autoproxy-Service',
'WinHTTP\/',
'WinPodder', 'WinPodder',
'wkhtmlto', 'wkhtmlto',
'wmtips', 'wmtips',
'Woko', 'Woko',
'Wolfram HTTPClient',
'woorankreview', 'woorankreview',
'Word\/',
'WordPress\/', 'WordPress\/',
'WordupinfoSearch', 'WordupinfoSearch',
'Word\/',
'worldping-api',
'wotbox', 'wotbox',
'WP Engine Install Performance API', 'WP Engine Install Performance API',
'WP Rocket',
'wpif', 'wpif',
'wprecon\.com survey', 'wprecon\.com survey',
'WPScan', 'WPScan',
@ -1194,20 +1374,21 @@ class Crawlers extends AbstractProvider
'WWW-Mechanize', 'WWW-Mechanize',
'WWW::Document', 'WWW::Document',
'WWW::Mechanize', 'WWW::Mechanize',
'www\.monitor\.us',
'WWWOFFLE', 'WWWOFFLE',
'www\.monitor\.us',
'x09Mozilla', 'x09Mozilla',
'x22Mozilla', 'x22Mozilla',
'XaxisSemanticsClassifier', 'XaxisSemanticsClassifier',
'XenForo\/',
'Xenu Link Sleuth', 'Xenu Link Sleuth',
'XING-contenttabreceiver', 'XING-contenttabreceiver',
'xpymep([0-9]?)\.exe', 'xpymep([0-9]?)\.exe',
'Y!J-(ASR|BSC)', 'Y!J-[A-Z][A-Z][A-Z]',
'Y\!J-BRW',
'Yaanb', 'Yaanb',
'yacy', 'yacy',
'Yahoo Link Preview', 'Yahoo Link Preview',
'YahooCacheSystem', 'YahooCacheSystem',
'YahooMailProxy',
'YahooYSMcm', 'YahooYSMcm',
'YandeG', 'YandeG',
'Yandex(?!Search)', 'Yandex(?!Search)',
@ -1215,6 +1396,7 @@ class Crawlers extends AbstractProvider
'yeti', 'yeti',
'Yo-yo', 'Yo-yo',
'Yoleo Consumer', 'Yoleo Consumer',
'yomins\.com',
'yoogliFetchAgent', 'yoogliFetchAgent',
'YottaaMonitor', 'YottaaMonitor',
'Your-Website-Sucks', 'Your-Website-Sucks',
@ -1226,15 +1408,17 @@ class Crawlers extends AbstractProvider
'Zao', 'Zao',
'Zauba', 'Zauba',
'Zemanta Aggregator', 'Zemanta Aggregator',
'Zend_Http_Client',
'Zend\\\\Http\\\\Client', 'Zend\\\\Http\\\\Client',
'Zend_Http_Client',
'Zermelo', 'Zermelo',
'Zeus ', 'Zeus ',
'zgrab', 'zgrab',
'ZnajdzFoto', 'ZnajdzFoto',
'ZnHTTP',
'Zombie\.js', 'Zombie\.js',
'Zoom\.Mac', 'Zoom\.Mac',
'ZoteroTranslationServer',
'ZyBorg', 'ZyBorg',
'[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)', '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)',
); );
} }

4
blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php vendored
View file

@ -67,6 +67,8 @@ class Exclusions extends AbstractProvider
'; IDbot', '; IDbot',
'; ID bot', '; ID bot',
'; POWER BOT', '; POWER BOT',
';', // Remove the following characters ; 'OCTOPUS-CORE',
'htc_botdugls',
'super\/\d+\/Android\/\d+',
); );
} }