From d838fc6421719e75e70f1a40547dfb3356aa138f Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Tue, 19 Mar 2024 22:51:17 -0400 Subject: [PATCH] [blockbot] Update Composer dependency ahead of release - Updating jaybizzle/crawler-detect (v1.2.80 => v1.2.116) --- blockbot/composer.json | 44 +-- blockbot/composer.lock | 16 +- blockbot/vendor/composer/ClassLoader.php | 4 +- blockbot/vendor/composer/autoload_real.php | 3 + blockbot/vendor/composer/installed.json | 15 +- .../.github/workflows/php-cs-fixer.yml | 23 ++ .../crawler-detect/.github/workflows/test.yml | 56 ++++ .../jaybizzle/crawler-detect/.php_cs.dist | 33 ++ .../vendor/jaybizzle/crawler-detect/LICENSE | 2 +- .../vendor/jaybizzle/crawler-detect/README.md | 30 +- .../jaybizzle/crawler-detect/composer.json | 3 +- .../jaybizzle/crawler-detect/export.php | 2 +- .../crawler-detect/raw/Crawlers.json | 2 +- .../jaybizzle/crawler-detect/raw/Crawlers.txt | 302 ++++++++++++++---- .../crawler-detect/raw/Exclusions.json | 2 +- .../crawler-detect/raw/Exclusions.txt | 4 +- .../crawler-detect/src/CrawlerDetect.php | 29 +- .../src/Fixtures/AbstractProvider.php | 4 +- .../crawler-detect/src/Fixtures/Crawlers.php | 302 ++++++++++++++---- .../src/Fixtures/Exclusions.php | 4 +- 20 files changed, 686 insertions(+), 194 deletions(-) create mode 100644 blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml create mode 100644 blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml create mode 100644 blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist diff --git a/blockbot/composer.json b/blockbot/composer.json index f13a2d17..40583e20 100644 --- a/blockbot/composer.json +++ b/blockbot/composer.json @@ -1,24 +1,24 @@ { - "name": "friendica-addons/blockbot", - "description": "Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.", - "type": "friendica-addon", - "authors": [ - { - "name": "Philipp Holzer", - "email": "admin@philipp.info", - "homepage": "https://friendica.philipp.info/profile/nupplaphil", - "role": "Developer" - } - ], - "require": { - "php": ">=5.6.0", - "jaybizzle/crawler-detect": "1.*" - }, - "license": "3-clause BSD license", - "minimum-stability": "stable", - "config": { - "optimize-autoloader": true, - "autoloader-suffix": "BlockBotAddon", - "preferred-install": "dist" - } + "name": "friendica-addons/blockbot", + "description": "Blocking bots based on detecting bots/crawlers/spiders via the user agent and http_from header.", + "type": "friendica-addon", + "authors": [ + { + "name": "Philipp Holzer", + "email": "admin@philipp.info", + "homepage": "https://friendica.philipp.info/profile/nupplaphil", + "role": "Developer" + } + ], + "require": { + "php": ">=5.6.0", + "jaybizzle/crawler-detect": "1.*" + }, + "license": "3-clause BSD license", + "minimum-stability": "stable", + "config": { + "optimize-autoloader": true, + "autoloader-suffix": "BlockBotAddon", + "preferred-install": "dist" + } } diff --git a/blockbot/composer.lock b/blockbot/composer.lock index 26b021b1..50f71a37 100644 --- a/blockbot/composer.lock +++ b/blockbot/composer.lock @@ -8,24 +8,23 @@ "packages": [ { "name": "jaybizzle/crawler-detect", - "version": "v1.2.80", + "version": "v1.2.116", "source": { "type": "git", "url": "https://github.com/JayBizzle/Crawler-Detect.git", - "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847" + "reference": "97e9fe30219e60092e107651abb379a38b342921" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847", - "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847", + "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921", + "reference": "97e9fe30219e60092e107651abb379a38b342921", "shasum": "" }, "require": { "php": ">=5.3.0" }, "require-dev": { - "phpunit/phpunit": "^4.8|^5.5|^6.5", - "satooshi/php-coveralls": "1.*" + "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4" }, "type": "library", "autoload": { @@ -53,7 +52,7 @@ "crawlerdetect", "php crawler detect" ], - "time": "2019-04-05T19:52:02+00:00" + "time": "2023-07-21T15:49:49+00:00" } ], "packages-dev": [], @@ -65,5 +64,6 @@ "platform": { "php": ">=5.6.0" }, - "platform-dev": [] + "platform-dev": [], + "plugin-api-version": "1.1.0" } diff --git a/blockbot/vendor/composer/ClassLoader.php b/blockbot/vendor/composer/ClassLoader.php index 95f7e097..03b9bb9c 100644 --- a/blockbot/vendor/composer/ClassLoader.php +++ b/blockbot/vendor/composer/ClassLoader.php @@ -60,7 +60,7 @@ class ClassLoader public function getPrefixes() { if (!empty($this->prefixesPsr0)) { - return call_user_func_array('array_merge', $this->prefixesPsr0); + return call_user_func_array('array_merge', array_values($this->prefixesPsr0)); } return array(); @@ -279,7 +279,7 @@ class ClassLoader */ public function setApcuPrefix($apcuPrefix) { - $this->apcuPrefix = function_exists('apcu_fetch') && ini_get('apc.enabled') ? $apcuPrefix : null; + $this->apcuPrefix = function_exists('apcu_fetch') && filter_var(ini_get('apc.enabled'), FILTER_VALIDATE_BOOLEAN) ? $apcuPrefix : null; } /** diff --git a/blockbot/vendor/composer/autoload_real.php b/blockbot/vendor/composer/autoload_real.php index ccb886a2..404079e0 100644 --- a/blockbot/vendor/composer/autoload_real.php +++ b/blockbot/vendor/composer/autoload_real.php @@ -13,6 +13,9 @@ class ComposerAutoloaderInitBlockBotAddon } } + /** + * @return \Composer\Autoload\ClassLoader + */ public static function getLoader() { if (null !== self::$loader) { diff --git a/blockbot/vendor/composer/installed.json b/blockbot/vendor/composer/installed.json index d255d573..db9064b6 100644 --- a/blockbot/vendor/composer/installed.json +++ b/blockbot/vendor/composer/installed.json @@ -1,27 +1,26 @@ [ { "name": "jaybizzle/crawler-detect", - "version": "v1.2.80", - "version_normalized": "1.2.80.0", + "version": "v1.2.116", + "version_normalized": "1.2.116.0", "source": { "type": "git", "url": "https://github.com/JayBizzle/Crawler-Detect.git", - "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847" + "reference": "97e9fe30219e60092e107651abb379a38b342921" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847", - "reference": "af6a36e6d69670df3f0a3ed8e21d4b8cc67a7847", + "url": "https://api.github.com/repos/JayBizzle/Crawler-Detect/zipball/97e9fe30219e60092e107651abb379a38b342921", + "reference": "97e9fe30219e60092e107651abb379a38b342921", "shasum": "" }, "require": { "php": ">=5.3.0" }, "require-dev": { - "phpunit/phpunit": "^4.8|^5.5|^6.5", - "satooshi/php-coveralls": "1.*" + "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4" }, - "time": "2019-04-05T19:52:02+00:00", + "time": "2023-07-21T15:49:49+00:00", "type": "library", "installation-source": "dist", "autoload": { diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml new file mode 100644 index 00000000..1c083c40 --- /dev/null +++ b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/php-cs-fixer.yml @@ -0,0 +1,23 @@ +name: Check & fix styling + +on: [ push ] + +jobs: + php-cs-fixer: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + ref: ${{ github.head_ref }} + + - name: Run PHP CS Fixer + uses: docker://oskarstark/php-cs-fixer-ga:2.18.6 + with: + args: --config=.php_cs.dist --allow-risky=yes + + - name: Commit changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: Fix styling \ No newline at end of file diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml new file mode 100644 index 00000000..22911114 --- /dev/null +++ b/blockbot/vendor/jaybizzle/crawler-detect/.github/workflows/test.yml @@ -0,0 +1,56 @@ +name: Test + +on: + push: + branches: + - "master" + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + php: [5.3, 5.4, 5.5, 5.6, 7.0, 7.1, 7.2, 7.3, 7.4, 8.0, 8.1, 8.2] + + name: PHP:${{ matrix.php }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup PHP, with composer + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php }} + tools: composer:v2 + coverage: xdebug + + - name: Get composer cache directory + id: composer-cache + run: | + echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache composer dependencies + uses: actions/cache@v3 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: dependencies-php-${{ matrix.php }}-composer-${{ hashFiles('composer.json') }} + restore-keys: dependencies-php-${{ matrix.php }}-composer- + + - name: Install Composer dependencies + run: | + composer install --prefer-dist --no-interaction --no-suggest + + - name: Run Unit tests + run: | + vendor/bin/phpunit --coverage-clover=tests/logs/clover.xml + + - name: Upload coverage results to Coveralls + env: + COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + composer global require php-coveralls/php-coveralls "^1.0" + coveralls --coverage_clover=tests/logs/clover.xml -v diff --git a/blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist b/blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist new file mode 100644 index 00000000..91c91af9 --- /dev/null +++ b/blockbot/vendor/jaybizzle/crawler-detect/.php_cs.dist @@ -0,0 +1,33 @@ +in([ + __DIR__.'/src', + __DIR__.'/tests', + ]) + ->name('*.php') + ->ignoreDotFiles(true) + ->ignoreVCS(true); + +return PhpCsFixer\Config::create() + ->setRules([ + '@PSR2' => true, + 'array_syntax' => ['syntax' => 'long'], + 'ordered_imports' => ['sortAlgorithm' => 'alpha'], + 'no_unused_imports' => true, + 'not_operator_with_successor_space' => true, + 'trailing_comma_in_multiline_array' => true, + 'phpdoc_scalar' => true, + 'unary_operator_spaces' => true, + 'binary_operator_spaces' => true, + 'blank_line_before_statement' => [ + 'statements' => ['break', 'continue', 'declare', 'return', 'throw', 'try'], + ], + 'phpdoc_single_line_var_spacing' => true, + 'phpdoc_var_without_name' => true, + 'method_argument_space' => [ + 'on_multiline' => 'ensure_fully_multiline', + 'keep_multiple_spaces_after_comma' => true, + ], + ]) + ->setFinder($finder); \ No newline at end of file diff --git a/blockbot/vendor/jaybizzle/crawler-detect/LICENSE b/blockbot/vendor/jaybizzle/crawler-detect/LICENSE index 2f4e15e2..569c7b4f 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/LICENSE +++ b/blockbot/vendor/jaybizzle/crawler-detect/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2018 Mark Beech +Copyright (c) 2015-2020 Mark Beech Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/blockbot/vendor/jaybizzle/crawler-detect/README.md b/blockbot/vendor/jaybizzle/crawler-detect/README.md index e7c25f3b..57ec8e8b 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/README.md +++ b/blockbot/vendor/jaybizzle/crawler-detect/README.md @@ -1,24 +1,23 @@ -



-crawlerdetect.io +



+crawlerdetect.io

-

- +GitHub Workflow Status - -

## About CrawlerDetect -CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the user agent and http_from header. Currently able to detect 1,000's of bots/spiders/crawlers. +CrawlerDetect is a PHP class for detecting bots/crawlers/spiders via the `user agent` and `http_from` header. Currently able to detect 1,000's of bots/spiders/crawlers. ### Installation -Run `composer require jaybizzle/crawler-detect 1.*` or add `"jaybizzle/crawler-detect" :"1.*"` to your `composer.json`. +``` +composer require jaybizzle/crawler-detect +``` ### Usage ```PHP @@ -46,7 +45,7 @@ If you find a bot/spider/crawler user agent that CrawlerDetect fails to detect, Failing that, just create an issue with the user agent you have found, and we'll take it from there :) ### Laravel Package -If you would like to use this with Laravel 4/5, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect) +If you would like to use this with Laravel, please see [Laravel-Crawler-Detect](https://github.com/JayBizzle/Laravel-Crawler-Detect) ### Symfony Bundle To use this library with Symfony 2/3/4, check out the [CrawlerDetectBundle](https://github.com/nicolasmure/CrawlerDetectBundle). @@ -57,16 +56,21 @@ To use this library with the YII2 framework, check out [yii2-crawler-detect](htt ### ES6 Library To use this library with NodeJS or any ES6 application based, check out [es6-crawler-detect](https://github.com/JefferyHus/es6-crawler-detect). +### Python Library +To use this library in a Python project, check out [crawlerdetect](https://github.com/moskrc/CrawlerDetect). + +### JVM Library (written in Java) +To use this library in a JVM project (including Java, Scala, Kotlin, etc.), check out [CrawlerDetect](https://github.com/nekosoftllc/crawler-detect). + ### .NET Library To use this library in a .net standard (including .net core) based project, check out [NetCrawlerDetect](https://github.com/gplumb/NetCrawlerDetect). -### Nette Extension -To use this library with the Nette framework, checkout [NetteCrawlerDetect](https://github.com/JanGalek/Crawler-Detect). - ### Ruby Gem - To use this library with Ruby on Rails or any Ruby-based application, check out [crawler_detect](https://github.com/loadkpi/crawler_detect) gem. +### Go Module +To use this library with Go, check out the [crawlerdetect](https://github.com/x-way/crawlerdetect) module. + _Parts of this class are based on the brilliant [MobileDetect](https://github.com/serbanghita/Mobile-Detect)_ [![Analytics](https://ga-beacon.appspot.com/UA-72430465-1/Crawler-Detect/readme?pixel)](https://github.com/JayBizzle/Crawler-Detect) diff --git a/blockbot/vendor/jaybizzle/crawler-detect/composer.json b/blockbot/vendor/jaybizzle/crawler-detect/composer.json index 0c0babe6..4774117e 100755 --- a/blockbot/vendor/jaybizzle/crawler-detect/composer.json +++ b/blockbot/vendor/jaybizzle/crawler-detect/composer.json @@ -16,8 +16,7 @@ "php": ">=5.3.0" }, "require-dev": { - "phpunit/phpunit": "^4.8|^5.5|^6.5", - "satooshi/php-coveralls": "1.*" + "phpunit/phpunit": "^4.8|^5.5|^6.5|^9.4" }, "autoload": { "psr-4": { diff --git a/blockbot/vendor/jaybizzle/crawler-detect/export.php b/blockbot/vendor/jaybizzle/crawler-detect/export.php index 4c4b9d5d..6c7459c4 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/export.php +++ b/blockbot/vendor/jaybizzle/crawler-detect/export.php @@ -37,5 +37,5 @@ function outputJson($object) function outputTxt($object) { $className = (new ReflectionClass($object))->getShortName(); - file_put_contents("raw/$className.txt", implode($object->getAll(), PHP_EOL)); + file_put_contents("raw/$className.txt", implode(PHP_EOL, $object->getAll())); } diff --git a/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json b/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json index a1e690eb..003b87c8 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json +++ b/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.json @@ -1 +1 @@ -[".*Java.*outbrain"," YLT","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^DangDang","^DavClnt","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^HTTPClient\\\/","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^NG\\\/[0-9\\.]","^NING\\\/","^PHP\\\/[0-9]","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2ip\\.ru","404enemy","7Siters","80legs","a\\.pr-cy\\.ru","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adscanner","Adstxtaggregator","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Ant\\.com","Anturis Agent","AnyEvent-HTTP\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppStoreScraperZ","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","AskQuickly","Ask Jeeves","ASPSeek","Asterias","Astute","asynchttp","Attach","autocite","Autonomy","axios\\\/","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","biz_Directory","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cg-eye","changedetection","ChangesMeter","Charlotte","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","Cosmos4j\\.feedback","Covario-IDS","Crescent","Crowsnest","Criteo","CSHttp","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Disqus\\\/","Dispatch\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconkit","faviconarchive","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedspot","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","fluffy","Flunky","flynxapp","forensiq","FoundSeoTool","http:\\\/\\\/www.neomo.de\\\/","free thumbnails","Freeuploader","Funnelback","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","github\\.com","Go [\\d\\.]* package http","Go http package","Go-Ahead-Got-It","Go-http-client","Go!Zilla","gobyus","gofetch","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-Publisher-Plugin","Google-SearchByImage","Google-Site-Verification","Google-Structured-Data-Testing-Tool","Google-Youtube-Links","google-xrawler","GoogleDocs","GoogleHC\\\/","GoogleProducer","GoogleSites","Google-Transparency-Report","Gookey","GoScraper","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hatena","Havij","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","HTTP_Compression_Test","http_request2","http_requester","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http\\.rb\\\/","http_get","HttpComponents","httphr","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","httrack","huaweisymantec","HubSpot ","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","internet_archive","Internet Ninja","InternetSeer","internetVista monitor","intraVnews","IODC","IOI","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","iZSearch","JAHHO","janforman","Jaunt\\\/","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kulturarw3","KumKie","L\\.webis","Larbin","Lavf\\\/","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lightspeedsystems","Lighthouse","Likse","Link Valet","link_thumbnailer","LinkAlarm\\\/","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreviewGenerator","LinkScan","LinksManager","LinkTiger","LinkWalker","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","looksystems\\.net","ltx71","lua-resty-http","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","mabontland","Mag-Net","MagpieRSS","Mail\\.Ru","MailChimp","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Office ","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft Data Access","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","Mister PiX","mixdata dot com","mixed-content-scan","Mixmax-LinkPreview","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Monit\\\/","monitis","Monitority\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","nagios","Najdi\\.si","Name Intelligence","Nameprotect","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","Nibbler","NICErsPRO","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-superagent","node-urllib","node\\.io","Nodemeter","NodePing","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","og-scraper","okhttp","omgili","OMSC","Online Domain Tools","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","ow\\.ly","Owler","ownCloud News","OxfordCloudService","Page Valet","page_verifier","page scorer","page2rss","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","ping\\.blo\\.gs","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","PostmanRuntime","PostPost","postrank","PowerPoint\\\/","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project 25499","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pulsepoint XT3 web scraper","Pump","Python-httplib2","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","Quora Link Preview","Qwantify","Radian6","RankActive","RankFlex","RankSonicSiteAuditor","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSOwl","safe-agent-scanner","SalesIntelligent","Saleslift","Sendsay\\.Ru","SauceNAO","SBIder","scalaj-http","scan\\.lol","ScanAlert","Scoop","scooter","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","SimpleScraper","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","search\\.thunderstone","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","SEOkicks","Seomoz","SEOprofiler","SEOsearch","seoscanners","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shrinktheweb","Sideqik","SimplePie","SimplyFast","Siphon","SISTRIX","Site-Shot\\\/","Site Sucker","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","spaziodati","SPDYCheck","Specificfeeds","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","StatusCake","Steeler","Stratagems Kumo","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","SwiteScraper","Symfony BrowserKit","Symfony2 BrowserKit","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","The Drop Reaper","The Expert HTML Source Viewer","The Knowledge AI","The Intraformant","theinternetrules","TheNomad","Thinklab","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","unchaos","unirest-java","UniversalFeedParser","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","Urlcheckr","URL Verifier","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","VidibleScraper","Virusdie","visionutils","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C_Unicorn","W3C-checklink","W3C-mobileOK","WAC-OFU","Wallpapers\\\/[0-9]+","WallpapersHD","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","web-capture\\.net","Web-sniffer","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web Sucker","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","Webthumb\\\/","WebThumbnail","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinPodder","wkhtmlto","wmtips","Woko","woorankreview","Word\\\/","WordPress\\\/","WordupinfoSearch","wotbox","WP Engine Install Performance API","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","www\\.monitor\\.us","WWWOFFLE","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-(ASR|BSC)","Y\\!J-BRW","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend_Http_Client","Zend\\\\Http\\\\Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","Zombie\\.js","Zoom\\.Mac","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)"] \ No newline at end of file +[" YLT","^Aether","^Amazon Simple Notification Service Agent$","^Amazon-Route53-Health-Check-Service","^b0t$","^bluefish ","^Calypso v\\\/","^COMODO DCV","^Corax","^DangDang","^DavClnt","^DHSH","^docker\\\/[0-9]","^Expanse","^FDM ","^git\\\/","^Goose\\\/","^Grabber","^Gradle\\\/","^HTTPClient\\\/","^HTTPing","^Java\\\/","^Jeode\\\/","^Jetty\\\/","^Mail\\\/","^Mget","^Microsoft URL Control","^Mikrotik\\\/","^Netlab360","^NG\\\/[0-9\\.]","^NING\\\/","^npm\\\/","^Nuclei","^PHP-AYMAPI\\\/","^PHP\\\/","^pip\\\/","^pnpm\\\/","^RMA\\\/","^Ruby|Ruby\\\/[0-9]","^Swurl ","^TLS tester ","^twine\\\/","^ureq","^VSE\\\/[0-9]","^WordPress\\.com","^XRL\\\/[0-9]","^ZmEu","008\\\/","13TABS","192\\.comAgent","2GDPR\\\/","2ip\\.ru","404enemy","7Siters","80legs","a3logics\\.in","A6-Indexer","Abonti","Aboundex","aboutthedomain","Accoona-AI-Agent","acebookexternalhit\\\/","acoon","acrylicapps\\.com\\\/pulp","Acunetix","AdAuth\\\/","adbeat","AddThis","ADmantX","AdminLabs","adressendeutschland","adreview\\\/","adscanner","adstxt-worker","Adstxtaggregator","adstxt\\.com","Adyen HttpClient","AffiliateLabz\\\/","affilimate-puppeteer","agentslug","AHC","aihit","aiohttp\\\/","Airmail","akka-http\\\/","akula\\\/","alertra","alexa site audit","Alibaba\\.Security\\.Heimdall","Alligator","allloadin","AllSubmitter","alyze\\.info","amagit","Anarchie","AndroidDownloadManager","Anemone","AngleSharp","annotate_google","Anthill","Anturis Agent","Ant\\.com","AnyEvent-HTTP\\\/","Apache Ant\\\/","Apache Droid","Apache OpenOffice","Apache-HttpAsyncClient","Apache-HttpClient","ApacheBench","Apexoo","apimon\\.de","APIs-Google","AportWorm\\\/","AppBeat\\\/","AppEngine-Google","AppleSyndication","Aprc\\\/[0-9]","Arachmo","arachnode","Arachnophilia","aria2","Arukereso","asafaweb","Asana\\\/","Ask Jeeves","AskQuickly","ASPSeek","Asterias","Astute","asynchttp","Attach","attohttpc","autocite","AutomaticWPTester","Autonomy","awin\\.com","AWS Security Scanner","axios\\\/","a\\.pr-cy\\.ru","B-l-i-t-z-B-O-T","Backlink-Ceck","backlink-check","BacklinkHttpStatus","BackStreet","BackupLand","BackWeb","Bad-Neighborhood","Badass","baidu\\.com","Bandit","basicstate","BatchFTP","Battleztar Bazinga","baypup\\\/","BazQux","BBBike","BCKLINKS","BDFetch","BegunAdvertising","Bewica-security-scan","Bidtellect","BigBozz","Bigfoot","biglotron","BingLocalSearch","BingPreview","binlar","biNu image cacher","Bitacle","Bitrix link preview","biz_Directory","BKCTwitterUnshortener\\\/","Black Hole","Blackboard Safeassign","BlackWidow","BlockNote\\.Net","BlogBridge","Bloglines","Bloglovin","BlogPulseLive","BlogSearch","Blogtrottr","BlowFish","boitho\\.com-dc","Boost\\.Beast","BPImageWalker","Braintree-Webhooks","Branch Metrics API","Branch-Passthrough","Brandprotect","BrandVerity","Brandwatch","Brodie\\\/","Browsershots","BUbiNG","Buck\\\/","Buddy","BuiltWith","Bullseye","BunnySlippers","Burf Search","Butterfly\\\/","BuzzSumo","CAAM\\\/[0-9]","CakePHP","Calculon","Canary%20Mail","CaretNail","catexplorador","CC Metadata Scaper","Cegbfeieh","censys","centuryb.o.t9[at]gmail.com","Cerberian Drtrs","CERT\\.at-Statistics-Survey","cf-facebook","cg-eye","changedetection","ChangesMeter","Charlotte","chatterino-api-cache","CheckHost","checkprivacy","CherryPicker","ChinaClaw","Chirp\\\/","chkme\\.com","Chlooe","Chromaxa","CirrusExplorer","CISPA Vulnerability Notification","CISPA Web Analyser","Citoid","CJNetworkQuality","Clarsentia","clips\\.ua\\.ac\\.be","Cloud mapping","CloudEndure","CloudFlare-AlwaysOnline","Cloudflare-Healthchecks","Cloudinary","cmcm\\.com","coccoc","cognitiveseo","ColdFusion","colly -","CommaFeed","Commons-HttpClient","commonscan","contactbigdatafr","contentkingapp","Contextual Code Sites Explorer","convera","CookieReports","copyright sheriff","CopyRightCheck","Copyscape","cortex\\\/","Cosmos4j\\.feedback","Covario-IDS","Craw\\\/","Crescent","Criteo","Crowsnest","CSHttp","CSSCheck","Cula\\\/","curb","Curious George","curl","cuwhois\\\/","cybo\\.com","DAP\\\/NetHTTP","DareBoost","DatabaseDriverMysqli","DataCha0s","DatadogSynthetics","Datafeedwatch","Datanyze","DataparkSearch","dataprovider","DataXu","Daum(oa)?[ \\\/][0-9]","dBpoweramp","ddline","deeris","delve\\.ai","Demon","DeuSu","developers\\.google\\.com\\\/\\+\\\/web\\\/snippet\\\/","Devil","Digg","Digincore","DigitalPebble","Dirbuster","Discourse Forum Onebox","Dispatch\\\/","Disqus\\\/","DittoSpyder","dlvr","DMBrowser","DNSPod-reporting","docoloc","Dolphin http client","DomainAppender","DomainLabz","Domains Project\\\/","Donuts Content Explorer","dotMailer content retrieval","dotSemantic","downforeveryoneorjustme","Download Wonder","downnotifier","DowntimeDetector","Drip","drupact","Drupal \\(\\+http:\\\/\\\/drupal\\.org\\\/\\)","DTS Agent","dubaiindex","DuplexWeb-Google","DynatraceSynthetic","EARTHCOM","Easy-Thumb","EasyDL","Ebingbong","ec2linkfinder","eCairn-Grabber","eCatch","ECCP","eContext\\\/","Ecxi","EirGrabber","ElectricMonk","elefent","EMail Exractor","EMail Wolf","EmailWolf","Embarcadero","Embed PHP Library","Embedly","endo\\\/","europarchive\\.org","evc-batch","EventMachine HttpClient","Everwall Link Expander","Evidon","Evrinid","ExactSearch","ExaleadCloudview","Excel\\\/","exif","ExoRank","Exploratodo","Express WebPictures","Extreme Picture Finder","EyeNetIE","ezooms","facebookexternalhit","facebookexternalua","facebookplatform","fairshare","Faraday v","fasthttp","Faveeo","Favicon downloader","faviconarchive","faviconkit","FavOrg","Feed Wrangler","Feedable\\\/","Feedbin","FeedBooster","FeedBucket","FeedBunch\\\/","FeedBurner","feeder","Feedly","FeedshowOnline","Feedshow\\\/","Feedspot","FeedViewer\\\/","Feedwind\\\/","FeedZcollector","feeltiptop","Fetch API","Fetch\\\/[0-9]","Fever\\\/[0-9]","FHscan","Fiery%20Feeds","Filestack","Fimap","findlink","findthatfile","FlashGet","FlipboardBrowserProxy","FlipboardProxy","FlipboardRSS","Flock\\\/","Florienzh\\\/","fluffy","Flunky","flynxapp","forensiq","ForusP","FoundSeoTool","fragFINN\\.de","free thumbnails","Freeuploader","FreshRSS","frontman","Funnelback","Fuzz Faster U Fool","G-i-g-a-b-o-t","g00g1e\\.net","ganarvisitas","gdnplus\\.com","geek-tools","Genieo","GentleSource","GetCode","Getintent","GetLinkInfo","getprismatic","GetRight","getroot","GetURLInfo\\\/","GetWeb","Geziyor","Ghost Inspector","GigablastOpenSource","GIS-LABS","github-camo","GitHub-Hookshot","github\\.com","Go http package","Go [\\d\\.]* package http","Go!Zilla","Go-Ahead-Got-It","Go-http-client","go-mtasts\\\/","gobuster","gobyus","Gofeed","gofetch","Goldfire Server","GomezAgent","gooblog","Goodzer\\\/","Google AppsViewer","Google Desktop","Google favicon","Google Keyword Suggestion","Google Keyword Tool","Google Page Speed Insights","Google PP Default","Google Search Console","Google Web Preview","Google-Ads-Creatives-Assistant","Google-Ads-Overview","Google-Adwords","Google-Apps-Script","Google-Calendar-Importer","Google-HotelAdsVerifier","Google-HTTP-Java-Client","Google-InspectionTool","Google-Podcast","Google-Publisher-Plugin","Google-Read-Aloud","Google-SearchByImage","Google-Site-Verification","Google-SMTP-STS","Google-speakr","Google-Structured-Data-Testing-Tool","Google-Transparency-Report","google-xrawler","Google-Youtube-Links","GoogleDocs","GoogleHC\\\/","GoogleProber","GoogleProducer","GoogleSites","Gookey","GoSpotCheck","gosquared-thumbnailer","Gotit","GoZilla","grabify","GrabNet","Grafula","Grammarly","GrapeFX","GreatNews","Gregarius","GRequests","grokkit","grouphigh","grub-client","gSOAP\\\/","GT::WWW","GTmetrix","GuzzleHttp","gvfs\\\/","HAA(A)?RTLAND http client","Haansoft","hackney\\\/","Hadi Agent","HappyApps-WebCheck","Hardenize","Hatena","Havij","HaxerMen","HeadlessChrome","HEADMasterSEO","HeartRails_Capture","help@dataminr\\.com","heritrix","Hexometer","historious","hkedcity","hledejLevne\\.cz","Hloader","HMView","Holmes","HonesoSearchEngine","HootSuite Image proxy","Hootsuite-WebFeed","hosterstats","HostTracker","ht:\\\/\\\/check","htdig","HTMLparser","htmlyse","HTTP Banner Detection","http-get","HTTP-Header-Abfrage","http-kit","http-request\\\/","HTTP-Tiny","HTTP::Lite","http:\\\/\\\/www.neomo.de\\\/","HttpComponents","httphr","HTTPie","HTTPMon","httpRequest","httpscheck","httpssites_power","httpunit","HttpUrlConnection","http\\.rb\\\/","HTTP_Compression_Test","http_get","http_request2","http_requester","httrack","huaweisymantec","HubSpot ","HubSpot-Link-Resolver","Humanlinks","i2kconnect\\\/","Iblog","ichiro","Id-search","IdeelaborPlagiaat","IDG Twitter Links Resolver","IDwhois\\\/","Iframely","igdeSpyder","iGooglePortal","IlTrovatore","Image Fetch","Image Sucker","ImageEngine\\\/","ImageVisu\\\/","Imagga","imagineeasy","imgsizer","InAGist","inbound\\.li parser","InDesign%20CC","Indy Library","InetURL","infegy","infohelfer","InfoTekies","InfoWizards Reciprocal Link","inpwrd\\.com","instabid","Instapaper","Integrity","integromedb","Intelliseek","InterGET","Internet Ninja","InternetSeer","internetVista monitor","internetwache","internet_archive","intraVnews","IODC","IOI","Inboxb0t","iplabel","ips-agent","IPS\\\/[0-9]","IPWorks HTTP\\\/S Component","iqdb\\\/","Iria","Irokez","isitup\\.org","iskanie","isUp\\.li","iThemes Sync\\\/","IZaBEE","iZSearch","JAHHO","janforman","Jaunt\\\/","Java.*outbrain","javelin\\.io","Jbrofuzz","Jersey\\\/","JetCar","Jigsaw","Jobboerse","JobFeed discovery","Jobg8 URL Monitor","jobo","Jobrapido","Jobsearch1\\.5","JoinVision Generic","JolokiaPwn","Joomla","Jorgee","JS-Kit","JungleKeyThumbnail","JustView","Kaspersky Lab CFR link resolver","Kelny\\\/","Kerrigan\\\/","KeyCDN","Keyword Density","Keywords Research","khttp\\\/","KickFire","KimonoLabs\\\/","Kml-Google","knows\\.is","KOCMOHABT","kouio","kube-probe","kubectl","kulturarw3","KumKie","Larbin","Lavf\\\/","leakix\\.net","LeechFTP","LeechGet","letsencrypt","Lftp","LibVLC","LibWeb","Libwhisker","libwww","Licorne","Liferea\\\/","Lighthouse","Lightspeedsystems","Likse","limber\\.io","Link Valet","LinkAlarm\\\/","LinkAnalyser","linkCheck","linkdex","LinkExaminer","linkfluence","linkpeek","LinkPreview","LinkScan","LinksManager","LinkTiger","LinkWalker","link_thumbnailer","Lipperhey","Litemage_walker","livedoor ScreenShot","LoadImpactRload","localsearch-web","LongURL API","longurl-r-package","looid\\.com","looksystems\\.net","ltx71","lua-resty-http","Lucee \\(CFML Engine\\)","Lush Http Client","lwp-request","lwp-trivial","LWP::Simple","lycos","LYT\\.SR","L\\.webis","mabontland","MacOutlook\\\/","Mag-Net","MagpieRSS","Mail::STS","MailChimp","Mail\\.Ru","Majestic12","makecontact\\\/","Mandrill","MapperCmd","marketinggrader","MarkMonitor","MarkWatch","Mass Downloader","masscan\\\/","Mata Hari","mattermost","Mediametric","Mediapartners-Google","mediawords","MegaIndex\\.ru","MeltwaterNews","Melvil Rawi","MemGator","Metaspinner","MetaURI","MFC_Tear_Sample","Microsearch","Microsoft Data Access","Microsoft Office","Microsoft Outlook","Microsoft Windows Network Diagnostics","Microsoft-WebDAV-MiniRedir","Microsoft\\.Data\\.Mashup","MIDown tool","MIIxpc","Mindjet","Miniature\\.io","Miniflux","mio_httpc","Miro-HttpClient","Mister PiX","mixdata dot com","mixed-content-scan","mixnode","Mnogosearch","mogimogi","Mojeek","Mojolicious \\(Perl\\)","Mollie","monitis","Monitority\\\/","Monit\\\/","montastic","MonTools","Moreover","Morfeus Fucking Scanner","Morning Paper","MovableType","mowser","Mrcgiguy","Mr\\.4x3 Powered","MS Web Services Client Protocol","MSFrontPage","mShots","MuckRack\\\/","muhstik-scan","MVAClient","MxToolbox\\\/","myseosnapshot","nagios","Najdi\\.si","Name Intelligence","NameFo\\.com","Nameprotect","nationalarchives","Navroad","NearSite","Needle","Nessus","Net Vampire","NetAnts","NETCRAFT","NetLyzer","NetMechanic","NetNewsWire","Netpursual","netresearch","NetShelter ContentScan","Netsparker","NetSystemsResearch","nettle","NetTrack","Netvibes","NetZIP","Neustar WPM","NeutrinoAPI","NewRelicPinger","NewsBlur .*Finder","NewsGator","newsme","newspaper\\\/","Nexgate Ruby Client","NG-Search","nghttp2","Nibbler","NICErsPRO","NihilScio","Nikto","nineconnections","NLNZ_IAHarvester","Nmap Scripting Engine","node-fetch","node-superagent","node-urllib","Nodemeter","NodePing","node\\.io","nominet\\.org\\.uk","nominet\\.uk","Norton-Safeweb","Notifixious","notifyninja","NotionEmbedder","nuhk","nutch","Nuzzel","nWormFeedFinder","nyawc\\\/","Nymesis","NYU","Observatory\\\/","Ocelli\\\/","Octopus","oegp","Offline Explorer","Offline Navigator","OgScrper","okhttp","omgili","OMSC","Online Domain Tools","Open Source RSS","OpenCalaisSemanticProxy","Openfind","OpenLinkProfiler","Openstat\\\/","OpenVAS","OPPO A33","Optimizer","Orbiter","OrgProbe\\\/","orion-semantics","Outlook-Express","Outlook-iOS","Owler","Owlin","ownCloud News","ow\\.ly","OxfordCloudService","page scorer","Page Valet","page2rss","PageFreezer","PageGrabber","PagePeeker","PageScorer","Pagespeed\\\/","PageThing","page_verifier","Panopta","panscient","Papa Foto","parsijoo","Pavuk","PayPal IPN","pcBrowser","Pcore-HTTP","PDF24 URL To PDF","Pearltrees","PECL::HTTP","peerindex","Peew","PeoplePal","Perlu -","PhantomJS Screenshoter","PhantomJS\\\/","Photon\\\/","php-requests","phpservermon","Pi-Monster","Picscout","Picsearch","PictureFinder","Pimonster","Pingability","PingAdmin\\.Ru","Pingdom","Pingoscope","PingSpot","ping\\.blo\\.gs","pinterest\\.com","Pixray","Pizilla","Plagger\\\/","Pleroma ","Ploetz \\+ Zeller","Plukkie","plumanalytics","PocketImageCache","PocketParser","Pockey","PodcastAddict\\\/","POE-Component-Client-HTTP","Polymail\\\/","Pompos","Porkbun","Port Monitor","postano","postfix-mta-sts-resolver","PostmanRuntime","postplanner\\.com","PostPost","postrank","PowerPoint\\\/","Prebid","Prerender","Priceonomics Analysis Engine","PrintFriendly","PritTorrent","Prlog","probethenet","Project ?25499","Project-Resonance","prospectb2b","Protopage","ProWebWalker","proximic","PRTG Network Monitor","pshtt, https scanning","PTST ","PTST\\\/[0-9]+","Pump","Python-httplib2","python-httpx","python-requests","Python-urllib","Qirina Hurdler","QQDownload","QrafterPro","Qseero","Qualidator","QueryN Metasearch","queuedriver","quic-go-HTTP\\\/","QuiteRSS","Quora Link Preview","Qwantify","Radian6","RadioPublicImageResizer","Railgun\\\/","RankActive","RankFlex","RankSonicSiteAuditor","RapidLoad\\\/","Re-re Studio","ReactorNetty","Readability","RealDownload","RealPlayer%20Downloader","RebelMouse","Recorder","RecurPost\\\/","redback\\\/","ReederForMac","Reeder\\\/","ReGet","RepoMonkey","request\\.js","reqwest\\\/","ResponseCodeTest","RestSharp","Riddler","Rival IQ","Robosourcer","Robozilla","ROI Hunter","RPT-HTTPClient","RSSMix\\\/","RSSOwl","RyowlEngine","safe-agent-scanner","SalesIntelligent","Saleslift","SAP NetWeaver Application Server","SauceNAO","SBIder","sc-downloader","scalaj-http","Scamadviser-Frontend","ScanAlert","scan\\.lol","Scoop","scooter","ScopeContentAG-HTTP-Client","ScoutJet","ScoutURLMonitor","ScrapeBox Page Scanner","Scrapy","Screaming","ScreenShotService","Scrubby","Scrutiny\\\/","Search37","searchenginepromotionhelp","Searchestate","SearchExpress","SearchSight","SearchWP","search\\.thunderstone","Seeker","semanticdiscovery","semanticjuice","Semiocast HTTP client","Semrush","Sendsay\\.Ru","sentry\\\/","SEO Browser","Seo Servis","seo-nastroj\\.cz","seo4ajax","Seobility","SEOCentro","SeoCheck","seocompany","SEOkicks","SEOlizer","Seomoz","SEOprofiler","seoscanners","SEOsearch","seositecheckup","SEOstats","servernfo","sexsearcher","Seznam","Shelob","Shodan","Shoppimon","ShopWiki","ShortLinkTranslate","shortURL lengthener","shrinktheweb","Sideqik","Siege","SimplePie","SimplyFast","Siphon","SISTRIX","Site Sucker","Site-Shot\\\/","Site24x7","SiteBar","Sitebeam","Sitebulb\\\/","SiteCondor","SiteExplorer","SiteGuardian","Siteimprove","SiteIndexed","Sitemap(s)? Generator","SitemapGenerator","SiteMonitor","Siteshooter B0t","SiteSnagger","SiteSucker","SiteTruth","Sitevigil","sitexy\\.com","SkypeUriPreview","Slack\\\/","sli-systems\\.com","slider\\.com","slurp","SlySearch","SmartDownload","SMRF URL Expander","SMUrlExpander","Snake","Snappy","SnapSearch","Snarfer\\\/","SniffRSS","sniptracker","Snoopy","SnowHaze Search","sogou web","SortSite","Sottopop","sovereign\\.ai","SpaceBison","SpamExperts","Spammen","Spanner","Spawning-AI","spaziodati","SPDYCheck","Specificfeeds","SpeedKit","speedy","SPEng","Spinn3r","spray-can","Sprinklr ","spyonweb","sqlmap","Sqlworm","Sqworm","SSL Labs","ssl-tools","StackRambler","Statastico\\\/","Statically-","StatusCake","Steeler","Stratagems Kumo","Stripe\\\/","Stroke\\.cz","StudioFACA","StumbleUpon","suchen","Sucuri","summify","SuperHTTP","Surphace Scout","Suzuran","swcd ","Symfony BrowserKit","Symfony2 BrowserKit","Synapse\\\/","Syndirella\\\/","SynHttpClient-Built","Sysomos","sysscan","Szukacz","T0PHackTeam","tAkeOut","Tarantula\\\/","Taringa UGC","TarmotGezgin","tchelebi\\.io","techiaith\\.cymru","Teleport","Telesoft","Telesphoreo","Telesphorep","Tenon\\.io","teoma","terrainformatica","Test Certificate Info","testuri","Tetrahedron","TextRazor Downloader","The Drop Reaper","The Expert HTML Source Viewer","The Intraformant","The Knowledge AI","theinternetrules","TheNomad","Thinklab","Thumbor","Thumbshots","ThumbSniper","timewe\\.net","TinEye","Tiny Tiny RSS","TLSProbe\\\/","Toata","topster","touche\\.com","Traackr\\.com","tracemyfile","Trackuity","TrapitAgent","Trendiction","Trendsmap","trendspottr","truwoGPS","TryJsoup","TulipChain","Turingos","Turnitin","tweetedtimes","Tweetminster","Tweezler\\\/","twibble","Twice","Twikle","Twingly","Twisted PageGetter","Typhoeus","ubermetrics-technologies","uclassify","UdmSearch","ultimate_sitemap_parser","unchaos","unirest-java","UniversalFeedParser","unshortenit","Unshorten\\.It","Untiny","UnwindFetchor","updated","updown\\.io daemon","Upflow","Uptimia","URL Verifier","Urlcheckr","URLitor","urlresolver","Urlstat","URLTester","UrlTrends Ranking Updater","URLy Warning","URLy\\.Warning","URL\\\/Emacs","Vacuum","Vagabondo","VB Project","vBSEO","VCI","via ggpht\\.com GoogleImageProxy","Virusdie","visionutils","Visual Rights Group","vkShare","VoidEYE","Voil","voltron","voyager\\\/","VSAgent\\\/","VSB-TUO\\\/","Vulnbusters Meter","VYU2","w3af\\.org","W3C-checklink","W3C-mobileOK","W3C_Unicorn","WAC-OFU","WakeletLinkExpander","WallpapersHD","Wallpapers\\\/[0-9]+","wangling","Wappalyzer","WatchMouse","WbSrch\\\/","WDT\\.io","Web Auto","Web Collage","Web Enhancer","Web Fetch","Web Fuck","Web Pix","Web Sauger","Web spyder","Web Sucker","web-capture\\.net","Web-sniffer","Webalta","Webauskunft","WebAuto","WebCapture","WebClient\\\/","webcollage","WebCookies","WebCopier","WebCorp","WebDataStats","WebDoc","WebEnhancer","WebFetch","WebFuck","WebGazer","WebGo IS","WebImageCollector","WebImages","WebIndex","webkit2png","WebLeacher","webmastercoffee","webmon ","WebPix","WebReaper","WebSauger","webscreenie","Webshag","Webshot","Website Quester","websitepulse agent","WebsiteQuester","Websnapr","WebSniffer","Webster","WebStripper","WebSucker","webtech\\\/","WebThumbnail","Webthumb\\\/","WebWhacker","WebZIP","WeLikeLinks","WEPA","WeSEE","wf84","Wfuzz\\\/","wget","WhatCMS","WhatsApp","WhatsMyIP","WhatWeb","WhereGoes\\?","Whibse","WhoAPI\\\/","WhoRunsCoinHive","Whynder Magnet","Windows-RSS-Platform","WinHttp-Autoproxy-Service","WinHTTP\\\/","WinPodder","wkhtmlto","wmtips","Woko","Wolfram HTTPClient","woorankreview","WordPress\\\/","WordupinfoSearch","Word\\\/","worldping-api","wotbox","WP Engine Install Performance API","WP Rocket","wpif","wprecon\\.com survey","WPScan","wscheck","Wtrace","WWW-Collector-E","WWW-Mechanize","WWW::Document","WWW::Mechanize","WWWOFFLE","www\\.monitor\\.us","x09Mozilla","x22Mozilla","XaxisSemanticsClassifier","XenForo\\\/","Xenu Link Sleuth","XING-contenttabreceiver","xpymep([0-9]?)\\.exe","Y!J-[A-Z][A-Z][A-Z]","Yaanb","yacy","Yahoo Link Preview","YahooCacheSystem","YahooMailProxy","YahooYSMcm","YandeG","Yandex(?!Search)","yanga","yeti","Yo-yo","Yoleo Consumer","yomins\\.com","yoogliFetchAgent","YottaaMonitor","Your-Website-Sucks","yourls\\.org","YoYs\\.net","YP\\.PL","Zabbix","Zade","Zao","Zauba","Zemanta Aggregator","Zend\\\\Http\\\\Client","Zend_Http_Client","Zermelo","Zeus ","zgrab","ZnajdzFoto","ZnHTTP","Zombie\\.js","Zoom\\.Mac","ZoteroTranslationServer","ZyBorg","[a-z0-9\\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)"] \ No newline at end of file diff --git a/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt b/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt index 1522796e..791d08bb 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt +++ b/blockbot/vendor/jaybizzle/crawler-detect/raw/Crawlers.txt @@ -1,27 +1,46 @@ -.*Java.*outbrain YLT +^Aether +^Amazon Simple Notification Service Agent$ +^Amazon-Route53-Health-Check-Service ^b0t$ ^bluefish ^Calypso v\/ ^COMODO DCV +^Corax ^DangDang ^DavClnt +^DHSH +^docker\/[0-9] +^Expanse ^FDM ^git\/ ^Goose\/ ^Grabber +^Gradle\/ ^HTTPClient\/ +^HTTPing ^Java\/ ^Jeode\/ ^Jetty\/ ^Mail\/ ^Mget ^Microsoft URL Control +^Mikrotik\/ +^Netlab360 ^NG\/[0-9\.] ^NING\/ -^PHP\/[0-9] +^npm\/ +^Nuclei +^PHP-AYMAPI\/ +^PHP\/ +^pip\/ +^pnpm\/ ^RMA\/ ^Ruby|Ruby\/[0-9] +^Swurl +^TLS tester +^twine\/ +^ureq ^VSE\/[0-9] ^WordPress\.com ^XRL\/[0-9] @@ -29,17 +48,18 @@ 008\/ 13TABS 192\.comAgent +2GDPR\/ 2ip\.ru 404enemy 7Siters 80legs -a\.pr-cy\.ru a3logics\.in A6-Indexer Abonti Aboundex aboutthedomain Accoona-AI-Agent +acebookexternalhit\/ acoon acrylicapps\.com\/pulp Acunetix @@ -49,8 +69,14 @@ AddThis ADmantX AdminLabs adressendeutschland +adreview\/ adscanner +adstxt-worker Adstxtaggregator +adstxt\.com +Adyen HttpClient +AffiliateLabz\/ +affilimate-puppeteer agentslug AHC aihit @@ -71,20 +97,23 @@ AndroidDownloadManager Anemone AngleSharp annotate_google -Ant\.com +Anthill Anturis Agent +Ant\.com AnyEvent-HTTP\/ +Apache Ant\/ Apache Droid Apache OpenOffice Apache-HttpAsyncClient Apache-HttpClient ApacheBench Apexoo +apimon\.de APIs-Google AportWorm\/ AppBeat\/ AppEngine-Google -AppStoreScraperZ +AppleSyndication Aprc\/[0-9] Arachmo arachnode @@ -92,21 +121,28 @@ Arachnophilia aria2 Arukereso asafaweb -AskQuickly +Asana\/ Ask Jeeves +AskQuickly ASPSeek Asterias Astute asynchttp Attach +attohttpc autocite +AutomaticWPTester Autonomy +awin\.com +AWS Security Scanner axios\/ +a\.pr-cy\.ru B-l-i-t-z-B-O-T Backlink-Ceck backlink-check BacklinkHttpStatus BackStreet +BackupLand BackWeb Bad-Neighborhood Badass @@ -121,6 +157,7 @@ BBBike BCKLINKS BDFetch BegunAdvertising +Bewica-security-scan Bidtellect BigBozz Bigfoot @@ -130,11 +167,14 @@ BingPreview binlar biNu image cacher Bitacle +Bitrix link preview biz_Directory +BKCTwitterUnshortener\/ Black Hole Blackboard Safeassign BlackWidow BlockNote\.Net +BlogBridge Bloglines Bloglovin BlogPulseLive @@ -142,6 +182,7 @@ BlogSearch Blogtrottr BlowFish boitho\.com-dc +Boost\.Beast BPImageWalker Braintree-Webhooks Branch Metrics API @@ -169,12 +210,15 @@ catexplorador CC Metadata Scaper Cegbfeieh censys +centuryb.o.t9[at]gmail.com Cerberian Drtrs CERT\.at-Statistics-Survey +cf-facebook cg-eye changedetection ChangesMeter Charlotte +chatterino-api-cache CheckHost checkprivacy CherryPicker @@ -185,6 +229,7 @@ Chlooe Chromaxa CirrusExplorer CISPA Vulnerability Notification +CISPA Web Analyser Citoid CJNetworkQuality Clarsentia @@ -192,27 +237,34 @@ clips\.ua\.ac\.be Cloud mapping CloudEndure CloudFlare-AlwaysOnline +Cloudflare-Healthchecks Cloudinary cmcm\.com coccoc cognitiveseo +ColdFusion colly - CommaFeed Commons-HttpClient commonscan contactbigdatafr contentkingapp +Contextual Code Sites Explorer convera CookieReports copyright sheriff CopyRightCheck Copyscape +cortex\/ Cosmos4j\.feedback Covario-IDS +Craw\/ Crescent -Crowsnest Criteo +Crowsnest CSHttp +CSSCheck +Cula\/ curb Curious George curl @@ -222,12 +274,17 @@ DAP\/NetHTTP DareBoost DatabaseDriverMysqli DataCha0s +DatadogSynthetics Datafeedwatch Datanyze DataparkSearch dataprovider DataXu Daum(oa)?[ \/][0-9] +dBpoweramp +ddline +deeris +delve\.ai Demon DeuSu developers\.google\.com\/\+\/web\/snippet\/ @@ -237,8 +294,8 @@ Digincore DigitalPebble Dirbuster Discourse Forum Onebox -Disqus\/ Dispatch\/ +Disqus\/ DittoSpyder dlvr DMBrowser @@ -246,6 +303,8 @@ DNSPod-reporting docoloc Dolphin http client DomainAppender +DomainLabz +Domains Project\/ Donuts Content Explorer dotMailer content retrieval dotSemantic @@ -258,6 +317,8 @@ drupact Drupal \(\+http:\/\/drupal\.org\/\) DTS Agent dubaiindex +DuplexWeb-Google +DynatraceSynthetic EARTHCOM Easy-Thumb EasyDL @@ -288,20 +349,22 @@ ExactSearch ExaleadCloudview Excel\/ exif +ExoRank Exploratodo Express WebPictures Extreme Picture Finder EyeNetIE ezooms facebookexternalhit +facebookexternalua facebookplatform fairshare Faraday v fasthttp Faveeo Favicon downloader -faviconkit faviconarchive +faviconkit FavOrg Feed Wrangler Feedable\/ @@ -313,7 +376,9 @@ FeedBurner feeder Feedly FeedshowOnline +Feedshow\/ Feedspot +FeedViewer\/ Feedwind\/ FeedZcollector feeltiptop @@ -321,6 +386,8 @@ Fetch API Fetch\/[0-9] Fever\/[0-9] FHscan +Fiery%20Feeds +Filestack Fimap findlink findthatfile @@ -329,18 +396,24 @@ FlipboardBrowserProxy FlipboardProxy FlipboardRSS Flock\/ +Florienzh\/ fluffy Flunky flynxapp forensiq +ForusP FoundSeoTool -http:\/\/www.neomo.de\/ +fragFINN\.de free thumbnails Freeuploader +FreshRSS +frontman Funnelback +Fuzz Faster U Fool G-i-g-a-b-o-t g00g1e\.net ganarvisitas +gdnplus\.com geek-tools Genieo GentleSource @@ -352,18 +425,24 @@ GetRight getroot GetURLInfo\/ GetWeb +Geziyor Ghost Inspector GigablastOpenSource GIS-LABS github-camo +GitHub-Hookshot github\.com -Go [\d\.]* package http Go http package +Go [\d\.]* package http +Go!Zilla Go-Ahead-Got-It Go-http-client -Go!Zilla +go-mtasts\/ +gobuster gobyus +Gofeed gofetch +Goldfire Server GomezAgent gooblog Goodzer\/ @@ -376,24 +455,31 @@ Google Page Speed Insights Google PP Default Google Search Console Google Web Preview +Google-Ads-Creatives-Assistant +Google-Ads-Overview Google-Adwords Google-Apps-Script Google-Calendar-Importer Google-HotelAdsVerifier Google-HTTP-Java-Client +Google-InspectionTool +Google-Podcast Google-Publisher-Plugin +Google-Read-Aloud Google-SearchByImage Google-Site-Verification +Google-SMTP-STS +Google-speakr Google-Structured-Data-Testing-Tool -Google-Youtube-Links +Google-Transparency-Report google-xrawler +Google-Youtube-Links GoogleDocs GoogleHC\/ +GoogleProber GoogleProducer GoogleSites -Google-Transparency-Report Gookey -GoScraper GoSpotCheck gosquared-thumbnailer Gotit @@ -419,13 +505,16 @@ Haansoft hackney\/ Hadi Agent HappyApps-WebCheck +Hardenize Hatena Havij +HaxerMen HeadlessChrome HEADMasterSEO HeartRails_Capture help@dataminr\.com heritrix +Hexometer historious hkedcity hledejLevne\.cz @@ -442,28 +531,31 @@ htdig HTMLparser htmlyse HTTP Banner Detection -HTTP_Compression_Test -http_request2 -http_requester http-get HTTP-Header-Abfrage http-kit http-request\/ HTTP-Tiny HTTP::Lite -http\.rb\/ -http_get +http:\/\/www.neomo.de\/ HttpComponents httphr +HTTPie HTTPMon httpRequest httpscheck httpssites_power httpunit HttpUrlConnection +http\.rb\/ +HTTP_Compression_Test +http_get +http_request2 +http_requester httrack huaweisymantec HubSpot +HubSpot-Link-Resolver Humanlinks i2kconnect\/ Iblog @@ -474,6 +566,7 @@ IDG Twitter Links Resolver IDwhois\/ Iframely igdeSpyder +iGooglePortal IlTrovatore Image Fetch Image Sucker @@ -498,13 +591,15 @@ Integrity integromedb Intelliseek InterGET -internet_archive Internet Ninja InternetSeer internetVista monitor +internetwache +internet_archive intraVnews IODC IOI +Inboxb0t iplabel ips-agent IPS\/[0-9] @@ -516,10 +611,13 @@ isitup\.org iskanie isUp\.li iThemes Sync\/ +IZaBEE iZSearch JAHHO janforman Jaunt\/ +Java.*outbrain +javelin\.io Jbrofuzz Jersey\/ JetCar @@ -535,6 +633,7 @@ JolokiaPwn Joomla Jorgee JS-Kit +JungleKeyThumbnail JustView Kaspersky Lab CFR link resolver Kelny\/ @@ -542,6 +641,7 @@ Kerrigan\/ KeyCDN Keyword Density Keywords Research +khttp\/ KickFire KimonoLabs\/ Kml-Google @@ -549,11 +649,12 @@ knows\.is KOCMOHABT kouio kube-probe +kubectl kulturarw3 KumKie -L\.webis Larbin Lavf\/ +leakix\.net LeechFTP LeechGet letsencrypt @@ -564,41 +665,50 @@ Libwhisker libwww Licorne Liferea\/ -Lightspeedsystems Lighthouse +Lightspeedsystems Likse +limber\.io Link Valet -link_thumbnailer LinkAlarm\/ +LinkAnalyser linkCheck linkdex LinkExaminer linkfluence linkpeek -LinkPreviewGenerator +LinkPreview LinkScan LinksManager LinkTiger LinkWalker +link_thumbnailer Lipperhey Litemage_walker livedoor ScreenShot LoadImpactRload localsearch-web LongURL API +longurl-r-package +looid\.com looksystems\.net ltx71 lua-resty-http +Lucee \(CFML Engine\) +Lush Http Client lwp-request lwp-trivial LWP::Simple lycos LYT\.SR +L\.webis mabontland +MacOutlook\/ Mag-Net MagpieRSS -Mail\.Ru +Mail::STS MailChimp +Mail\.Ru Majestic12 makecontact\/ Mandrill @@ -609,6 +719,8 @@ MarkWatch Mass Downloader masscan\/ Mata Hari +mattermost +Mediametric Mediapartners-Google mediawords MegaIndex\.ru @@ -619,28 +731,31 @@ Metaspinner MetaURI MFC_Tear_Sample Microsearch -Microsoft Office +Microsoft Data Access +Microsoft Office Microsoft Outlook Microsoft Windows Network Diagnostics Microsoft-WebDAV-MiniRedir -Microsoft Data Access +Microsoft\.Data\.Mashup MIDown tool MIIxpc Mindjet Miniature\.io Miniflux +mio_httpc +Miro-HttpClient Mister PiX mixdata dot com mixed-content-scan -Mixmax-LinkPreview mixnode Mnogosearch mogimogi Mojeek Mojolicious \(Perl\) -Monit\/ +Mollie monitis Monitority\/ +Monit\/ montastic MonTools Moreover @@ -649,6 +764,7 @@ Morning Paper MovableType mowser Mrcgiguy +Mr\.4x3 Powered MS Web Services Client Protocol MSFrontPage mShots @@ -656,10 +772,13 @@ MuckRack\/ muhstik-scan MVAClient MxToolbox\/ +myseosnapshot nagios Najdi\.si Name Intelligence +NameFo\.com Nameprotect +nationalarchives Navroad NearSite Needle @@ -674,6 +793,8 @@ Netpursual netresearch NetShelter ContentScan Netsparker +NetSystemsResearch +nettle NetTrack Netvibes NetZIP @@ -686,22 +807,26 @@ newsme newspaper\/ Nexgate Ruby Client NG-Search +nghttp2 Nibbler NICErsPRO +NihilScio Nikto nineconnections NLNZ_IAHarvester Nmap Scripting Engine +node-fetch node-superagent node-urllib -node\.io Nodemeter NodePing +node\.io nominet\.org\.uk nominet\.uk Norton-Safeweb Notifixious notifyninja +NotionEmbedder nuhk nutch Nuzzel @@ -709,39 +834,45 @@ nWormFeedFinder nyawc\/ Nymesis NYU +Observatory\/ Ocelli\/ Octopus oegp Offline Explorer Offline Navigator -og-scraper +OgScrper okhttp omgili OMSC Online Domain Tools +Open Source RSS OpenCalaisSemanticProxy Openfind OpenLinkProfiler Openstat\/ OpenVAS +OPPO A33 Optimizer Orbiter OrgProbe\/ orion-semantics Outlook-Express Outlook-iOS -ow\.ly Owler +Owlin ownCloud News +ow\.ly OxfordCloudService -Page Valet -page_verifier page scorer +Page Valet page2rss +PageFreezer PageGrabber PagePeeker PageScorer Pagespeed\/ +PageThing +page_verifier Panopta panscient Papa Foto @@ -750,6 +881,7 @@ Pavuk PayPal IPN pcBrowser Pcore-HTTP +PDF24 URL To PDF Pearltrees PECL::HTTP peerindex @@ -759,44 +891,52 @@ Perlu - PhantomJS Screenshoter PhantomJS\/ Photon\/ +php-requests phpservermon Pi-Monster Picscout Picsearch PictureFinder Pimonster -ping\.blo\.gs Pingability PingAdmin\.Ru Pingdom Pingoscope PingSpot +ping\.blo\.gs pinterest\.com Pixray Pizilla Plagger\/ +Pleroma Ploetz \+ Zeller Plukkie plumanalytics PocketImageCache PocketParser Pockey +PodcastAddict\/ POE-Component-Client-HTTP Polymail\/ Pompos Porkbun Port Monitor postano +postfix-mta-sts-resolver PostmanRuntime +postplanner\.com PostPost postrank PowerPoint\/ +Prebid +Prerender Priceonomics Analysis Engine PrintFriendly PritTorrent Prlog probethenet -Project 25499 +Project ?25499 +Project-Resonance prospectb2b Protopage ProWebWalker @@ -805,9 +945,9 @@ PRTG Network Monitor pshtt, https scanning PTST PTST\/[0-9]+ -Pulsepoint XT3 web scraper Pump Python-httplib2 +python-httpx python-requests Python-urllib Qirina Hurdler @@ -817,12 +957,17 @@ Qseero Qualidator QueryN Metasearch queuedriver +quic-go-HTTP\/ +QuiteRSS Quora Link Preview Qwantify Radian6 +RadioPublicImageResizer +Railgun\/ RankActive RankFlex RankSonicSiteAuditor +RapidLoad\/ Re-re Studio ReactorNetty Readability @@ -833,6 +978,7 @@ Recorder RecurPost\/ redback\/ ReederForMac +Reeder\/ ReGet RepoMonkey request\.js @@ -845,38 +991,44 @@ Robosourcer Robozilla ROI Hunter RPT-HTTPClient +RSSMix\/ RSSOwl +RyowlEngine safe-agent-scanner SalesIntelligent Saleslift -Sendsay\.Ru +SAP NetWeaver Application Server SauceNAO SBIder +sc-downloader scalaj-http -scan\.lol +Scamadviser-Frontend ScanAlert +scan\.lol Scoop scooter +ScopeContentAG-HTTP-Client ScoutJet ScoutURLMonitor ScrapeBox Page Scanner -SimpleScraper Scrapy Screaming ScreenShotService Scrubby Scrutiny\/ -search\.thunderstone Search37 searchenginepromotionhelp Searchestate SearchExpress SearchSight +SearchWP +search\.thunderstone Seeker semanticdiscovery semanticjuice Semiocast HTTP client Semrush +Sendsay\.Ru sentry\/ SEO Browser Seo Servis @@ -885,11 +1037,13 @@ seo4ajax Seobility SEOCentro SeoCheck +seocompany SEOkicks +SEOlizer Seomoz SEOprofiler -SEOsearch seoscanners +SEOsearch seositecheckup SEOstats servernfo @@ -900,14 +1054,16 @@ Shodan Shoppimon ShopWiki ShortLinkTranslate +shortURL lengthener shrinktheweb Sideqik +Siege SimplePie SimplyFast Siphon SISTRIX -Site-Shot\/ Site Sucker +Site-Shot\/ Site24x7 SiteBar Sitebeam @@ -928,6 +1084,7 @@ Sitevigil sitexy\.com SkypeUriPreview Slack\/ +sli-systems\.com slider\.com slurp SlySearch @@ -950,9 +1107,11 @@ SpaceBison SpamExperts Spammen Spanner +Spawning-AI spaziodati SPDYCheck Specificfeeds +SpeedKit speedy SPEng Spinn3r @@ -966,9 +1125,11 @@ SSL Labs ssl-tools StackRambler Statastico\/ +Statically- StatusCake Steeler Stratagems Kumo +Stripe\/ Stroke\.cz StudioFACA StumbleUpon @@ -978,9 +1139,11 @@ summify SuperHTTP Surphace Scout Suzuran -SwiteScraper +swcd Symfony BrowserKit Symfony2 BrowserKit +Synapse\/ +Syndirella\/ SynHttpClient-Built Sysomos sysscan @@ -990,6 +1153,8 @@ tAkeOut Tarantula\/ Taringa UGC TarmotGezgin +tchelebi\.io +techiaith\.cymru Teleport Telesoft Telesphoreo @@ -1000,13 +1165,15 @@ terrainformatica Test Certificate Info testuri Tetrahedron +TextRazor Downloader The Drop Reaper The Expert HTML Source Viewer -The Knowledge AI The Intraformant +The Knowledge AI theinternetrules TheNomad Thinklab +Thumbor Thumbshots ThumbSniper timewe\.net @@ -1040,9 +1207,11 @@ Typhoeus ubermetrics-technologies uclassify UdmSearch +ultimate_sitemap_parser unchaos unirest-java UniversalFeedParser +unshortenit Unshorten\.It Untiny UnwindFetchor @@ -1050,8 +1219,8 @@ updated updown\.io daemon Upflow Uptimia -Urlcheckr URL Verifier +Urlcheckr URLitor urlresolver Urlstat @@ -1059,15 +1228,16 @@ URLTester UrlTrends Ranking Updater URLy Warning URLy\.Warning +URL\/Emacs Vacuum Vagabondo VB Project vBSEO VCI via ggpht\.com GoogleImageProxy -VidibleScraper Virusdie visionutils +Visual Rights Group vkShare VoidEYE Voil @@ -1078,19 +1248,18 @@ VSB-TUO\/ Vulnbusters Meter VYU2 w3af\.org -W3C_Unicorn W3C-checklink W3C-mobileOK +W3C_Unicorn WAC-OFU -Wallpapers\/[0-9]+ +WakeletLinkExpander WallpapersHD +Wallpapers\/[0-9]+ wangling Wappalyzer WatchMouse WbSrch\/ WDT\.io -web-capture\.net -Web-sniffer Web Auto Web Collage Web Enhancer @@ -1098,7 +1267,10 @@ Web Fetch Web Fuck Web Pix Web Sauger +Web spyder Web Sucker +web-capture\.net +Web-sniffer Webalta Webauskunft WebAuto @@ -1136,8 +1308,9 @@ WebSniffer Webster WebStripper WebSucker -Webthumb\/ +webtech\/ WebThumbnail +Webthumb\/ WebWhacker WebZIP WeLikeLinks @@ -1146,24 +1319,31 @@ WeSEE wf84 Wfuzz\/ wget +WhatCMS WhatsApp WhatsMyIP WhatWeb WhereGoes\? Whibse +WhoAPI\/ WhoRunsCoinHive Whynder Magnet Windows-RSS-Platform +WinHttp-Autoproxy-Service +WinHTTP\/ WinPodder wkhtmlto wmtips Woko +Wolfram HTTPClient woorankreview -Word\/ WordPress\/ WordupinfoSearch +Word\/ +worldping-api wotbox WP Engine Install Performance API +WP Rocket wpif wprecon\.com survey WPScan @@ -1173,20 +1353,21 @@ WWW-Collector-E WWW-Mechanize WWW::Document WWW::Mechanize -www\.monitor\.us WWWOFFLE +www\.monitor\.us x09Mozilla x22Mozilla XaxisSemanticsClassifier +XenForo\/ Xenu Link Sleuth XING-contenttabreceiver xpymep([0-9]?)\.exe -Y!J-(ASR|BSC) -Y\!J-BRW +Y!J-[A-Z][A-Z][A-Z] Yaanb yacy Yahoo Link Preview YahooCacheSystem +YahooMailProxy YahooYSMcm YandeG Yandex(?!Search) @@ -1194,6 +1375,7 @@ yanga yeti Yo-yo Yoleo Consumer +yomins\.com yoogliFetchAgent YottaaMonitor Your-Website-Sucks @@ -1205,13 +1387,15 @@ Zade Zao Zauba Zemanta Aggregator -Zend_Http_Client Zend\\Http\\Client +Zend_Http_Client Zermelo Zeus zgrab ZnajdzFoto +ZnHTTP Zombie\.js Zoom\.Mac +ZoteroTranslationServer ZyBorg -[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer) \ No newline at end of file +[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper) \ No newline at end of file diff --git a/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json b/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json index a18eb985..e7e01416 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json +++ b/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.json @@ -1 +1 @@ -["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT",";"] \ No newline at end of file +["Safari.[\\d\\.]*","Firefox.[\\d\\.]*"," Chrome.[\\d\\.]*","Chromium.[\\d\\.]*","MSIE.[\\d\\.]","Opera\\\/[\\d\\.]*","Mozilla.[\\d\\.]*","AppleWebKit.[\\d\\.]*","Trident.[\\d\\.]*","Windows NT.[\\d\\.]*","Android [\\d\\.]*","Macintosh.","Ubuntu","Linux","[ ]Intel","Mac OS X [\\d_]*","(like )?Gecko(.[\\d\\.]*)?","KHTML,","CriOS.[\\d\\.]*","CPU iPhone OS ([0-9_])* like Mac OS X","CPU OS ([0-9_])* like Mac OS X","iPod","compatible","x86_..","i686","x64","X11","rv:[\\d\\.]*","Version.[\\d\\.]*","WOW64","Win64","Dalvik.[\\d\\.]*"," \\.NET CLR [\\d\\.]*","Presto.[\\d\\.]*","Media Center PC","BlackBerry","Build","Opera Mini\\\/\\d{1,2}\\.\\d{1,2}\\.[\\d\\.]*\\\/\\d{1,2}\\.","Opera"," \\.NET[\\d\\.]*","cubot","; M bot","; CRONO","; B bot","; IDbot","; ID bot","; POWER BOT","OCTOPUS-CORE","htc_botdugls","super\\\/\\d+\\\/Android\\\/\\d+"] \ No newline at end of file diff --git a/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt b/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt index da56db9b..a44a99cb 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt +++ b/blockbot/vendor/jaybizzle/crawler-detect/raw/Exclusions.txt @@ -45,4 +45,6 @@ cubot ; IDbot ; ID bot ; POWER BOT -; \ No newline at end of file +OCTOPUS-CORE +htc_botdugls +super\/\d+\/Android\/\d+ \ No newline at end of file diff --git a/blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php b/blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php index 1067976b..3ea284a7 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php +++ b/blockbot/vendor/jaybizzle/crawler-detect/src/CrawlerDetect.php @@ -20,9 +20,9 @@ class CrawlerDetect /** * The user agent. * - * @var null + * @var string|null */ - protected $userAgent = null; + protected $userAgent; /** * Headers that contain a user agent. @@ -93,7 +93,7 @@ class CrawlerDetect * Compile the regex patterns into one regex string. * * @param array - * + * * @return string */ public function compileRegex($patterns) @@ -138,7 +138,7 @@ class CrawlerDetect /** * Set the user agent. * - * @param string $userAgent + * @param string|null $userAgent */ public function setUserAgent($userAgent) { @@ -165,20 +165,14 @@ class CrawlerDetect $agent = trim(preg_replace( "/{$this->compiledExclusions}/i", '', - $userAgent ?: $this->userAgent + $userAgent ?: $this->userAgent ?: '' )); - if ($agent == '') { + if ($agent === '') { return false; } - $result = preg_match("/{$this->compiledRegex}/i", $agent, $matches); - - if ($matches) { - $this->matches = $matches; - } - - return (bool) $result; + return (bool) preg_match("/{$this->compiledRegex}/i", $agent, $this->matches); } /** @@ -190,4 +184,13 @@ class CrawlerDetect { return isset($this->matches[0]) ? $this->matches[0] : null; } + + + /** + * @return string|null + */ + public function getUserAgent() + { + return $this->userAgent; + } } diff --git a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php index 26ea8e5f..ffe10f51 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php +++ b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/AbstractProvider.php @@ -15,14 +15,14 @@ abstract class AbstractProvider { /** * The data set. - * + * * @var array */ protected $data; /** * Return the data set. - * + * * @return array */ public function getAll() diff --git a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php index a9070565..a44edd23 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php +++ b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Crawlers.php @@ -19,30 +19,49 @@ class Crawlers extends AbstractProvider * @var array */ protected $data = array( - '.*Java.*outbrain', ' YLT', + '^Aether', + '^Amazon Simple Notification Service Agent$', + '^Amazon-Route53-Health-Check-Service', '^b0t$', '^bluefish ', '^Calypso v\/', '^COMODO DCV', + '^Corax', '^DangDang', '^DavClnt', + '^DHSH', + '^docker\/[0-9]', + '^Expanse', '^FDM ', '^git\/', '^Goose\/', '^Grabber', + '^Gradle\/', '^HTTPClient\/', + '^HTTPing', '^Java\/', '^Jeode\/', '^Jetty\/', '^Mail\/', '^Mget', '^Microsoft URL Control', + '^Mikrotik\/', + '^Netlab360', '^NG\/[0-9\.]', '^NING\/', - '^PHP\/[0-9]', + '^npm\/', + '^Nuclei', + '^PHP-AYMAPI\/', + '^PHP\/', + '^pip\/', + '^pnpm\/', '^RMA\/', '^Ruby|Ruby\/[0-9]', + '^Swurl ', + '^TLS tester ', + '^twine\/', + '^ureq', '^VSE\/[0-9]', '^WordPress\.com', '^XRL\/[0-9]', @@ -50,17 +69,18 @@ class Crawlers extends AbstractProvider '008\/', '13TABS', '192\.comAgent', + '2GDPR\/', '2ip\.ru', '404enemy', '7Siters', '80legs', - 'a\.pr-cy\.ru', 'a3logics\.in', 'A6-Indexer', 'Abonti', 'Aboundex', 'aboutthedomain', 'Accoona-AI-Agent', + 'acebookexternalhit\/', 'acoon', 'acrylicapps\.com\/pulp', 'Acunetix', @@ -70,8 +90,14 @@ class Crawlers extends AbstractProvider 'ADmantX', 'AdminLabs', 'adressendeutschland', + 'adreview\/', 'adscanner', + 'adstxt-worker', 'Adstxtaggregator', + 'adstxt\.com', + 'Adyen HttpClient', + 'AffiliateLabz\/', + 'affilimate-puppeteer', 'agentslug', 'AHC', 'aihit', @@ -92,20 +118,23 @@ class Crawlers extends AbstractProvider 'Anemone', 'AngleSharp', 'annotate_google', - 'Ant\.com', + 'Anthill', 'Anturis Agent', + 'Ant\.com', 'AnyEvent-HTTP\/', + 'Apache Ant\/', 'Apache Droid', 'Apache OpenOffice', 'Apache-HttpAsyncClient', 'Apache-HttpClient', 'ApacheBench', 'Apexoo', + 'apimon\.de', 'APIs-Google', 'AportWorm\/', 'AppBeat\/', 'AppEngine-Google', - 'AppStoreScraperZ', + 'AppleSyndication', 'Aprc\/[0-9]', 'Arachmo', 'arachnode', @@ -113,21 +142,28 @@ class Crawlers extends AbstractProvider 'aria2', 'Arukereso', 'asafaweb', - 'AskQuickly', + 'Asana\/', 'Ask Jeeves', + 'AskQuickly', 'ASPSeek', 'Asterias', 'Astute', 'asynchttp', 'Attach', + 'attohttpc', 'autocite', + 'AutomaticWPTester', 'Autonomy', + 'awin\.com', + 'AWS Security Scanner', 'axios\/', + 'a\.pr-cy\.ru', 'B-l-i-t-z-B-O-T', 'Backlink-Ceck', 'backlink-check', 'BacklinkHttpStatus', 'BackStreet', + 'BackupLand', 'BackWeb', 'Bad-Neighborhood', 'Badass', @@ -142,6 +178,7 @@ class Crawlers extends AbstractProvider 'BCKLINKS', 'BDFetch', 'BegunAdvertising', + 'Bewica-security-scan', 'Bidtellect', 'BigBozz', 'Bigfoot', @@ -151,11 +188,14 @@ class Crawlers extends AbstractProvider 'binlar', 'biNu image cacher', 'Bitacle', + 'Bitrix link preview', 'biz_Directory', + 'BKCTwitterUnshortener\/', 'Black Hole', 'Blackboard Safeassign', 'BlackWidow', 'BlockNote\.Net', + 'BlogBridge', 'Bloglines', 'Bloglovin', 'BlogPulseLive', @@ -163,6 +203,7 @@ class Crawlers extends AbstractProvider 'Blogtrottr', 'BlowFish', 'boitho\.com-dc', + 'Boost\.Beast', 'BPImageWalker', 'Braintree-Webhooks', 'Branch Metrics API', @@ -190,12 +231,15 @@ class Crawlers extends AbstractProvider 'CC Metadata Scaper', 'Cegbfeieh', 'censys', + 'centuryb.o.t9[at]gmail.com', 'Cerberian Drtrs', 'CERT\.at-Statistics-Survey', + 'cf-facebook', 'cg-eye', 'changedetection', 'ChangesMeter', 'Charlotte', + 'chatterino-api-cache', 'CheckHost', 'checkprivacy', 'CherryPicker', @@ -206,6 +250,7 @@ class Crawlers extends AbstractProvider 'Chromaxa', 'CirrusExplorer', 'CISPA Vulnerability Notification', + 'CISPA Web Analyser', 'Citoid', 'CJNetworkQuality', 'Clarsentia', @@ -213,27 +258,34 @@ class Crawlers extends AbstractProvider 'Cloud mapping', 'CloudEndure', 'CloudFlare-AlwaysOnline', + 'Cloudflare-Healthchecks', 'Cloudinary', 'cmcm\.com', 'coccoc', 'cognitiveseo', + 'ColdFusion', 'colly -', 'CommaFeed', 'Commons-HttpClient', 'commonscan', 'contactbigdatafr', 'contentkingapp', + 'Contextual Code Sites Explorer', 'convera', 'CookieReports', 'copyright sheriff', 'CopyRightCheck', 'Copyscape', + 'cortex\/', 'Cosmos4j\.feedback', 'Covario-IDS', + 'Craw\/', 'Crescent', - 'Crowsnest', 'Criteo', + 'Crowsnest', 'CSHttp', + 'CSSCheck', + 'Cula\/', 'curb', 'Curious George', 'curl', @@ -243,12 +295,17 @@ class Crawlers extends AbstractProvider 'DareBoost', 'DatabaseDriverMysqli', 'DataCha0s', + 'DatadogSynthetics', 'Datafeedwatch', 'Datanyze', 'DataparkSearch', 'dataprovider', 'DataXu', 'Daum(oa)?[ \/][0-9]', + 'dBpoweramp', + 'ddline', + 'deeris', + 'delve\.ai', 'Demon', 'DeuSu', 'developers\.google\.com\/\+\/web\/snippet\/', @@ -258,8 +315,8 @@ class Crawlers extends AbstractProvider 'DigitalPebble', 'Dirbuster', 'Discourse Forum Onebox', - 'Disqus\/', 'Dispatch\/', + 'Disqus\/', 'DittoSpyder', 'dlvr', 'DMBrowser', @@ -267,6 +324,8 @@ class Crawlers extends AbstractProvider 'docoloc', 'Dolphin http client', 'DomainAppender', + 'DomainLabz', + 'Domains Project\/', 'Donuts Content Explorer', 'dotMailer content retrieval', 'dotSemantic', @@ -279,6 +338,8 @@ class Crawlers extends AbstractProvider 'Drupal \(\+http:\/\/drupal\.org\/\)', 'DTS Agent', 'dubaiindex', + 'DuplexWeb-Google', + 'DynatraceSynthetic', 'EARTHCOM', 'Easy-Thumb', 'EasyDL', @@ -309,20 +370,22 @@ class Crawlers extends AbstractProvider 'ExaleadCloudview', 'Excel\/', 'exif', + 'ExoRank', 'Exploratodo', 'Express WebPictures', 'Extreme Picture Finder', 'EyeNetIE', 'ezooms', 'facebookexternalhit', + 'facebookexternalua', 'facebookplatform', 'fairshare', 'Faraday v', 'fasthttp', 'Faveeo', 'Favicon downloader', - 'faviconkit', 'faviconarchive', + 'faviconkit', 'FavOrg', 'Feed Wrangler', 'Feedable\/', @@ -334,7 +397,9 @@ class Crawlers extends AbstractProvider 'feeder', 'Feedly', 'FeedshowOnline', + 'Feedshow\/', 'Feedspot', + 'FeedViewer\/', 'Feedwind\/', 'FeedZcollector', 'feeltiptop', @@ -342,6 +407,8 @@ class Crawlers extends AbstractProvider 'Fetch\/[0-9]', 'Fever\/[0-9]', 'FHscan', + 'Fiery%20Feeds', + 'Filestack', 'Fimap', 'findlink', 'findthatfile', @@ -350,18 +417,24 @@ class Crawlers extends AbstractProvider 'FlipboardProxy', 'FlipboardRSS', 'Flock\/', + 'Florienzh\/', 'fluffy', 'Flunky', 'flynxapp', 'forensiq', + 'ForusP', 'FoundSeoTool', - 'http:\/\/www.neomo.de\/', //'Francis [Bot]' + 'fragFINN\.de', 'free thumbnails', 'Freeuploader', + 'FreshRSS', + 'frontman', 'Funnelback', + 'Fuzz Faster U Fool', 'G-i-g-a-b-o-t', 'g00g1e\.net', 'ganarvisitas', + 'gdnplus\.com', 'geek-tools', 'Genieo', 'GentleSource', @@ -373,18 +446,24 @@ class Crawlers extends AbstractProvider 'getroot', 'GetURLInfo\/', 'GetWeb', + 'Geziyor', 'Ghost Inspector', 'GigablastOpenSource', 'GIS-LABS', 'github-camo', + 'GitHub-Hookshot', 'github\.com', - 'Go [\d\.]* package http', 'Go http package', + 'Go [\d\.]* package http', + 'Go!Zilla', 'Go-Ahead-Got-It', 'Go-http-client', - 'Go!Zilla', + 'go-mtasts\/', + 'gobuster', 'gobyus', + 'Gofeed', 'gofetch', + 'Goldfire Server', 'GomezAgent', 'gooblog', 'Goodzer\/', @@ -397,24 +476,31 @@ class Crawlers extends AbstractProvider 'Google PP Default', 'Google Search Console', 'Google Web Preview', + 'Google-Ads-Creatives-Assistant', + 'Google-Ads-Overview', 'Google-Adwords', 'Google-Apps-Script', 'Google-Calendar-Importer', 'Google-HotelAdsVerifier', 'Google-HTTP-Java-Client', + 'Google-InspectionTool', + 'Google-Podcast', 'Google-Publisher-Plugin', + 'Google-Read-Aloud', 'Google-SearchByImage', 'Google-Site-Verification', + 'Google-SMTP-STS', + 'Google-speakr', 'Google-Structured-Data-Testing-Tool', - 'Google-Youtube-Links', + 'Google-Transparency-Report', 'google-xrawler', + 'Google-Youtube-Links', 'GoogleDocs', 'GoogleHC\/', + 'GoogleProber', 'GoogleProducer', 'GoogleSites', - 'Google-Transparency-Report', 'Gookey', - 'GoScraper', 'GoSpotCheck', 'gosquared-thumbnailer', 'Gotit', @@ -440,13 +526,16 @@ class Crawlers extends AbstractProvider 'hackney\/', 'Hadi Agent', 'HappyApps-WebCheck', + 'Hardenize', 'Hatena', 'Havij', + 'HaxerMen', 'HeadlessChrome', 'HEADMasterSEO', 'HeartRails_Capture', 'help@dataminr\.com', 'heritrix', + 'Hexometer', 'historious', 'hkedcity', 'hledejLevne\.cz', @@ -463,28 +552,31 @@ class Crawlers extends AbstractProvider 'HTMLparser', 'htmlyse', 'HTTP Banner Detection', - 'HTTP_Compression_Test', - 'http_request2', - 'http_requester', 'http-get', 'HTTP-Header-Abfrage', 'http-kit', 'http-request\/', 'HTTP-Tiny', 'HTTP::Lite', - 'http\.rb\/', - 'http_get', + 'http:\/\/www.neomo.de\/', //'Francis [Bot]' 'HttpComponents', 'httphr', + 'HTTPie', 'HTTPMon', 'httpRequest', 'httpscheck', 'httpssites_power', 'httpunit', 'HttpUrlConnection', + 'http\.rb\/', + 'HTTP_Compression_Test', + 'http_get', + 'http_request2', + 'http_requester', 'httrack', 'huaweisymantec', 'HubSpot ', + 'HubSpot-Link-Resolver', 'Humanlinks', 'i2kconnect\/', 'Iblog', @@ -495,6 +587,7 @@ class Crawlers extends AbstractProvider 'IDwhois\/', 'Iframely', 'igdeSpyder', + 'iGooglePortal', 'IlTrovatore', 'Image Fetch', 'Image Sucker', @@ -519,13 +612,15 @@ class Crawlers extends AbstractProvider 'integromedb', 'Intelliseek', 'InterGET', - 'internet_archive', 'Internet Ninja', 'InternetSeer', 'internetVista monitor', + 'internetwache', + 'internet_archive', 'intraVnews', 'IODC', 'IOI', + 'Inboxb0t', 'iplabel', 'ips-agent', 'IPS\/[0-9]', @@ -537,10 +632,13 @@ class Crawlers extends AbstractProvider 'iskanie', 'isUp\.li', 'iThemes Sync\/', + 'IZaBEE', 'iZSearch', 'JAHHO', 'janforman', 'Jaunt\/', + 'Java.*outbrain', + 'javelin\.io', 'Jbrofuzz', 'Jersey\/', 'JetCar', @@ -556,6 +654,7 @@ class Crawlers extends AbstractProvider 'Joomla', 'Jorgee', 'JS-Kit', + 'JungleKeyThumbnail', 'JustView', 'Kaspersky Lab CFR link resolver', 'Kelny\/', @@ -563,6 +662,7 @@ class Crawlers extends AbstractProvider 'KeyCDN', 'Keyword Density', 'Keywords Research', + 'khttp\/', 'KickFire', 'KimonoLabs\/', 'Kml-Google', @@ -570,11 +670,12 @@ class Crawlers extends AbstractProvider 'KOCMOHABT', 'kouio', 'kube-probe', + 'kubectl', 'kulturarw3', 'KumKie', - 'L\.webis', 'Larbin', 'Lavf\/', + 'leakix\.net', 'LeechFTP', 'LeechGet', 'letsencrypt', @@ -585,41 +686,50 @@ class Crawlers extends AbstractProvider 'libwww', 'Licorne', 'Liferea\/', - 'Lightspeedsystems', 'Lighthouse', + 'Lightspeedsystems', 'Likse', + 'limber\.io', 'Link Valet', - 'link_thumbnailer', 'LinkAlarm\/', + 'LinkAnalyser', 'linkCheck', 'linkdex', 'LinkExaminer', 'linkfluence', 'linkpeek', - 'LinkPreviewGenerator', + 'LinkPreview', 'LinkScan', 'LinksManager', 'LinkTiger', 'LinkWalker', + 'link_thumbnailer', 'Lipperhey', 'Litemage_walker', 'livedoor ScreenShot', 'LoadImpactRload', 'localsearch-web', 'LongURL API', + 'longurl-r-package', + 'looid\.com', 'looksystems\.net', 'ltx71', 'lua-resty-http', + 'Lucee \(CFML Engine\)', + 'Lush Http Client', 'lwp-request', 'lwp-trivial', 'LWP::Simple', 'lycos', 'LYT\.SR', + 'L\.webis', 'mabontland', + 'MacOutlook\/', 'Mag-Net', 'MagpieRSS', - 'Mail\.Ru', + 'Mail::STS', 'MailChimp', + 'Mail\.Ru', 'Majestic12', 'makecontact\/', 'Mandrill', @@ -630,6 +740,8 @@ class Crawlers extends AbstractProvider 'Mass Downloader', 'masscan\/', 'Mata Hari', + 'mattermost', + 'Mediametric', 'Mediapartners-Google', 'mediawords', 'MegaIndex\.ru', @@ -640,28 +752,31 @@ class Crawlers extends AbstractProvider 'MetaURI', 'MFC_Tear_Sample', 'Microsearch', - 'Microsoft Office ', + 'Microsoft Data Access', + 'Microsoft Office', 'Microsoft Outlook', 'Microsoft Windows Network Diagnostics', 'Microsoft-WebDAV-MiniRedir', - 'Microsoft Data Access', + 'Microsoft\.Data\.Mashup', 'MIDown tool', 'MIIxpc', 'Mindjet', 'Miniature\.io', 'Miniflux', + 'mio_httpc', + 'Miro-HttpClient', 'Mister PiX', 'mixdata dot com', 'mixed-content-scan', - 'Mixmax-LinkPreview', 'mixnode', 'Mnogosearch', 'mogimogi', 'Mojeek', 'Mojolicious \(Perl\)', - 'Monit\/', + 'Mollie', 'monitis', 'Monitority\/', + 'Monit\/', 'montastic', 'MonTools', 'Moreover', @@ -670,6 +785,7 @@ class Crawlers extends AbstractProvider 'MovableType', 'mowser', 'Mrcgiguy', + 'Mr\.4x3 Powered', 'MS Web Services Client Protocol', 'MSFrontPage', 'mShots', @@ -677,10 +793,13 @@ class Crawlers extends AbstractProvider 'muhstik-scan', 'MVAClient', 'MxToolbox\/', + 'myseosnapshot', 'nagios', 'Najdi\.si', 'Name Intelligence', + 'NameFo\.com', 'Nameprotect', + 'nationalarchives', 'Navroad', 'NearSite', 'Needle', @@ -695,6 +814,8 @@ class Crawlers extends AbstractProvider 'netresearch', 'NetShelter ContentScan', 'Netsparker', + 'NetSystemsResearch', + 'nettle', 'NetTrack', 'Netvibes', 'NetZIP', @@ -707,22 +828,26 @@ class Crawlers extends AbstractProvider 'newspaper\/', 'Nexgate Ruby Client', 'NG-Search', + 'nghttp2', 'Nibbler', 'NICErsPRO', + 'NihilScio', 'Nikto', 'nineconnections', 'NLNZ_IAHarvester', 'Nmap Scripting Engine', + 'node-fetch', 'node-superagent', 'node-urllib', - 'node\.io', 'Nodemeter', 'NodePing', + 'node\.io', 'nominet\.org\.uk', 'nominet\.uk', 'Norton-Safeweb', 'Notifixious', 'notifyninja', + 'NotionEmbedder', 'nuhk', 'nutch', 'Nuzzel', @@ -730,39 +855,45 @@ class Crawlers extends AbstractProvider 'nyawc\/', 'Nymesis', 'NYU', + 'Observatory\/', 'Ocelli\/', 'Octopus', 'oegp', 'Offline Explorer', 'Offline Navigator', - 'og-scraper', + 'OgScrper', 'okhttp', 'omgili', 'OMSC', 'Online Domain Tools', + 'Open Source RSS', 'OpenCalaisSemanticProxy', 'Openfind', 'OpenLinkProfiler', 'Openstat\/', 'OpenVAS', + 'OPPO A33', 'Optimizer', 'Orbiter', 'OrgProbe\/', 'orion-semantics', 'Outlook-Express', 'Outlook-iOS', - 'ow\.ly', 'Owler', + 'Owlin', 'ownCloud News', + 'ow\.ly', 'OxfordCloudService', - 'Page Valet', - 'page_verifier', 'page scorer', + 'Page Valet', 'page2rss', + 'PageFreezer', 'PageGrabber', 'PagePeeker', 'PageScorer', 'Pagespeed\/', + 'PageThing', + 'page_verifier', 'Panopta', 'panscient', 'Papa Foto', @@ -771,6 +902,7 @@ class Crawlers extends AbstractProvider 'PayPal IPN', 'pcBrowser', 'Pcore-HTTP', + 'PDF24 URL To PDF', 'Pearltrees', 'PECL::HTTP', 'peerindex', @@ -780,44 +912,52 @@ class Crawlers extends AbstractProvider 'PhantomJS Screenshoter', 'PhantomJS\/', 'Photon\/', + 'php-requests', 'phpservermon', 'Pi-Monster', 'Picscout', 'Picsearch', 'PictureFinder', 'Pimonster', - 'ping\.blo\.gs', 'Pingability', 'PingAdmin\.Ru', 'Pingdom', 'Pingoscope', 'PingSpot', + 'ping\.blo\.gs', 'pinterest\.com', 'Pixray', 'Pizilla', 'Plagger\/', + 'Pleroma ', 'Ploetz \+ Zeller', 'Plukkie', 'plumanalytics', 'PocketImageCache', 'PocketParser', 'Pockey', + 'PodcastAddict\/', 'POE-Component-Client-HTTP', 'Polymail\/', 'Pompos', 'Porkbun', 'Port Monitor', 'postano', + 'postfix-mta-sts-resolver', 'PostmanRuntime', + 'postplanner\.com', 'PostPost', 'postrank', 'PowerPoint\/', + 'Prebid', + 'Prerender', 'Priceonomics Analysis Engine', 'PrintFriendly', 'PritTorrent', 'Prlog', 'probethenet', - 'Project 25499', + 'Project ?25499', + 'Project-Resonance', 'prospectb2b', 'Protopage', 'ProWebWalker', @@ -826,9 +966,9 @@ class Crawlers extends AbstractProvider 'pshtt, https scanning', 'PTST ', 'PTST\/[0-9]+', - 'Pulsepoint XT3 web scraper', 'Pump', 'Python-httplib2', + 'python-httpx', 'python-requests', 'Python-urllib', 'Qirina Hurdler', @@ -838,12 +978,17 @@ class Crawlers extends AbstractProvider 'Qualidator', 'QueryN Metasearch', 'queuedriver', + 'quic-go-HTTP\/', + 'QuiteRSS', 'Quora Link Preview', 'Qwantify', 'Radian6', + 'RadioPublicImageResizer', + 'Railgun\/', 'RankActive', 'RankFlex', 'RankSonicSiteAuditor', + 'RapidLoad\/', 'Re-re Studio', 'ReactorNetty', 'Readability', @@ -854,6 +999,7 @@ class Crawlers extends AbstractProvider 'RecurPost\/', 'redback\/', 'ReederForMac', + 'Reeder\/', 'ReGet', 'RepoMonkey', 'request\.js', @@ -866,38 +1012,44 @@ class Crawlers extends AbstractProvider 'Robozilla', 'ROI Hunter', 'RPT-HTTPClient', + 'RSSMix\/', 'RSSOwl', + 'RyowlEngine', 'safe-agent-scanner', 'SalesIntelligent', 'Saleslift', - 'Sendsay\.Ru', + 'SAP NetWeaver Application Server', 'SauceNAO', 'SBIder', + 'sc-downloader', 'scalaj-http', - 'scan\.lol', + 'Scamadviser-Frontend', 'ScanAlert', + 'scan\.lol', 'Scoop', 'scooter', + 'ScopeContentAG-HTTP-Client', 'ScoutJet', 'ScoutURLMonitor', 'ScrapeBox Page Scanner', - 'SimpleScraper', 'Scrapy', 'Screaming', 'ScreenShotService', 'Scrubby', 'Scrutiny\/', - 'search\.thunderstone', 'Search37', 'searchenginepromotionhelp', 'Searchestate', 'SearchExpress', 'SearchSight', + 'SearchWP', + 'search\.thunderstone', 'Seeker', 'semanticdiscovery', 'semanticjuice', 'Semiocast HTTP client', 'Semrush', + 'Sendsay\.Ru', 'sentry\/', 'SEO Browser', 'Seo Servis', @@ -906,11 +1058,13 @@ class Crawlers extends AbstractProvider 'Seobility', 'SEOCentro', 'SeoCheck', + 'seocompany', 'SEOkicks', + 'SEOlizer', 'Seomoz', 'SEOprofiler', - 'SEOsearch', 'seoscanners', + 'SEOsearch', 'seositecheckup', 'SEOstats', 'servernfo', @@ -921,14 +1075,16 @@ class Crawlers extends AbstractProvider 'Shoppimon', 'ShopWiki', 'ShortLinkTranslate', + 'shortURL lengthener', 'shrinktheweb', 'Sideqik', + 'Siege', 'SimplePie', 'SimplyFast', 'Siphon', 'SISTRIX', - 'Site-Shot\/', 'Site Sucker', + 'Site-Shot\/', 'Site24x7', 'SiteBar', 'Sitebeam', @@ -949,6 +1105,7 @@ class Crawlers extends AbstractProvider 'sitexy\.com', 'SkypeUriPreview', 'Slack\/', + 'sli-systems\.com', 'slider\.com', 'slurp', 'SlySearch', @@ -971,9 +1128,11 @@ class Crawlers extends AbstractProvider 'SpamExperts', 'Spammen', 'Spanner', + 'Spawning-AI', 'spaziodati', 'SPDYCheck', 'Specificfeeds', + 'SpeedKit', 'speedy', 'SPEng', 'Spinn3r', @@ -987,9 +1146,11 @@ class Crawlers extends AbstractProvider 'ssl-tools', 'StackRambler', 'Statastico\/', + 'Statically-', 'StatusCake', 'Steeler', 'Stratagems Kumo', + 'Stripe\/', 'Stroke\.cz', 'StudioFACA', 'StumbleUpon', @@ -999,9 +1160,11 @@ class Crawlers extends AbstractProvider 'SuperHTTP', 'Surphace Scout', 'Suzuran', - 'SwiteScraper', + 'swcd ', 'Symfony BrowserKit', 'Symfony2 BrowserKit', + 'Synapse\/', + 'Syndirella\/', 'SynHttpClient-Built', 'Sysomos', 'sysscan', @@ -1011,6 +1174,8 @@ class Crawlers extends AbstractProvider 'Tarantula\/', 'Taringa UGC', 'TarmotGezgin', + 'tchelebi\.io', + 'techiaith\.cymru', 'Teleport', 'Telesoft', 'Telesphoreo', @@ -1021,13 +1186,15 @@ class Crawlers extends AbstractProvider 'Test Certificate Info', 'testuri', 'Tetrahedron', + 'TextRazor Downloader', 'The Drop Reaper', 'The Expert HTML Source Viewer', - 'The Knowledge AI', 'The Intraformant', + 'The Knowledge AI', 'theinternetrules', 'TheNomad', 'Thinklab', + 'Thumbor', 'Thumbshots', 'ThumbSniper', 'timewe\.net', @@ -1061,9 +1228,11 @@ class Crawlers extends AbstractProvider 'ubermetrics-technologies', 'uclassify', 'UdmSearch', + 'ultimate_sitemap_parser', 'unchaos', 'unirest-java', 'UniversalFeedParser', + 'unshortenit', 'Unshorten\.It', 'Untiny', 'UnwindFetchor', @@ -1071,8 +1240,8 @@ class Crawlers extends AbstractProvider 'updown\.io daemon', 'Upflow', 'Uptimia', - 'Urlcheckr', 'URL Verifier', + 'Urlcheckr', 'URLitor', 'urlresolver', 'Urlstat', @@ -1080,15 +1249,16 @@ class Crawlers extends AbstractProvider 'UrlTrends Ranking Updater', 'URLy Warning', 'URLy\.Warning', + 'URL\/Emacs', 'Vacuum', 'Vagabondo', 'VB Project', 'vBSEO', 'VCI', 'via ggpht\.com GoogleImageProxy', - 'VidibleScraper', 'Virusdie', 'visionutils', + 'Visual Rights Group', 'vkShare', 'VoidEYE', 'Voil', @@ -1099,19 +1269,18 @@ class Crawlers extends AbstractProvider 'Vulnbusters Meter', 'VYU2', 'w3af\.org', - 'W3C_Unicorn', 'W3C-checklink', 'W3C-mobileOK', + 'W3C_Unicorn', 'WAC-OFU', - 'Wallpapers\/[0-9]+', + 'WakeletLinkExpander', 'WallpapersHD', + 'Wallpapers\/[0-9]+', 'wangling', 'Wappalyzer', 'WatchMouse', 'WbSrch\/', 'WDT\.io', - 'web-capture\.net', - 'Web-sniffer', 'Web Auto', 'Web Collage', 'Web Enhancer', @@ -1119,7 +1288,10 @@ class Crawlers extends AbstractProvider 'Web Fuck', 'Web Pix', 'Web Sauger', + 'Web spyder', 'Web Sucker', + 'web-capture\.net', + 'Web-sniffer', 'Webalta', 'Webauskunft', 'WebAuto', @@ -1157,8 +1329,9 @@ class Crawlers extends AbstractProvider 'Webster', 'WebStripper', 'WebSucker', - 'Webthumb\/', + 'webtech\/', 'WebThumbnail', + 'Webthumb\/', 'WebWhacker', 'WebZIP', 'WeLikeLinks', @@ -1167,24 +1340,31 @@ class Crawlers extends AbstractProvider 'wf84', 'Wfuzz\/', 'wget', + 'WhatCMS', 'WhatsApp', 'WhatsMyIP', 'WhatWeb', 'WhereGoes\?', 'Whibse', + 'WhoAPI\/', 'WhoRunsCoinHive', 'Whynder Magnet', 'Windows-RSS-Platform', + 'WinHttp-Autoproxy-Service', + 'WinHTTP\/', 'WinPodder', 'wkhtmlto', 'wmtips', 'Woko', + 'Wolfram HTTPClient', 'woorankreview', - 'Word\/', 'WordPress\/', 'WordupinfoSearch', + 'Word\/', + 'worldping-api', 'wotbox', 'WP Engine Install Performance API', + 'WP Rocket', 'wpif', 'wprecon\.com survey', 'WPScan', @@ -1194,20 +1374,21 @@ class Crawlers extends AbstractProvider 'WWW-Mechanize', 'WWW::Document', 'WWW::Mechanize', - 'www\.monitor\.us', 'WWWOFFLE', + 'www\.monitor\.us', 'x09Mozilla', 'x22Mozilla', 'XaxisSemanticsClassifier', + 'XenForo\/', 'Xenu Link Sleuth', 'XING-contenttabreceiver', 'xpymep([0-9]?)\.exe', - 'Y!J-(ASR|BSC)', - 'Y\!J-BRW', + 'Y!J-[A-Z][A-Z][A-Z]', 'Yaanb', 'yacy', 'Yahoo Link Preview', 'YahooCacheSystem', + 'YahooMailProxy', 'YahooYSMcm', 'YandeG', 'Yandex(?!Search)', @@ -1215,6 +1396,7 @@ class Crawlers extends AbstractProvider 'yeti', 'Yo-yo', 'Yoleo Consumer', + 'yomins\.com', 'yoogliFetchAgent', 'YottaaMonitor', 'Your-Website-Sucks', @@ -1226,15 +1408,17 @@ class Crawlers extends AbstractProvider 'Zao', 'Zauba', 'Zemanta Aggregator', - 'Zend_Http_Client', 'Zend\\\\Http\\\\Client', + 'Zend_Http_Client', 'Zermelo', 'Zeus ', 'zgrab', 'ZnajdzFoto', + 'ZnHTTP', 'Zombie\.js', 'Zoom\.Mac', + 'ZoteroTranslationServer', 'ZyBorg', - '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer)', + '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)', ); } diff --git a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php index e6b3ca89..62745572 100644 --- a/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php +++ b/blockbot/vendor/jaybizzle/crawler-detect/src/Fixtures/Exclusions.php @@ -67,6 +67,8 @@ class Exclusions extends AbstractProvider '; IDbot', '; ID bot', '; POWER BOT', - ';', // Remove the following characters ; + 'OCTOPUS-CORE', + 'htc_botdugls', + 'super\/\d+\/Android\/\d+', ); }