Merge branch 'master' into master

This commit is contained in:
Andy H 2018-11-13 20:18:03 +07:00 committed by GitHub
commit c22ce2292d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 102 additions and 11 deletions

View file

@ -65,7 +65,6 @@ DocumentRoot /path/to/friendica-directory/public/
### Nginx ### Nginx
Include this line your nginx config file. Include this line your nginx config file.
<<<<<<< master
``` ```
root /path/to/friendica-directory/public; root /path/to/friendica-directory/public;
``` ```

View file

@ -21,6 +21,10 @@ This is an opt-in directory, meaning that each node can choose not to submit its
Please refer to the provided [installation instructions](INSTALL.md). Please refer to the provided [installation instructions](INSTALL.md).
## Update from a previous version
Please refer to the provided [update instructions](UPDATE.md).
## See also ## See also
- [Project Concepts](docs/Concepts.md) - [Project Concepts](docs/Concepts.md)

42
UPDATE.md Normal file
View file

@ -0,0 +1,42 @@
# Friendica Directory Update Instructions
## 1. Update the source code
If you installed Friendica Directory in `/path/to/friendica-directory`.
### Using Git
```
cd /path/to/friendica-directory
git pull
composer install
```
### Using an archive
1. Create a temporary folder to unpack the new archive.
2. Copy your old `config/local.json` to the new folder.
3. Swap the folder names.
4. Remove the temporary folder.
Sample Linux commands:
```
cd /path/to
mkdir friendica-directory-new
unzip friendica-<version>.zip friendica-directory-new
cp friendica-directory/config/local.json friendica-directory-new/config
mv friendica-directory friendica-directory-old
mv friendica-directory-new friendica-directory
rm -r friendica-directory-old
```
## 2. Update the database structure
The database structure may have changed since the last update, fortunately a console command allows to run the migration scripts up to the latest version:
```
cd /path/to/friendica-directory
bin/console dbupdate
```
You're all set!

View file

@ -1 +1 @@
2.0.3 2.0.5

View file

@ -43,9 +43,7 @@ class Console extends \Asika\SimpleConsole\Console
{ {
$commandList = ''; $commandList = '';
foreach ($this->routes as $command => $class) { foreach ($this->routes as $command => $class) {
$this->out($class); $commandList .= ' ' . $command . "\n";
$commandList .= ' ' . $command . ' ' . $class::description . "\n";
} }
$help = <<<HELP $help = <<<HELP

View file

@ -2,6 +2,8 @@
namespace Friendica\Directory\Models; namespace Friendica\Directory\Models;
use Friendica\Directory\Utils\Network;
/** /**
* @author Hypolite Petovan <mrpetovan@gmail.com> * @author Hypolite Petovan <mrpetovan@gmail.com>
*/ */
@ -15,6 +17,15 @@ class ProfilePollQueue extends \Friendica\Directory\Model
return false; return false;
} }
$host = parse_url($url, PHP_URL_HOST);
if (!$host) {
return false;
}
if (Network::isPublicHost($host)) {
return false;
}
$this->atlas->perform( $this->atlas->perform(
'INSERT IGNORE INTO `profile_poll_queue` SET `profile_url` = :profile_url', 'INSERT IGNORE INTO `profile_poll_queue` SET `profile_url` = :profile_url',
['profile_url' => $url] ['profile_url' => $url]

View file

@ -2,6 +2,8 @@
namespace Friendica\Directory\Pollers; namespace Friendica\Directory\Pollers;
use Friendica\Directory\Utils\Network;
/** /**
* @author Hypolite Petovan <mrpetovan@gmail.com> * @author Hypolite Petovan <mrpetovan@gmail.com>
*/ */
@ -127,7 +129,12 @@ class Profile
$noscrape = !!$params; //If the result was false, do a scrape after all. $noscrape = !!$params; //If the result was false, do a scrape after all.
} }
if (!$noscrape) { $available = true;
if ($noscrape) {
$available = Network::testURL($profile_uri);
$this->logger->debug('Testing ' . $profile_uri . ': ' . ($available?'Success':'Failure'));
} else {
$this->logger->notice('Parsing profile page ' . $profile_uri); $this->logger->notice('Parsing profile page ' . $profile_uri);
$params = \Friendica\Directory\Utils\Scrape::retrieveProfileData($profile_uri); $params = \Friendica\Directory\Utils\Scrape::retrieveProfileData($profile_uri);
} }
@ -204,7 +211,7 @@ class Profile
`account_type` = :account_type, `account_type` = :account_type,
`filled_fields` = :filled_fields, `filled_fields` = :filled_fields,
`last_activity` = :last_activity, `last_activity` = :last_activity,
`available` = 1, `available` = :available,
`created` = NOW(), `created` = NOW(),
`updated` = NOW() `updated` = NOW()
ON DUPLICATE KEY UPDATE ON DUPLICATE KEY UPDATE
@ -223,7 +230,7 @@ class Profile
`account_type` = :account_type, `account_type` = :account_type,
`filled_fields` = :filled_fields, `filled_fields` = :filled_fields,
`last_activity` = :last_activity, `last_activity` = :last_activity,
`available` = 1, `available` = :available,
`updated` = NOW()', `updated` = NOW()',
[ [
'profile_id' => $profile_id, 'profile_id' => $profile_id,
@ -242,6 +249,7 @@ class Profile
'account_type' => $account_type, 'account_type' => $account_type,
'filled_fields' => $filled_fields, 'filled_fields' => $filled_fields,
'last_activity' => $params['last-activity'] ?? null, 'last_activity' => $params['last-activity'] ?? null,
'available' => $available,
] ]
); );

View file

@ -173,8 +173,10 @@ class Server
); );
//Add the admin to the directory //Add the admin to the directory
if (!empty($probe_result['data']['admin']['profile'])) {
$this->profilePollQueueModel->add($probe_result['data']['admin']['profile']); $this->profilePollQueueModel->add($probe_result['data']['admin']['profile']);
} }
}
if ($server) { if ($server) {
//Get the new health. //Get the new health.

View file

@ -23,7 +23,7 @@ class Network
} }
curl_setopt($ch, CURLOPT_HEADER, 0); curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_TIMEOUT, max(intval($timeout), 1)); //Minimum of 1 second timeout. curl_setopt($ch, CURLOPT_TIMEOUT, max($timeout, 1)); //Minimum of 1 second timeout.
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_MAXREDIRS, 8); curl_setopt($ch, CURLOPT_MAXREDIRS, 8);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@ -36,6 +36,31 @@ class Network
return $s; return $s;
} }
public static function testURL(string $url, int $timeout = 20): bool
{
$ch = curl_init($url);
if (!$ch) {
return false;
}
curl_setopt($ch, CURLOPT_HEADER , 0);
curl_setopt($ch, CURLOPT_TIMEOUT , max($timeout, 1)); //Minimum of 1 second timeout.
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_MAXREDIRS , 8);
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($ch, CURLOPT_NOBODY , true);
curl_exec($ch);
$responseCode = intval(curl_getinfo($ch, CURLINFO_RESPONSE_CODE));
$testSuccess = curl_errno($ch) === 0 && $responseCode < 400;
curl_close($ch);
return $testSuccess;
}
/** /**
* Check if a hostname is public and non-reserved * Check if a hostname is public and non-reserved
* *

View file

@ -56,7 +56,7 @@ class Scrape
$scrape_start = microtime(true); $scrape_start = microtime(true);
$params = []; $params = [];
$html = Network::fetchURL($url, $timeout); $html = Network::fetchURL($url, false, $timeout);
$scrape_fetch_end = microtime(true); $scrape_fetch_end = microtime(true);

View file

View file

@ -0,0 +1,2 @@
ALTER table `profile` DROP INDEX `profile-ft`;
ALTER table `profile` ADD FULLTEXT KEY `profile-ft` (`name`, `pdesc`, `profile_url`, `locality`, `region`, `country`, `tags`);