From e03e10e0326887fb3b943ec1569720205c36d13a Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 14:15:26 -0500 Subject: [PATCH 01/10] Update install/update documentation --- INSTALL.md | 40 +++++++++++++++++----------------------- README.md | 4 ++++ UPDATE.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 23 deletions(-) create mode 100644 UPDATE.md diff --git a/INSTALL.md b/INSTALL.md index 97a3afb..b5e1a19 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,36 +1,30 @@ -# Friendica Directory Install +# Friendica Directory Install Instructions -## Get the source code +## 1. Get the source code -If you want to install Friendica Directory in `/path/to/friendica-directory` +For a planned install of Friendica Directory in the `/path/to/friendica-directory` folder. -### Using Composer +### Using Git and Composer -Composer is a popular dependency management tool for PHP projects. [Getting Started with Composer](https://getcomposer.org/doc/00-intro.md). - -``` -cd /path/to -composer create-project friendica/friendica-directory --no-dev -``` - -### Using Git - -Git is a popular version control management tool. [Getting Started with Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git). +Git is a popular version control management tool. +[Getting Started with Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git). You'll also need Composer to grab the project dependencies. +Composer is a popular dependency management tool for PHP projects. +[Getting Started with Composer](https://getcomposer.org/doc/00-intro.md). ``` cd /path/to -git clone https://github.com/friendica/friendica-directory +git clone https://github.com/friendica/friendica-directory friendica-directory cd friendica-directory composer install ``` -### Using an archive +### Using a stable release archive On the [Friendica Directory Github Releases page](https://github.com/friendica/friendica-directory/releases), you can find the latest source archive named `friendica-directory-.zip`. -Simply unpack the archive in `/path/to/friendica-directory`. +Simply unpack the archive in `/path/to/friendica-directory`, the dependencies are already included. ## Set up a database and a user @@ -45,7 +39,7 @@ Sample commands from the MariaDB console for a local install: > GRANT ALL ON `friendica-directory`.* TO 'friendica-directory'@'localhost' IDENTIFIED BY "password"; ``` -## Initialize database schema +## 2. Initialize database schema Using the details gathered from the previous step, follow the instructions in the Friendica Directory Install Wizard. @@ -54,7 +48,7 @@ cd /path/to/friendica-directory bin/console install ``` -## Configure your web server +## 3. Configure your web server The document root of Friendica Directory is `/public`. @@ -68,7 +62,7 @@ In your Virtual Host file, set your document root as follow: DocumentRoot /path/to/friendica-directory/public/ ``` -## Set up the background task +## 4. Set up the background task Friendica Directory relies on a background task running every minute to keep the directory up to date. @@ -79,8 +73,8 @@ Add this line to your crontab: * * * * * cd /path/to/friendica-directory && php bin/cron.php ``` -## Seed your directory +## 5. Seed your directory Your directory is ready, but empty. To start filling it, you can: -- Set your hostname as the main directory in [Friendica](https://github.com/friendica/friendica)'s admin settings -- Add existing directories in your polling queue: `bin/console directory-add https://dir.friendica.social` +- Set your host name as the main directory in [Friendica](https://github.com/friendica/friendica)'s admin settings. +- Add existing directories in your polling queue: `bin/console directory-add https://dir.friendica.social`. diff --git a/README.md b/README.md index 04b68a7..f03db63 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,10 @@ This is an opt-in directory, meaning that each node can choose not to submit its Please refer to the provided [installation instructions](INSTALL.md). +## Update from a previous version + +Please refer to the provided [update instructions](UPDATE.md). + ## See also - [Project Concepts](docs/Concepts.md) diff --git a/UPDATE.md b/UPDATE.md new file mode 100644 index 0000000..a8353df --- /dev/null +++ b/UPDATE.md @@ -0,0 +1,42 @@ +# Friendica Directory Update Instructions + +## 1. Update the source code + +If you installed Friendica Directory in `/path/to/friendica-directory`. + +### Using Git + +``` +cd /path/to/friendica-directory +git pull +composer install +``` + +### Using an archive + +1. Create a temporary folder to unpack the new archive. +2. Copy your old `config/local.json` to the new folder. +3. Swap the folder names. +4. Remove the temporary folder. + +Sample Linux commands: +``` +cd /path/to +mkdir friendica-directory-new +unzip friendica-.zip friendica-directory-new +cp friendica-directory/config/local.json friendica-directory-new/config +mv friendica-directory friendica-directory-old +mv friendica-directory-new friendica-directory +rm -r friendica-directory-old +``` + +## 2. Update the database structure + +The database structure may have changed since the last update, fortunately a console command allows to run the migration scripts up to the latest version: + +``` +cd /path/to/friendica-directory +bin/console dbupdate +``` + +You're all set! \ No newline at end of file From 66dca727a6cacce6875dd8fd81e749797e667e75 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 14:15:51 -0500 Subject: [PATCH 02/10] Add missing column in profile fulltext index --- src/sql/migrations/down/0001.sql | 0 src/sql/migrations/up/0002.sql | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 src/sql/migrations/down/0001.sql create mode 100644 src/sql/migrations/up/0002.sql diff --git a/src/sql/migrations/down/0001.sql b/src/sql/migrations/down/0001.sql new file mode 100644 index 0000000..e69de29 diff --git a/src/sql/migrations/up/0002.sql b/src/sql/migrations/up/0002.sql new file mode 100644 index 0000000..85dc9ed --- /dev/null +++ b/src/sql/migrations/up/0002.sql @@ -0,0 +1,2 @@ +ALTER table `profile` DROP INDEX `profile-ft`; +ALTER table `profile` ADD FULLTEXT KEY `profile-ft` (`name`, `pdesc`, `profile_url`, `locality`, `region`, `country`, `tags`); From 63886737a667da07215b0f067c84cd9d1b270d5e Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 14:16:00 -0500 Subject: [PATCH 03/10] Release 2.0.4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 50ffc5a..2165f8f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.3 +2.0.4 From a319f9991cd5dc8dc913261301c4d47bd3852740 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 21:48:13 -0500 Subject: [PATCH 04/10] Remove WIP --- src/classes/Controllers/Console.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/classes/Controllers/Console.php b/src/classes/Controllers/Console.php index 480433c..286eed6 100644 --- a/src/classes/Controllers/Console.php +++ b/src/classes/Controllers/Console.php @@ -43,9 +43,7 @@ class Console extends \Asika\SimpleConsole\Console { $commandList = ''; foreach ($this->routes as $command => $class) { - $this->out($class); - - $commandList .= ' ' . $command . ' ' . $class::description . "\n"; + $commandList .= ' ' . $command . "\n"; } $help = << Date: Mon, 12 Nov 2018 21:48:40 -0500 Subject: [PATCH 05/10] Fix empty admin profile in server poller --- src/classes/Pollers/Server.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/classes/Pollers/Server.php b/src/classes/Pollers/Server.php index 6773d31..99d4977 100644 --- a/src/classes/Pollers/Server.php +++ b/src/classes/Pollers/Server.php @@ -173,7 +173,9 @@ class Server ); //Add the admin to the directory - $this->profilePollQueueModel->add($probe_result['data']['admin']['profile']); + if (!empty($probe_result['data']['admin']['profile'])) { + $this->profilePollQueueModel->add($probe_result['data']['admin']['profile']); + } } if ($server) { From 5207196e1a3895f022b66eac79436580675a86d9 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 21:49:06 -0500 Subject: [PATCH 06/10] Fix wrong Network::fetchUrl call in Utils\Scrape --- src/classes/Utils/Scrape.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/classes/Utils/Scrape.php b/src/classes/Utils/Scrape.php index dfbc751..6c8fee5 100644 --- a/src/classes/Utils/Scrape.php +++ b/src/classes/Utils/Scrape.php @@ -56,7 +56,7 @@ class Scrape $scrape_start = microtime(true); $params = []; - $html = Network::fetchURL($url, $timeout); + $html = Network::fetchURL($url, false, $timeout); $scrape_fetch_end = microtime(true); From 93e0153112a53944b8351d76114213abfabac660 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 21:49:49 -0500 Subject: [PATCH 07/10] Add test for public host in profile poll queue add --- src/classes/Models/ProfilePollQueue.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/classes/Models/ProfilePollQueue.php b/src/classes/Models/ProfilePollQueue.php index ad660c7..e3868f0 100644 --- a/src/classes/Models/ProfilePollQueue.php +++ b/src/classes/Models/ProfilePollQueue.php @@ -2,6 +2,8 @@ namespace Friendica\Directory\Models; +use Friendica\Directory\Utils\Network; + /** * @author Hypolite Petovan */ @@ -15,6 +17,15 @@ class ProfilePollQueue extends \Friendica\Directory\Model return false; } + $host = parse_url($url, PHP_URL_HOST); + if (!$host) { + return false; + } + + if (Network::isPublicHost($host)) { + return false; + } + $this->atlas->perform( 'INSERT IGNORE INTO `profile_poll_queue` SET `profile_url` = :profile_url', ['profile_url' => $url] From 6797eb72cbea3d86ed3bce1d41c8b93783996593 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 21:50:31 -0500 Subject: [PATCH 08/10] Add new Network::testURL method - Remove obsolete intval() call in Network::fetchURL --- src/classes/Utils/Network.php | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/classes/Utils/Network.php b/src/classes/Utils/Network.php index c6d6629..67e0084 100644 --- a/src/classes/Utils/Network.php +++ b/src/classes/Utils/Network.php @@ -23,7 +23,7 @@ class Network } curl_setopt($ch, CURLOPT_HEADER, 0); - curl_setopt($ch, CURLOPT_TIMEOUT, max(intval($timeout), 1)); //Minimum of 1 second timeout. + curl_setopt($ch, CURLOPT_TIMEOUT, max($timeout, 1)); //Minimum of 1 second timeout. curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); curl_setopt($ch, CURLOPT_MAXREDIRS, 8); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); @@ -36,6 +36,31 @@ class Network return $s; } + public static function testURL(string $url, int $timeout = 20): bool + { + $ch = curl_init($url); + if (!$ch) { + return false; + } + + curl_setopt($ch, CURLOPT_HEADER , 0); + curl_setopt($ch, CURLOPT_TIMEOUT , max($timeout, 1)); //Minimum of 1 second timeout. + curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + curl_setopt($ch, CURLOPT_MAXREDIRS , 8); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($ch, CURLOPT_NOBODY , true); + + curl_exec($ch); + + $responseCode = intval(curl_getinfo($ch, CURLINFO_RESPONSE_CODE)); + + $testSuccess = curl_errno($ch) === 0 && $responseCode < 400; + + curl_close($ch); + + return $testSuccess; + } + /** * Check if a hostname is public and non-reserved * From 76162496106eb2471d7e233fbee93d875ca119c2 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 21:50:50 -0500 Subject: [PATCH 09/10] Add profile URL test on profile poll --- src/classes/Pollers/Profile.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/classes/Pollers/Profile.php b/src/classes/Pollers/Profile.php index d4d71fd..06ce6f0 100644 --- a/src/classes/Pollers/Profile.php +++ b/src/classes/Pollers/Profile.php @@ -2,6 +2,8 @@ namespace Friendica\Directory\Pollers; +use Friendica\Directory\Utils\Network; + /** * @author Hypolite Petovan */ @@ -127,7 +129,12 @@ class Profile $noscrape = !!$params; //If the result was false, do a scrape after all. } - if (!$noscrape) { + $available = true; + + if ($noscrape) { + $available = Network::testURL($profile_uri); + $this->logger->debug('Testing ' . $profile_uri . ': ' . ($available?'Success':'Failure')); + } else { $this->logger->notice('Parsing profile page ' . $profile_uri); $params = \Friendica\Directory\Utils\Scrape::retrieveProfileData($profile_uri); } @@ -204,7 +211,7 @@ class Profile `account_type` = :account_type, `filled_fields` = :filled_fields, `last_activity` = :last_activity, - `available` = 1, + `available` = :available, `created` = NOW(), `updated` = NOW() ON DUPLICATE KEY UPDATE @@ -223,7 +230,7 @@ class Profile `account_type` = :account_type, `filled_fields` = :filled_fields, `last_activity` = :last_activity, - `available` = 1, + `available` = :available, `updated` = NOW()', [ 'profile_id' => $profile_id, @@ -242,6 +249,7 @@ class Profile 'account_type' => $account_type, 'filled_fields' => $filled_fields, 'last_activity' => $params['last-activity'] ?? null, + 'available' => $available, ] ); From 8d702801a5cd54c1ddc7f47af7de3c8f2e8c54af Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 12 Nov 2018 21:51:23 -0500 Subject: [PATCH 10/10] Release v2.0.5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 2165f8f..e010258 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.4 +2.0.5