Merge branch 'master' into master

INSTALL.md

@@ -65,7 +65,6 @@ DocumentRoot /path/to/friendica-directory/public/
 ### Nginx
 Include this line your nginx config file.
 
-<<<<<<< master
 ```
 root /path/to/friendica-directory/public;
 ```

README.md

@@ -21,6 +21,10 @@ This is an opt-in directory, meaning that each node can choose not to submit its
 
 Please refer to the provided [installation instructions](INSTALL.md).
 
+## Update from a previous version
+
+Please refer to the provided [update instructions](UPDATE.md).
+
 ## See also
 
 - [Project Concepts](docs/Concepts.md)

UPDATE.md

@@ -0,0 +1,42 @@
+# Friendica Directory Update Instructions
+
+## 1. Update the source code
+
+If you installed Friendica Directory in `/path/to/friendica-directory`.
+
+### Using Git
+
+```
+cd /path/to/friendica-directory
+git pull
+composer install
+```
+
+### Using an archive
+
+1. Create a temporary folder to unpack the new archive.
+2. Copy your old `config/local.json` to the new folder.
+3. Swap the folder names.
+4. Remove the temporary folder.
+
+Sample Linux commands:
+```
+cd /path/to
+mkdir friendica-directory-new
+unzip friendica-<version>.zip friendica-directory-new
+cp friendica-directory/config/local.json friendica-directory-new/config
+mv friendica-directory friendica-directory-old
+mv friendica-directory-new friendica-directory
+rm -r friendica-directory-old
+```
+
+## 2. Update the database structure
+
+The database structure may have changed since the last update, fortunately a console command allows to run the migration scripts up to the latest version:
+
+```
+cd /path/to/friendica-directory
+bin/console dbupdate
+```
+
+You're all set!

VERSION

@@ -1 +1 @@
-2.0.3
+2.0.5

src/classes/Controllers/Console.php

@@ -43,9 +43,7 @@ class Console extends \Asika\SimpleConsole\Console
 	{
 		$commandList = '';
 		foreach ($this->routes as $command => $class) {
-			$this->out($class);
-
-			$commandList .= '	' . $command . ' ' . $class::description . "\n";
+			$commandList .= '	' . $command . "\n";
 		}
 
 		$help = <<<HELP

src/classes/Models/ProfilePollQueue.php

@@ -2,6 +2,8 @@
 
 namespace Friendica\Directory\Models;
 
+use Friendica\Directory\Utils\Network;
+
 /**
  * @author Hypolite Petovan <mrpetovan@gmail.com>
  */
@@ -15,6 +17,15 @@ class ProfilePollQueue extends \Friendica\Directory\Model
 			return false;
 		}
 
+		$host = parse_url($url, PHP_URL_HOST);
+		if (!$host) {
+			return false;
+		}
+
+		if (Network::isPublicHost($host)) {
+			return false;
+		}
+
 		$this->atlas->perform(
 			'INSERT IGNORE INTO `profile_poll_queue` SET `profile_url` = :profile_url',
 			['profile_url' => $url]

src/classes/Pollers/Profile.php

@@ -2,6 +2,8 @@
 
 namespace Friendica\Directory\Pollers;
 
+use Friendica\Directory\Utils\Network;
+
 /**
  * @author Hypolite Petovan <mrpetovan@gmail.com>
  */
@@ -127,7 +129,12 @@ class Profile
 			$noscrape = !!$params; //If the result was false, do a scrape after all.
 		}
 
-		if (!$noscrape) {
+		$available = true;
+
+		if ($noscrape) {
+			$available = Network::testURL($profile_uri);
+			$this->logger->debug('Testing ' . $profile_uri . ': ' . ($available?'Success':'Failure'));
+		} else {
 			$this->logger->notice('Parsing profile page ' . $profile_uri);
 			$params = \Friendica\Directory\Utils\Scrape::retrieveProfileData($profile_uri);
 		}
@@ -204,7 +211,7 @@ class Profile
 			`account_type` = :account_type,
 			`filled_fields` = :filled_fields,
 			`last_activity` = :last_activity,
-			`available` = 1,
+			`available` = :available,
 			`created` = NOW(),
 			`updated` = NOW()
 			ON DUPLICATE KEY UPDATE
@@ -223,7 +230,7 @@ class Profile
 			`account_type` = :account_type,
 			`filled_fields` = :filled_fields,
 			`last_activity` = :last_activity,
-			`available` = 1,
+			`available` = :available,
 			`updated` = NOW()',
 			[
 				'profile_id' => $profile_id,
@@ -242,6 +249,7 @@ class Profile
 				'account_type' => $account_type,
 				'filled_fields' => $filled_fields,
 				'last_activity' => $params['last-activity'] ?? null,
+				'available' => $available,
 			]
 		);
 

src/classes/Pollers/Server.php

@@ -173,8 +173,10 @@ class Server
 			);
 
 			//Add the admin to the directory
+			if (!empty($probe_result['data']['admin']['profile'])) {
 				$this->profilePollQueueModel->add($probe_result['data']['admin']['profile']);
 			}
+		}
 
 		if ($server) {
 			//Get the new health.

src/classes/Utils/Network.php

@@ -23,7 +23,7 @@ class Network
 		}
 
 		curl_setopt($ch, CURLOPT_HEADER, 0);
-		curl_setopt($ch, CURLOPT_TIMEOUT, max(intval($timeout), 1)); //Minimum of 1 second timeout.
+		curl_setopt($ch, CURLOPT_TIMEOUT, max($timeout, 1)); //Minimum of 1 second timeout.
 		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
 		curl_setopt($ch, CURLOPT_MAXREDIRS, 8);
 		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@@ -36,6 +36,31 @@ class Network
 		return $s;
 	}
 
+	public static function testURL(string $url, int $timeout = 20): bool
+	{
+		$ch = curl_init($url);
+		if (!$ch) {
+			return false;
+		}
+
+		curl_setopt($ch, CURLOPT_HEADER        , 0);
+		curl_setopt($ch, CURLOPT_TIMEOUT       , max($timeout, 1)); //Minimum of 1 second timeout.
+		curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+		curl_setopt($ch, CURLOPT_MAXREDIRS     , 8);
+		curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($ch, CURLOPT_NOBODY        , true);
+
+		curl_exec($ch);
+
+		$responseCode = intval(curl_getinfo($ch, CURLINFO_RESPONSE_CODE));
+
+		$testSuccess = curl_errno($ch) === 0 && $responseCode < 400;
+
+		curl_close($ch);
+
+		return $testSuccess;
+	}
+
 	/**
 	 * Check if a hostname is public and non-reserved
 	 *

src/classes/Utils/Scrape.php

@@ -56,7 +56,7 @@ class Scrape
 		$scrape_start = microtime(true);
 
 		$params = [];
-		$html = Network::fetchURL($url, $timeout);
+		$html = Network::fetchURL($url, false, $timeout);
 
 		$scrape_fetch_end = microtime(true);
 

src/sql/migrations/up/0002.sql

@@ -0,0 +1,2 @@
+ALTER table `profile` DROP INDEX `profile-ft`;
+ALTER table `profile` ADD FULLTEXT KEY `profile-ft` (`name`, `pdesc`, `profile_url`, `locality`, `region`, `country`, `tags`);