Add language storage during server and profile polling

- Add index on language field in profile and server tables

src/classes/Models/Profile.php

@@ -85,9 +85,9 @@ class Profile extends \Friendica\Directory\Model
 			'limit' => [$limit, \PDO::PARAM_INT]
 		]);
 
-		$stmt = 'SELECT p.`id`, p.`name`, p.`username`, p.`addr`, p.`account_type`, p.`pdesc`,
- 				p.`locality`, p.`region`, p.`country`, p.`profile_url`, p.`dfrn_request`, p.`photo`,
- 				p.`tags`, p.`last_activity`
+		$stmt = 'SELECT p.`id`, p.`name`, p.`username`, p.`addr`, p.`account_type`, p.`language`,
+				p.`pdesc`, p.`locality`, p.`region`, p.`country`, p.`profile_url`, p.`dfrn_request`,
+				p.`photo`, p.`tags`, p.`last_activity`
 			FROM `profile` p
 			JOIN `server` s ON s.`id` = p.`server_id` AND s.`available` AND NOT s.`hidden`
 			WHERE p.`available`

src/classes/Pollers/Profile.php

@@ -137,6 +137,7 @@ class Profile
 		} else {
 			$this->logger->notice('Parsing profile page ' . $profile_uri);
 			$params = \Friendica\Directory\Utils\Scrape::retrieveProfileData($profile_uri);
+			$params['language'] = $server['language'];
 		}
 
 		// Empty result is due to an offline site.
@@ -195,62 +196,71 @@ class Profile
 
 		$filled_fields = intval(!empty($params['pdesc'])) * 4 + intval(!empty($params['tags'])) * 2 + intval(!empty($params['locality']) || !empty($params['region']) || !empty($params['country-name']));
 
+		$this->logger->debug(var_export($params, true));
+
+		$values = [
+			'profile_id'   => $profile_id,
+			'server_id'    => $server['id'],
+			'username'     => $username,
+			'name'         => $params['fn'],
+			'pdesc'        => $params['pdesc'] ?? '',
+			'locality'     => $params['locality'] ?? '',
+			'region'       => $params['region'] ?? '',
+			'country'      => $params['country-name'] ?? '',
+			'profile_url'  => $profile_uri,
+			'dfrn_request' => $params['dfrn-request'] ?? null,
+			'photo'        => $params['photo'],
+			'tags'         => implode(' ', $tags),
+			'addr'         => $addr,
+			'account_type' => $account_type,
+			'language'     => $params['language'] ?? null,
+			'filled_fields'=> $filled_fields,
+			'last_activity'=> $params['last-activity'] ?? null,
+			'available'    => $available,
+		];
+
+		$this->logger->debug(var_export($values, true));
+
 		$this->atlas->perform('INSERT INTO `profile` SET
-			`id` = :profile_id,
-			`server_id` = :server_id,
-			`username` = :username,
-			`name` = :name,
-			`pdesc` = :pdesc,
-			`locality` = :locality,
-			`region` = :region,
-			`country` = :country,
-			`profile_url` = :profile_url,
-			`dfrn_request` = :dfrn_request,
-			`tags` = :tags,
-			`addr` = :addr,
-			`account_type` = :account_type,
+			`server_id`     = :server_id,
+			`username`      = :username,
+			`name`          = :name,
+			`pdesc`         = :pdesc,
+			`locality`      = :locality,
+			`region`        = :region,
+			`country`       = :country,
+			`profile_url`   = :profile_url,
+			`dfrn_request`  = :dfrn_request,
+			`photo`         = :photo,
+			`tags`          = :tags,
+			`addr`          = :addr,
+			`account_type`  = :account_type,
+			`language`      = :language,
 			`filled_fields` = :filled_fields,
 			`last_activity` = :last_activity,
-			`available` = :available,
+			`available`     = :available,
 			`created` = NOW(),
 			`updated` = NOW()
 			ON DUPLICATE KEY UPDATE
-			`server_id` = :server_id,
-			`username` = :username,
-			`name` = :name,
-			`pdesc` = :pdesc,
-			`locality` = :locality,
-			`region` = :region,
-			`country` = :country,
-			`profile_url` = :profile_url,
-			`dfrn_request` = :dfrn_request,
-			`photo` = :photo,
-			`tags` = :tags,
-			`addr` = :addr,
-			`account_type` = :account_type,
+			`server_id`     = :server_id,
+			`username`      = :username,
+			`name`          = :name,
+			`pdesc`         = :pdesc,
+			`locality`      = :locality,
+			`region`        = :region,
+			`country`       = :country,
+			`profile_url`   = :profile_url,
+			`dfrn_request`  = :dfrn_request,
+			`photo`         = :photo,
+			`tags`          = :tags,
+			`addr`          = :addr,
+			`account_type`  = :account_type,
+			`language`      = :language,
 			`filled_fields` = :filled_fields,
 			`last_activity` = :last_activity,
-			`available` = :available,
-			`updated` = NOW()',
-			[
-				'profile_id' => $profile_id,
-				'server_id' => $server['id'],
-				'username' => $username,
-				'name' => $params['fn'],
-				'pdesc' => $params['pdesc'] ?? '',
-				'locality' => $params['locality'] ?? '',
-				'region' => $params['region'] ?? '',
-				'country' => $params['country-name'] ?? '',
-				'profile_url' => $profile_uri,
-				'dfrn_request' => $params['dfrn-request'] ?? null,
-				'photo' => $params['photo'],
-				'tags' => implode(' ', $tags),
-				'addr' => $addr,
-				'account_type' => $account_type,
-				'filled_fields' => $filled_fields,
-				'last_activity' => $params['last-activity'] ?? null,
-				'available' => $available,
-			]
+			`available`     = :available,
+			`updated`       = NOW()',
+			$values
 		);
 
 		if (!$profile_id) {

src/classes/Pollers/Server.php

@@ -148,6 +148,7 @@ class Server
 					`last_seen`     = NOW(),
 					`base_url`      = :base_url,
 					`name`          = :name,
+					`language`      = :language,
 					`version`       = :version,
 					`addons`        = :addons,
 					`reg_policy`    = :reg_policy,
@@ -158,17 +159,18 @@ class Server
 					`ssl_state`     = :ssl_state
 				WHERE `id` = :server_id',
 				[
-					'server_id' => $server['id'],
-					'base_url' => strtolower($probe_result['data']['url']),
-					'name' => $probe_result['data']['site_name'],
-					'version' => $probe_result['data']['version'],
-					'addons' => implode(',', $addons),
-					'reg_policy' => $probe_result['data']['register_policy'],
-					'info' => $probe_result['data']['info'],
-					'admin_name' => $probe_result['data']['admin']['name'],
+					'server_id'     => $server['id'],
+					'base_url'      => strtolower($probe_result['data']['url']),
+					'name'          => $probe_result['data']['site_name'],
+					'language'      => $probe_result['data']['language'] ?? null,
+					'version'       => $probe_result['data']['version'],
+					'addons'        => implode(',', $addons),
+					'reg_policy'    => $probe_result['data']['register_policy'],
+					'info'          => $probe_result['data']['info'],
+					'admin_name'    => $probe_result['data']['admin']['name'],
 					'admin_profile' => $probe_result['data']['admin']['profile'],
-					'noscrape_url' => $probe_result['data']['no_scrape_url'] ?? null,
-					'ssl_state' => $probe_result['ssl_state']
+					'noscrape_url'  => $probe_result['data']['no_scrape_url'] ?? null,
+					'ssl_state'     => $probe_result['ssl_state']
 				]
 			);
 

src/sql/migrations/down/0002.sql

@@ -0,0 +1,3 @@
+ALTER table `profile` DROP KEY `language` (`language`);
+
+ALTER table `server` DROP KEY `language` (`language`);

src/sql/migrations/up/0003.sql

@@ -0,0 +1,3 @@
+ALTER table `profile` ADD KEY `language` (`language`);
+
+ALTER table `server` ADD KEY `language` (`language`);