Add language storage during server and profile polling

- Add index on language field in profile and server tables
This commit is contained in:
Hypolite Petovan 2018-11-15 23:32:52 -05:00
parent 650f0b4009
commit bc5a0fb590
5 changed files with 79 additions and 61 deletions

View file

@ -85,9 +85,9 @@ class Profile extends \Friendica\Directory\Model
'limit' => [$limit, \PDO::PARAM_INT] 'limit' => [$limit, \PDO::PARAM_INT]
]); ]);
$stmt = 'SELECT p.`id`, p.`name`, p.`username`, p.`addr`, p.`account_type`, p.`pdesc`, $stmt = 'SELECT p.`id`, p.`name`, p.`username`, p.`addr`, p.`account_type`, p.`language`,
p.`locality`, p.`region`, p.`country`, p.`profile_url`, p.`dfrn_request`, p.`photo`, p.`pdesc`, p.`locality`, p.`region`, p.`country`, p.`profile_url`, p.`dfrn_request`,
p.`tags`, p.`last_activity` p.`photo`, p.`tags`, p.`last_activity`
FROM `profile` p FROM `profile` p
JOIN `server` s ON s.`id` = p.`server_id` AND s.`available` AND NOT s.`hidden` JOIN `server` s ON s.`id` = p.`server_id` AND s.`available` AND NOT s.`hidden`
WHERE p.`available` WHERE p.`available`

View file

@ -137,6 +137,7 @@ class Profile
} else { } else {
$this->logger->notice('Parsing profile page ' . $profile_uri); $this->logger->notice('Parsing profile page ' . $profile_uri);
$params = \Friendica\Directory\Utils\Scrape::retrieveProfileData($profile_uri); $params = \Friendica\Directory\Utils\Scrape::retrieveProfileData($profile_uri);
$params['language'] = $server['language'];
} }
// Empty result is due to an offline site. // Empty result is due to an offline site.
@ -195,62 +196,71 @@ class Profile
$filled_fields = intval(!empty($params['pdesc'])) * 4 + intval(!empty($params['tags'])) * 2 + intval(!empty($params['locality']) || !empty($params['region']) || !empty($params['country-name'])); $filled_fields = intval(!empty($params['pdesc'])) * 4 + intval(!empty($params['tags'])) * 2 + intval(!empty($params['locality']) || !empty($params['region']) || !empty($params['country-name']));
$this->logger->debug(var_export($params, true));
$values = [
'profile_id' => $profile_id,
'server_id' => $server['id'],
'username' => $username,
'name' => $params['fn'],
'pdesc' => $params['pdesc'] ?? '',
'locality' => $params['locality'] ?? '',
'region' => $params['region'] ?? '',
'country' => $params['country-name'] ?? '',
'profile_url' => $profile_uri,
'dfrn_request' => $params['dfrn-request'] ?? null,
'photo' => $params['photo'],
'tags' => implode(' ', $tags),
'addr' => $addr,
'account_type' => $account_type,
'language' => $params['language'] ?? null,
'filled_fields'=> $filled_fields,
'last_activity'=> $params['last-activity'] ?? null,
'available' => $available,
];
$this->logger->debug(var_export($values, true));
$this->atlas->perform('INSERT INTO `profile` SET $this->atlas->perform('INSERT INTO `profile` SET
`id` = :profile_id, `server_id` = :server_id,
`server_id` = :server_id, `username` = :username,
`username` = :username, `name` = :name,
`name` = :name, `pdesc` = :pdesc,
`pdesc` = :pdesc, `locality` = :locality,
`locality` = :locality, `region` = :region,
`region` = :region, `country` = :country,
`country` = :country, `profile_url` = :profile_url,
`profile_url` = :profile_url, `dfrn_request` = :dfrn_request,
`dfrn_request` = :dfrn_request, `photo` = :photo,
`tags` = :tags, `tags` = :tags,
`addr` = :addr, `addr` = :addr,
`account_type` = :account_type, `account_type` = :account_type,
`language` = :language,
`filled_fields` = :filled_fields, `filled_fields` = :filled_fields,
`last_activity` = :last_activity, `last_activity` = :last_activity,
`available` = :available, `available` = :available,
`created` = NOW(), `created` = NOW(),
`updated` = NOW() `updated` = NOW()
ON DUPLICATE KEY UPDATE ON DUPLICATE KEY UPDATE
`server_id` = :server_id, `server_id` = :server_id,
`username` = :username, `username` = :username,
`name` = :name, `name` = :name,
`pdesc` = :pdesc, `pdesc` = :pdesc,
`locality` = :locality, `locality` = :locality,
`region` = :region, `region` = :region,
`country` = :country, `country` = :country,
`profile_url` = :profile_url, `profile_url` = :profile_url,
`dfrn_request` = :dfrn_request, `dfrn_request` = :dfrn_request,
`photo` = :photo, `photo` = :photo,
`tags` = :tags, `tags` = :tags,
`addr` = :addr, `addr` = :addr,
`account_type` = :account_type, `account_type` = :account_type,
`language` = :language,
`filled_fields` = :filled_fields, `filled_fields` = :filled_fields,
`last_activity` = :last_activity, `last_activity` = :last_activity,
`available` = :available, `available` = :available,
`updated` = NOW()', `updated` = NOW()',
[ $values
'profile_id' => $profile_id,
'server_id' => $server['id'],
'username' => $username,
'name' => $params['fn'],
'pdesc' => $params['pdesc'] ?? '',
'locality' => $params['locality'] ?? '',
'region' => $params['region'] ?? '',
'country' => $params['country-name'] ?? '',
'profile_url' => $profile_uri,
'dfrn_request' => $params['dfrn-request'] ?? null,
'photo' => $params['photo'],
'tags' => implode(' ', $tags),
'addr' => $addr,
'account_type' => $account_type,
'filled_fields' => $filled_fields,
'last_activity' => $params['last-activity'] ?? null,
'available' => $available,
]
); );
if (!$profile_id) { if (!$profile_id) {

View file

@ -148,6 +148,7 @@ class Server
`last_seen` = NOW(), `last_seen` = NOW(),
`base_url` = :base_url, `base_url` = :base_url,
`name` = :name, `name` = :name,
`language` = :language,
`version` = :version, `version` = :version,
`addons` = :addons, `addons` = :addons,
`reg_policy` = :reg_policy, `reg_policy` = :reg_policy,
@ -158,17 +159,18 @@ class Server
`ssl_state` = :ssl_state `ssl_state` = :ssl_state
WHERE `id` = :server_id', WHERE `id` = :server_id',
[ [
'server_id' => $server['id'], 'server_id' => $server['id'],
'base_url' => strtolower($probe_result['data']['url']), 'base_url' => strtolower($probe_result['data']['url']),
'name' => $probe_result['data']['site_name'], 'name' => $probe_result['data']['site_name'],
'version' => $probe_result['data']['version'], 'language' => $probe_result['data']['language'] ?? null,
'addons' => implode(',', $addons), 'version' => $probe_result['data']['version'],
'reg_policy' => $probe_result['data']['register_policy'], 'addons' => implode(',', $addons),
'info' => $probe_result['data']['info'], 'reg_policy' => $probe_result['data']['register_policy'],
'admin_name' => $probe_result['data']['admin']['name'], 'info' => $probe_result['data']['info'],
'admin_name' => $probe_result['data']['admin']['name'],
'admin_profile' => $probe_result['data']['admin']['profile'], 'admin_profile' => $probe_result['data']['admin']['profile'],
'noscrape_url' => $probe_result['data']['no_scrape_url'] ?? null, 'noscrape_url' => $probe_result['data']['no_scrape_url'] ?? null,
'ssl_state' => $probe_result['ssl_state'] 'ssl_state' => $probe_result['ssl_state']
] ]
); );

View file

@ -0,0 +1,3 @@
ALTER table `profile` DROP KEY `language` (`language`);
ALTER table `server` DROP KEY `language` (`language`);

View file

@ -0,0 +1,3 @@
ALTER table `profile` ADD KEY `language` (`language`);
ALTER table `server` ADD KEY `language` (`language`);