forked from friendica/friendica-addons
887 lines
44 KiB
PHP
887 lines
44 KiB
PHP
<?php
|
|
/**
|
|
* Name: Retriever
|
|
* Description: Follow the permalink of RSS/Atom feed items and replace the summary with the full content.
|
|
* Version: 1.0
|
|
* Author: Matthew Exon <http://mat.exon.name>
|
|
*/
|
|
|
|
use Friendica\Core\Addon;
|
|
use Friendica\Core\Config;
|
|
use Friendica\Core\PConfig;
|
|
use Friendica\Core\Logger;
|
|
use Friendica\Core\Renderer;
|
|
use Friendica\Core\System;
|
|
use Friendica\Content\Text\HTML;
|
|
use Friendica\Content\Text\BBCode;
|
|
use Friendica\Model\Photo;
|
|
use Friendica\Object\Image;
|
|
use Friendica\Util\Network;
|
|
use Friendica\Core\L10n;
|
|
use Friendica\Database\DBA;
|
|
use Friendica\Model\ItemURI;
|
|
use Friendica\Model\Item;
|
|
use Friendica\Util\DateTimeFormat;
|
|
|
|
function retriever_install() {
|
|
Addon::registerHook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
|
|
Addon::registerHook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
|
|
Addon::registerHook('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
|
|
Addon::registerHook('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
|
|
Addon::registerHook('cron', 'addon/retriever/retriever.php', 'retriever_cron');
|
|
|
|
if (Config::get('retriever', 'dbversion') == '0.10') {
|
|
q('ALTER TABLE `retriever_resource` MODIFY COLUMN `type` char(255) NULL DEFAULT NULL');
|
|
q('ALTER TABLE `retriever_resource` MODIFY COLUMN `data` mediumblob NULL DEFAULT NULL');
|
|
q('ALTER TABLE `retriever_rule` MODIFY COLUMN `data` mediumtext NULL DEFAULT NULL');
|
|
Config::set('retriever', 'dbversion', '0.11');
|
|
}
|
|
if (Config::get('retriever', 'dbversion') == '0.11') {
|
|
q('ALTER TABLE `retriever_resource` ADD INDEX `url` (`url`)');
|
|
q('ALTER TABLE `retriever_resource` ADD INDEX `completed` (`completed`)');
|
|
q('ALTER TABLE `retriever_item` ADD INDEX `finished` (`finished`)');
|
|
q('ALTER TABLE `retriever_item` ADD INDEX `item-uid` (`item-uid`)');
|
|
Config::set('retriever', 'dbversion', '0.12');
|
|
}
|
|
if (Config::get('retriever', 'dbversion') == '0.12') {
|
|
q("ALTER TABLE `retriever_resource` ADD COLUMN `contact-id` int(10) unsigned NOT NULL DEFAULT '0' AFTER `id`");
|
|
q("ALTER TABLE `retriever_resource` ADD COLUMN `item-uid` int(10) unsigned NOT NULL DEFAULT '0' AFTER `id`");
|
|
Config::set('retriever', 'dbversion', '0.13');
|
|
}
|
|
if (Config::get('retriever', 'dbversion') == '0.13') {
|
|
Config::set('retriever', 'downloads_per_cron', '100');
|
|
}
|
|
if (Config::get('retriever', 'dbversion') != '0.14') {
|
|
$schema = file_get_contents(dirname(__file__).'/database.sql');
|
|
$arr = explode(';', $schema);
|
|
foreach ($arr as $a) {
|
|
if (!DBA::e($a)) {
|
|
Logger::warning('Unable to create database table: ' . DBA::errorMessage());
|
|
return;
|
|
}
|
|
}
|
|
Config::set('retriever', 'downloads_per_cron', '100');
|
|
Config::set('retriever', 'dbversion', '0.14');
|
|
}
|
|
}
|
|
|
|
function retriever_uninstall() {
|
|
Addon::unregisterHook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
|
|
Addon::unregisterHook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
|
|
Addon::unregisterHook('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
|
|
Addon::unregisterHook('plugin_settings', 'addon/retriever/retriever.php', 'retriever_plugin_settings');
|
|
Addon::unregisterHook('plugin_settings_post', 'addon/retriever/retriever.php', 'retriever_plugin_settings_post');
|
|
Addon::unregisterHook('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
|
|
Addon::unregisterHook('cron', 'addon/retriever/retriever.php', 'retriever_cron');
|
|
}
|
|
|
|
function retriever_module() {}
|
|
|
|
function retriever_addon_admin(&$a, &$o) {
|
|
$downloads_per_cron = Config::get('retriever', 'downloads_per_cron');
|
|
$template = Renderer::getMarkupTemplate('admin.tpl', 'addon/retriever/');
|
|
$config = ['downloads_per_cron',
|
|
L10n::t('Downloads per Cron'),
|
|
$downloads_per_cron,
|
|
L10n::t('Maximum number of downloads to attempt during each run of the cron job.')];
|
|
$o .= Renderer::replaceMacros($template, [
|
|
'$downloads_per_cron' => $config,
|
|
'$submit' => L10n::t('Save Settings')]);
|
|
}
|
|
|
|
function retriever_addon_admin_post ($a) {
|
|
if (!empty($_POST['downloads_per_cron'])) {
|
|
Config::set('retriever', 'downloads_per_cron', $_POST['downloads_per_cron']);
|
|
}
|
|
}
|
|
|
|
function retriever_cron($a, $b) {
|
|
$downloads_per_cron = Config::get('retriever', 'downloads_per_cron');
|
|
|
|
// Do this first, otherwise it can interfere with retriever_retrieve_items
|
|
retriever_clean_up_completed_resources($downloads_per_cron, $a);
|
|
|
|
retriever_retrieve_items($downloads_per_cron, $a);
|
|
retriever_tidy();
|
|
}
|
|
|
|
$retriever_item_count = 0;
|
|
|
|
function retriever_retrieve_items($max_items, $a) {
|
|
global $retriever_item_count;
|
|
|
|
$retriever_schedule = array(array(1,'minute'),
|
|
array(10,'minute'),
|
|
array(1,'hour'),
|
|
array(1,'day'),
|
|
array(2,'day'),
|
|
array(1,'week'),
|
|
array(1,'month'));
|
|
|
|
$schedule_clauses = array();
|
|
for ($i = 0; $i < count($retriever_schedule); $i++) {
|
|
$num = $retriever_schedule[$i][0];
|
|
$unit = $retriever_schedule[$i][1];
|
|
array_push($schedule_clauses,
|
|
'(`num-tries` = ' . $i . ' AND TIMESTAMPADD(' . DBA::escape($unit) .
|
|
', ' . intval($num) . ', `last-try`) < now())');
|
|
}
|
|
|
|
$retrieve_items = $max_items - $retriever_item_count;
|
|
do {
|
|
Logger::debug('retriever_retrieve_items: asked for maximum ' . $max_items . ', already retrieved ' . intval($retriever_item_count) . ', retrieve ' . $retrieve_items);
|
|
// TODO: figure out how to do this with DBA module //@@@ this is possible
|
|
$retriever_resources2 = DBA::selectToArray('retriever_resource', [], ['`completed` IS NULL AND (`last-try` IS NULL OR ' . implode($schedule_clauses, ' OR ') . ')'], ['order' => ['last-try' => 0], 'limit' => $retrieve_items]);
|
|
$retriever_resources = q("SELECT * FROM `retriever_resource` WHERE `completed` IS NULL AND (`last-try` IS NULL OR %s) ORDER BY `last-try` ASC LIMIT %d",
|
|
DBA::escape(implode($schedule_clauses, ' OR ')),
|
|
intval($retrieve_items));
|
|
if (!is_array($retriever_resources)) {
|
|
break;
|
|
}
|
|
if (count($retriever_resources) == 0) {
|
|
break;
|
|
}
|
|
Logger::debug('retriever_retrieve_items: found ' . count($retriever_resources) . ' waiting resources in database');
|
|
Logger::debug('@@@ retriever_retrieve_items: alternative found ' . count($retriever_resources2) . ': ' . print_r($retriever_resources2, true));
|
|
foreach ($retriever_resources as $retriever_resource) {
|
|
retrieve_resource($retriever_resource);
|
|
$retriever_item_count++;
|
|
}
|
|
$retrieve_items = $max_items - $retriever_item_count;
|
|
}
|
|
while ($retrieve_items > 0);
|
|
Logger::debug('retriever_retrieve_items: finished retrieving items');
|
|
}
|
|
|
|
// Look for items that are waiting even though the resource has completed. This shouldn't happen, but is worth cleaning up if it does.
|
|
function retriever_clean_up_completed_resources($max_items, $a) {
|
|
// TODO: figure out how to do this with DBA module
|
|
$r = q('SELECT retriever_resource.`id` as resource, retriever_item.`id` as item FROM retriever_resource, retriever_item, retriever_rule WHERE retriever_item.`finished` = 0 AND retriever_item.`resource` = retriever_resource.`id` AND retriever_resource.`completed` IS NOT NULL AND retriever_item.`contact-id` = retriever_rule.`contact-id` AND retriever_item.`item-uid` = retriever_rule.`uid` LIMIT %d',
|
|
intval($max_items));
|
|
if (!$r) {
|
|
$r = array();
|
|
}
|
|
Logger::debug('retriever_clean_up_completed_resources: items waiting even though resource has completed: ' . count($r));
|
|
foreach ($r as $rr) {
|
|
$retriever_item = retriever_get_retriever_item($rr['item']);
|
|
if (!DBA::isResult($retriever_item)) {
|
|
Logger::warning('retriever_clean_up_completed_resources: no retriever item with id ' . $rr['item']);
|
|
continue;
|
|
}
|
|
$item = retriever_get_item($retriever_item);
|
|
if (!$item) {
|
|
Logger::warning('retriever_clean_up_completed_resources: no item ' . $retriever_item['item-uri']);
|
|
continue;
|
|
}
|
|
$retriever_rule = get_retriever_rule($retriever_item['contact-id'], $item['uid']);
|
|
if (!$retriever_rule) {
|
|
Logger::warning('retriever_clean_up_completed_resources: no retriever for uri ' . $retriever_item['item-uri'] . ' uid ' . $retriever_item['uid'] . ' ' . $retriever_item['contact-id']);
|
|
continue;
|
|
}
|
|
$resource = DBA::selectFirst('retriever_resource', [], ['id' => intval($rr['resource'])]);
|
|
retriever_apply_completed_resource_to_item($retriever_rule, $item, $resource, $a);
|
|
Logger::info('@@@ retriever_clean_up_completed_resources tried to update id ' . $retriever_item['id'] . ' to finished, better check that it really worked!');
|
|
DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]);
|
|
retriever_check_item_completed($item);
|
|
}
|
|
}
|
|
|
|
function retriever_tidy() {
|
|
DBA::delete('retriever_resource', ['completed IS NOT NULL AND completed < DATE_SUB(now(), INTERVAL 1 WEEK)']);
|
|
DBA::delete('retriever_resource', ['completed IS NULL AND created < DATE_SUB(now(), INTERVAL 3 MONTH)']);
|
|
|
|
$r = q("SELECT retriever_item.id FROM retriever_item LEFT OUTER JOIN retriever_resource ON (retriever_item.resource = retriever_resource.id) WHERE retriever_resource.id is null");
|
|
Logger::info('retriever_tidy: found ' . count($r) . ' retriever_items with no retriever_resource');
|
|
foreach ($r as $rr) {
|
|
q('DELETE FROM retriever_item WHERE id = %d', intval($rr['id']));
|
|
}
|
|
}
|
|
|
|
function retrieve_dataurl_resource($resource) {
|
|
if (!preg_match("/date:(.*);base64,(.*)/", $resource['url'], $matches)) {
|
|
Logger::info('retrieve_dataurl_resource: ' . $resource['id'] . ' does not match pattern');
|
|
} else {
|
|
$resource['type'] = $matches[1];
|
|
$resource['data'] = base64url_decode($matches[2]);
|
|
}
|
|
|
|
// Succeed or fail, there's no point retrying
|
|
q("UPDATE `retriever_resource` SET `last-try` = now(), `num-tries` = `num-tries` + 1, `completed` = now(), `data` = '%s', `type` = '%s' WHERE id = %d",
|
|
DBA::escape($resource['data']),
|
|
DBA::escape($resource['type']),
|
|
intval($resource['id']));
|
|
retriever_resource_completed($resource, $a);
|
|
}
|
|
|
|
function retrieve_resource($resource) {
|
|
if (substr($resource['url'], 0, 5) == "data:") {
|
|
return retrieve_dataurl_resource($resource);
|
|
}
|
|
|
|
$a = get_app();
|
|
|
|
$retriever_rule = get_retriever_rule($resource['contact-id'], $resource['item-uid']);
|
|
if (!$retriever_rule) {
|
|
Logger::warning('retrieve_resource: no rule found for contact ' . $resource['contact-id'] . ' item ' . $resource['item-uid']);
|
|
return;
|
|
}
|
|
$rule_data = $retriever_rule['data'];
|
|
if (!$rule_data) {
|
|
Logger::warning('retrieve_resource: no rule data found for contact ' . $resource['contact-id'] . ' item ' . $resource['item-uid']);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
Logger::debug('retrieve_resource: ' . ($resource['num-tries'] + 1) . ' attempt at resource ' . $resource['id'] . ' ' . $resource['url']);
|
|
$redirects = 0;
|
|
$cookiejar = '';
|
|
if (array_key_exists('storecookies', $rule_data) && $rule_data['storecookies']) {
|
|
$cookiejar = tempnam(get_temppath(), 'cookiejar-retriever-');
|
|
file_put_contents($cookiejar, $rule_data['cookiedata']);
|
|
}
|
|
$fetch_result = Network::fetchUrlFull($resource['url'], $resource['binary'], $redirects, '', $cookiejar);
|
|
if (array_key_exists('storecookies', $rule_data) && $rule_data['storecookies']) {
|
|
$retriever_rule['data']['cookiedata'] = file_get_contents($cookiejar);
|
|
DBA::update('retriever_rule', ['data' => json_encode($retriever_rule['data'])], ['id' => intval($retriever_rule["id"])], $retriever_rule);
|
|
//@@@ check the update worked
|
|
unlink($cookiejar);
|
|
}
|
|
$resource['data'] = $fetch_result->getBody();
|
|
$resource['http-code'] = $fetch_result->getReturnCode();
|
|
$resource['type'] = $fetch_result->getContentType();
|
|
$resource['redirect-url'] = $fetch_result->getRedirectUrl();
|
|
Logger::debug('retrieve_resource: got code ' . $resource['http-code'] . ' retrieving resource ' . $resource['id'] . ' final url ' . $resource['redirect-url']);
|
|
} catch (Exception $e) {
|
|
Logger::info('retrieve_resource: unable to retrieve ' . $resource['url'] . ' - ' . $e->getMessage());
|
|
}
|
|
// TODO: figure out how to do this with DBA module
|
|
q("UPDATE `retriever_resource` SET `last-try` = now(), `num-tries` = `num-tries` + 1, `http-code` = %d, `redirect-url` = '%s' WHERE id = %d",
|
|
intval($resource['http-code']),
|
|
DBA::escape($resource['redirect-url']),
|
|
intval($resource['id']));
|
|
if ($resource['data']) {
|
|
// TODO: figure out how to do this with DBA module
|
|
q("UPDATE `retriever_resource` SET `completed` = now(), `data` = '%s', `type` = '%s' WHERE id = %d",
|
|
DBA::escape($resource['data']),
|
|
DBA::escape($resource['type']),
|
|
intval($resource['id']));
|
|
retriever_resource_completed($resource, $a);
|
|
}
|
|
}
|
|
|
|
function get_retriever_rule($contact_id, $uid, $create = false) {
|
|
$retriever_rule = DBA::selectFirst('retriever_rule', [], ['contact-id' => intval($contact_id), 'uid' => intval($uid)]);
|
|
if ($retriever_rule) {
|
|
$retriever_rule['data'] = json_decode($retriever_rule['data'], true);
|
|
return $retriever_rule;
|
|
}
|
|
if ($create) {
|
|
DBA::insert('retriever_rule', ['uid' => intval($uid), 'contact-id' => intval($contact_id)]);
|
|
$retriever_rule = DBA::selectFirst('retriever_rule', [], ['contact-id' => intval($contact_id), 'uid' => intval($uid)]);
|
|
return $retriever_rule;
|
|
}
|
|
}
|
|
|
|
function retriever_get_retriever_item($id) {
|
|
return DBA::selectFirst('retriever_item', [], ['id' => intval($id)]);
|
|
}
|
|
|
|
function retriever_get_item($retriever_item) {
|
|
$item = Item::selectFirst([], ['uri' => $retriever_item['item-uri'], 'uid' => intval($retriever_item['item-uid']), 'contact-id' => intval($retriever_item['contact-id'])]);
|
|
if (!DBA::isResult($item)) {
|
|
Logger::warning('retriever_get_item: no item found for uri ' . $retriever_item['item-uri']);
|
|
return;
|
|
}
|
|
return $item;
|
|
}
|
|
|
|
function retriever_item_completed($retriever_item_id, $resource) {
|
|
Logger::debug('retriever_item_completed: id ' . $retriever_item_id . ' url ' . $resource['url']);
|
|
|
|
$retriever_item = retriever_get_retriever_item($retriever_item_id);
|
|
if (!DBA::isResult($retriever_item)) {
|
|
Logger::info('retriever_item_completed: no retriever item with id ' . $retriever_item_id);
|
|
return;
|
|
}
|
|
$item = retriever_get_item($retriever_item);
|
|
if (!$item) {
|
|
Logger::warning('retriever_item_completed: no item ' . $retriever_item['item-uri']);
|
|
return;
|
|
}
|
|
// Note: the retriever might be null. Doesn't matter.
|
|
$retriever_rule = get_retriever_rule($retriever_item['contact-id'], $retriever_item['item-uid']);
|
|
|
|
retriever_apply_completed_resource_to_item($retriever_rule, $item, $resource);
|
|
|
|
DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]);
|
|
retriever_check_item_completed($item);
|
|
}
|
|
|
|
function retriever_resource_completed($resource) {
|
|
Logger::debug('retriever_resource_completed: id ' . $resource['id'] . ' url ' . $resource['url']);
|
|
foreach (DBA::selectToArray('retriever_item', ['id'], ['resource' => intval($resource['id'])]) as $retriever_item) {
|
|
retriever_item_completed($retriever_item['id'], $resource);
|
|
}
|
|
}
|
|
|
|
function apply_retrospective($retriever, $num) {
|
|
foreach (Item::selectToArray([], ['contact-id' => intval($retriever['contact-id'])], ['order' => ['received' => true], 'limit' => $num]) as $item) {
|
|
Item::update(['visible' => 0], ['id' => intval($item['id'])]);
|
|
//@@@ check that this works
|
|
foreach (DBA::selectToArray('retriever_item', [], ['item-uri' => $item['uri'], 'item-uid' => $item['uid'], 'contact-id' => $item['contact-id']]) as $retriever_item) {
|
|
DBA::delete('retriever_resource', ['id' => $retriever_item['resource']]);
|
|
DBA::delete('retriever_item', ['id' => $retriever_item['id']]);
|
|
}
|
|
retriever_on_item_insert($retriever, $item);
|
|
}
|
|
}
|
|
|
|
// TODO: Currently this waits until the next cron before actually downloading. Should do it immediately.
|
|
//@@@ I think the above statement is wrong. Check!
|
|
// TODO: This queries then inserts. It should use some kind of lock to avoid requesting the same resource twice.
|
|
function retriever_on_item_insert($retriever, &$item) {
|
|
if (!$retriever || !$retriever['id']) {
|
|
Logger::info('retriever_on_item_insert: No retriever supplied');
|
|
return;
|
|
}
|
|
if (!array_key_exists('enable', $retriever['data']) || !$retriever['data']['enable'] == "on") {
|
|
return;
|
|
}
|
|
if (array_key_exists('plink', $item) && strlen($item['plink'])) {
|
|
$url = $item['plink'];
|
|
}
|
|
else {
|
|
if (!array_key_exists('uri-id', $item)) {
|
|
Logger::warning('retriever_on_item_insert: item ' . $item['id'] . ' has no plink and no uri-id');
|
|
return;
|
|
}
|
|
$content = DBA::selectFirst('item-content', [], ['uri-id' => $item['uri-id']]);
|
|
$url = $content['plink'];
|
|
}
|
|
|
|
if (array_key_exists('modurl', $retriever['data']) && $retriever['data']['modurl']) {
|
|
$orig_url = $url;
|
|
$url = preg_replace('/' . $retriever['data']['pattern'] . '/', $retriever['data']['replace'], $orig_url);
|
|
Logger::debug('retriever_on_item_insert: Changed ' . $orig_url . ' to ' . $url);
|
|
}
|
|
|
|
$resource = add_retriever_resource($url, $item['uid'], $item['contact-id']);
|
|
$retriever_item_id = add_retriever_item($item, $resource);
|
|
}
|
|
|
|
function add_retriever_resource($url, $uid, $cid, $binary = false) {
|
|
Logger::debug('add_retriever_resource: url ' . $url . ' uid ' . $uid . ' contact-id ' . $cid);
|
|
|
|
$scheme = parse_url($url, PHP_URL_SCHEME);
|
|
if ($scheme == 'data') {
|
|
$fp = fopen($url, 'r');
|
|
$meta = stream_get_meta_data($fp);
|
|
$type = $meta['mediatype'];
|
|
$data = stream_get_contents($fp);
|
|
fclose($fp);
|
|
|
|
$url = 'md5://' . hash('md5', $url);
|
|
$resource = DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]);
|
|
if ($resource) {
|
|
Logger::debug('add_retriever_resource: Resource ' . $url . ' already requested');
|
|
return $resource;
|
|
}
|
|
|
|
DBA::insert('retriever_resource', ['item-uid' => intval($uid), 'contact-id' => intval($cid), 'type' => $type, 'binary' => ($binary ? 1 : 0), 'url' => $url, 'completed' => DateTimeFormat::utcNow(), 'data' => $data]);
|
|
// @@@ check that this makes sense
|
|
$resource = DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]);
|
|
if ($resource) {
|
|
retriever_resource_completed($resource);
|
|
}
|
|
return $resource;
|
|
}
|
|
|
|
if (strlen($url) > 800) {
|
|
Logger::warning('add_retriever_resource: URL is longer than 800 characters');
|
|
}
|
|
|
|
$resource = DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]);
|
|
if ($resource) {
|
|
Logger::debug('add_retriever_resource: Resource ' . $url . ' uid ' . $uid . ' cid ' . $cid . ' already requested');
|
|
return $resource;
|
|
}
|
|
|
|
DBA::insert('retriever_resource', ['item-uid' => intval($uid), 'contact-id' => intval($cid), 'binary' => ($binary ? 1 : 0), 'url' => $url]);
|
|
return DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]);
|
|
}
|
|
|
|
function add_retriever_item(&$item, $resource) {
|
|
Logger::debug('add_retriever_item: ' . $resource['url'] . ' for ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
|
|
|
|
if (!array_key_exists('id', $resource) || !$resource['id']) {
|
|
Logger::warning('add_retriever_item: resource is empty');
|
|
//@@@ check that this does not happen
|
|
return;
|
|
}
|
|
if (DBA::selectFirst('retriever_item', [], ['item-uri' => $item['uri'], 'item-uid' => intval($item['uid']), 'resource' => intval($resource['id'])])) {
|
|
//@@@ check that this worked
|
|
Logger::info("add_retriever_item: retriever item already present for " . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
|
|
return;
|
|
}
|
|
DBA::insert('retriever_item', ['item-uri' => $item['uri'], 'item-uid' => intval($item['uid']), 'contact-id' => intval($item['contact-id']), 'resource' => intval($resource['id'])]);
|
|
$retriever_item = DBA::selectFirst('retriever_item', ['id'], ['item-uri' => $item['uri'], 'item-uid' => intval($item['uid']), 'resource' => intval($resource['id'])]);
|
|
if (!$retriever_item) {
|
|
Logger::info("add_retriever_item: couldn't create retriever item for " . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
|
|
return;
|
|
}
|
|
Logger::debug('add_retriever_item: created retriever_item ' . $retriever_item['id'] . ' for item ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
|
|
return $retriever_item['id'];
|
|
}
|
|
|
|
function retriever_get_encoding($resource) {
|
|
$matches = array();
|
|
if (preg_match('/charset=(.*)/', $resource['type'], $matches)) {
|
|
return trim(array_pop($matches));
|
|
}
|
|
return 'utf-8';
|
|
}
|
|
|
|
function retriever_apply_xslt_text($xslt_text, $doc) {
|
|
if (!$xslt_text) {
|
|
Logger::info('retriever_apply_xslt_text: empty XSLT text');
|
|
return $doc;
|
|
}
|
|
$xslt_doc = new DOMDocument();
|
|
if (!$xslt_doc->loadXML($xslt_text)) {
|
|
Logger::info('retriever_apply_xslt_text: could not load XML');
|
|
return $doc;
|
|
}
|
|
$xp = new XsltProcessor();
|
|
$xp->importStylesheet($xslt_doc);
|
|
$result = $xp->transformToDoc($doc);
|
|
return $result;
|
|
}
|
|
|
|
function retriever_apply_dom_filter($retriever, &$item, $resource) {
|
|
Logger::debug('retriever_apply_dom_filter: applying XSLT to uri ' . $item['uri'] . ' uid ' . $item['uid'] . ' contact ' . $item['contact-id']);
|
|
|
|
if (!array_key_exists('include', $retriever['data']) && !array_key_exists('customxslt', $retriever['data'])) {
|
|
Logger::info('retriever_apply_dom_filter: no include and no customxslt');
|
|
return;
|
|
}
|
|
if (!$resource['data']) {
|
|
Logger::info('retriever_apply_dom_filter: no text to work with');
|
|
return;
|
|
}
|
|
|
|
$doc = retriever_load_into_dom($resource);
|
|
|
|
$doc = retriever_extract($doc, $retriever);
|
|
if (!$doc) {
|
|
Logger::info('retriever_apply_dom_filter: failed to apply extract XSLT template');
|
|
return;
|
|
}
|
|
|
|
$doc = retriever_globalise_urls($doc, $resource);
|
|
if (!$doc) {
|
|
Logger::info('retriever_apply_dom_filter: failed to apply fix urls XSLT template');
|
|
return;
|
|
}
|
|
|
|
$body = HTML::toBBCode($doc->saveHTML());
|
|
if (!strlen($body)) {
|
|
Logger::info('retriever_apply_dom_filter retriever ' . $retriever['id'] . ' item ' . $item['id'] . ': output was empty');
|
|
return;
|
|
}
|
|
$body .= "\n\n" . L10n::t('Retrieved') . ' ' . date("Y-m-d") . ': [url=';
|
|
$body .= $item['plink'];
|
|
$body .= ']' . $item['plink'] . '[/url]';
|
|
|
|
Logger::debug('retriever_apply_dom_filter: XSLT result \"' . $body . '\"');
|
|
$item['body'] = $body;
|
|
if (array_key_exists('id', $item) && $item['id']) { //@@@ this should be a separate function
|
|
//@@@ check that this works
|
|
Logger::debug('@@@ retriever_apply_dom_filter updating item by id ' . $item['id']);
|
|
Item::update(['body' => $body], ['id' => $item['id']]);
|
|
}
|
|
}
|
|
|
|
function retriever_load_into_dom($resource) {
|
|
$encoding = retriever_get_encoding($resource);
|
|
$content = mb_convert_encoding($resource['data'], 'HTML-ENTITIES', $encoding);
|
|
$doc = new DOMDocument('1.0', 'UTF-8');
|
|
if (strpos($resource['type'], 'html') !== false) {
|
|
@$doc->loadHTML($content);
|
|
}
|
|
else {
|
|
$doc->loadXML($content);
|
|
}
|
|
return $doc;
|
|
}
|
|
|
|
function retriever_extract($doc, $retriever) {
|
|
$params = array('$spec' => $retriever['data']);
|
|
$extract_template = Renderer::getMarkupTemplate('extract.tpl', 'addon/retriever/');
|
|
$extract_xslt = Renderer::replaceMacros($extract_template, $params);
|
|
if ($retriever['data']['include']) {
|
|
Logger::debug('retriever_apply_dom_filter: applying include/exclude template \"' . $extract_xslt . '\"');
|
|
$doc = retriever_apply_xslt_text($extract_xslt, $doc);
|
|
}
|
|
if (array_key_exists('customxslt', $retriever['data']) && $retriever['data']['customxslt']) {
|
|
Logger::debug('retriever_extract: applying custom XSLT \"' . $retriever['data']['customxslt'] . '\"');
|
|
$doc = retriever_apply_xslt_text($retriever['data']['customxslt'], $doc);
|
|
}
|
|
return $doc;
|
|
}
|
|
|
|
function retriever_globalise_urls($doc, $resource) {
|
|
$components = parse_url($resource['redirect-url']);
|
|
$rooturl = $components['scheme'] . "://" . $components['host'];
|
|
$dirurl = $rooturl . dirname($components['path']) . "/";
|
|
$params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl);
|
|
$fix_urls_template = Renderer::getMarkupTemplate('fix-urls.tpl', 'addon/retriever/');
|
|
$fix_urls_xslt = Renderer::replaceMacros($fix_urls_template, $params);
|
|
$doc = retriever_apply_xslt_text($fix_urls_xslt, $doc);
|
|
return $doc;
|
|
}
|
|
|
|
function retriever_get_body($item) {
|
|
if (array_key_exists('id', $item) && $item['id']) {
|
|
// item has already been stored in database
|
|
if (!array_key_exists('uri-id', $item) || !$item['uri-id']) {
|
|
Logger::warning('retriever_get_body: item uri ' . $item['uri'] . ' has id but no uri-id');
|
|
//@@@ check never happens
|
|
return $item['body'];
|
|
}
|
|
$content = DBA::selectFirst('item-content', [], ['body'], ['uri-id' => $item['uri-id']]);
|
|
if (!$content) {
|
|
Logger::warning('retriever_get_body: item-content uri-id ' . $item['uri-id'] . ' has no content');
|
|
//@@@ check never happens
|
|
return $item['body'];
|
|
}
|
|
if (!$content['body']) {
|
|
Logger::warning('retriever_get_body: item-content uri-id ' . $item['uri-id'] . ' has no body');
|
|
//@@@ check never happens
|
|
return $item['body'];
|
|
}
|
|
if ($content['body'] != $item['body']) {
|
|
Logger::warning('@@@ this is probably bad content: ' . $content['body'] . ' item ' . $item['body']);
|
|
//@@@ check for this.
|
|
}
|
|
Logger::debug('@@@ retriever_get_body uri-id ' . $item['uri-id'] . ' body: ' . $content['body']);
|
|
return $content['body'];
|
|
}
|
|
// item has not yet been stored in database
|
|
Logger::debug('@@@ retriever_get_body id ' . $item['id'] . ' body: ' . $item['body']);
|
|
return $item['body'];
|
|
}
|
|
|
|
function retriever_set_body(&$item, $body, $allow_empty = false) {
|
|
if (!$body && !$allow_empty) {
|
|
Logger::debug('retriever_set_body: will not set empty body in item id ' . $item['id'] . ' uri ' . $item['uri']);
|
|
return;
|
|
}
|
|
$item['body'] = $body;
|
|
Logger::debug('@@@ retriever_set_body set array value to ' . $body);
|
|
if (array_key_exists('id', $item) && $item['id']) {
|
|
// item has already been stored in database
|
|
Logger::debug('@@@ retriever_set_body updating item ' . print_r($item, true) . ' to ' . $body);
|
|
Item::update(['body' => $body], ['id' => intval($item['id'])]);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief @@@
|
|
*
|
|
* @param array &$item Row from the item table (by ref)
|
|
*/
|
|
function retrieve_images(&$item) {
|
|
$body = retriever_get_body($item);
|
|
if (!strlen($body)) {
|
|
Logger::warning('retrieve_images: no body for item ' . $item['uri']);
|
|
return;
|
|
}
|
|
|
|
// I suspect that the first two are not used any more?
|
|
preg_match_all("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", $body, $matches1);
|
|
preg_match_all("/\[img\](.*?)\[\/img\]/ism", $body, $matches2);
|
|
preg_match_all("/\[img\=([^\]]*)\]([^[]*)\[\/img\]/ism", $body, $matches3);
|
|
$matches = array_merge($matches1[3], $matches2[1], $matches3[1]);
|
|
Logger::debug('retrieve_images: found ' . count($matches) . ' images for item ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
|
|
foreach ($matches as $url) {
|
|
if (!$url) {
|
|
continue;
|
|
}
|
|
if (strpos($url, get_app()->getBaseUrl()) === FALSE) {
|
|
$resource = add_retriever_resource($url, $item['uid'], $item['contact-id'], true);
|
|
Logger::debug('@@@ check this makes sense 2: ' . $resource['id'] . ' url ' . $resource['url']);
|
|
if (!$resource['completed']) {
|
|
add_retriever_item($item, $resource);
|
|
}
|
|
else {
|
|
retriever_transform_images($item, $resource);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function retriever_check_item_completed(&$item)
|
|
{
|
|
$waiting = DBA::selectFirst('retriever_item', [], ['item-uri' => $item['uri'], 'item-uid' => intval($item['uid']), 'contact-id' => intval($item['contact-id']), 'finished' => 0]);
|
|
Logger::debug('retriever_check_item_completed: item ' . $item['uri'] . ' ' . $item['uid'] . ' '. $item['contact-id'] . ' waiting for resources');
|
|
$old_visible = $item['visible'];
|
|
$item['visible'] = $waiting ? 0 : 1;
|
|
if (array_key_exists('id', $item) && ($item['id'] > 0) && ($old_visible != $item['visible'])) {
|
|
Logger::debug('retriever_check_item_completed: changing visible flag to ' . $item['visible']);
|
|
Item::update(['visible' => $item['visible']], ['id' => intval($item['id'])]);
|
|
}
|
|
}
|
|
|
|
function retriever_apply_completed_resource_to_item($retriever, &$item, $resource) {
|
|
Logger::debug('retriever_apply_completed_resource_to_item: retriever ' . ($retriever ? $retriever['id'] : 'none') . ' resource ' . $resource['url'] . ' plink ' . $item['plink']);
|
|
if (strpos($resource['type'], 'image') !== false) {
|
|
retriever_transform_images($item, $resource);
|
|
}
|
|
if (!$retriever) {
|
|
Logger::warning('retriever_apply_completed_resource_to_item: no retriever');
|
|
return;
|
|
}
|
|
if ((strpos($resource['type'], 'html') !== false) ||
|
|
(strpos($resource['type'], 'xml') !== false)) {
|
|
retriever_apply_dom_filter($retriever, $item, $resource);
|
|
if ($retriever['data']['images'] ) {
|
|
retrieve_images($item);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* @brief Stores the image downloaded in the supplied resource and updates the item body by replacing the remote URL with the local URL. The body will be updated in the supplied item array. If the item has already been stored, and therefore has an ID already, the row in the database will be updated too.
|
|
*
|
|
* @param array &$item Row from the item table (by ref)
|
|
* @param array $resource Row from the resource table containing successfully downloaded image
|
|
*/
|
|
// TODO: split this into two functions, one to store the image, the other to change the item body
|
|
function retriever_transform_images(&$item, $resource) {
|
|
if (!$resource['data']) {
|
|
Logger::info('retriever_transform_images: no data available for ' . $resource['id'] . ' ' . $resource['url']);
|
|
return;
|
|
}
|
|
|
|
$uri_id = ItemURI::getIdByURI($item['uri']); //@@@ why can't I get this from the item itself?
|
|
|
|
$data = $resource['data'];
|
|
$type = $resource['type'];
|
|
$uid = $item['uid'];
|
|
$cid = $item['contact-id'];
|
|
$rid = Photo::newResource();
|
|
$path = parse_url($resource['url'], PHP_URL_PATH);
|
|
$parts = pathinfo($path);
|
|
$filename = $parts['filename'] . (array_key_exists('extension', $parts) ? '.' . $parts['extension'] : '');
|
|
$album = 'Wall Photos';
|
|
$scale = 0;
|
|
$desc = ''; // TODO: store alt text with resource when it's requested so we can fill this in
|
|
Logger::debug('retriever_transform_images storing ' . strlen($data) . ' bytes type ' . $type . ': uid ' . $uid . ' cid ' . $cid . ' rid ' . $rid . ' filename ' . $filename . ' album ' . $album . ' scale ' . $scale . ' desc ' . $desc);
|
|
$image = new Image($data, $type);
|
|
if (!$image->isValid()) {
|
|
Logger::warning('retriever_transform_images: invalid image found at URL ' . $resource['url'] . ' for item ' . $item['id']);
|
|
return;
|
|
}
|
|
$photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, 0, 0, "", "", "", "", $desc);
|
|
$new_url = System::baseUrl() . '/photo/' . $rid . '-0.' . $image->getExt();
|
|
if (!strlen($new_url)) {
|
|
Logger::warning('retriever_transform_images: no replacement URL for image ' . $resource['url']);
|
|
return;
|
|
}
|
|
|
|
$body = retriever_get_body($item);
|
|
|
|
Logger::debug('retriever_transform_images: replacing ' . $resource['url'] . ' with ' . $new_url . ' in item ' . $item['uri']);
|
|
$body = str_replace($resource["url"], $new_url, $body);
|
|
retriever_set_body($item, $body);
|
|
}
|
|
|
|
function retriever_content($a) {
|
|
if (!local_user()) {
|
|
$a->page['content'] .= "<p>Please log in</p>";
|
|
return;
|
|
}
|
|
if ($a->argv[1] === 'help') {
|
|
$feeds = DBA::selectToArray('contact', ['id', 'name', 'thumb'], ['uid' => local_user(), 'network' => 'feed']);
|
|
for ($i = 0; $i < count($feeds); ++$i) {
|
|
$feeds[$i]['url'] = $a->getBaseUrl() . '/retriever/' . $feeds[$i]['id'];
|
|
}
|
|
$template = Renderer::getMarkupTemplate('/help.tpl', 'addon/retriever/');
|
|
$a->page['content'] .= Renderer::replaceMacros($template, array(
|
|
'$config' => $a->getBaseUrl() . '/settings/addon',
|
|
'$feeds' => $feeds));
|
|
return;
|
|
}
|
|
if ($a->argv[1]) {
|
|
$retriever_rule = get_retriever_rule($a->argv[1], local_user(), false);
|
|
|
|
if (!empty($_POST["id"])) {
|
|
$retriever_rule = get_retriever_rule($a->argv[1], local_user(), true);
|
|
$retriever_rule['data'] = array();
|
|
foreach (array('modurl', 'pattern', 'replace', 'enable', 'images', 'customxslt', 'storecookies', 'cookiedata') as $setting) {
|
|
if (empty($_POST['retriever_' . $setting])) {
|
|
$retriever_rule['data'][$setting] = NULL;
|
|
}
|
|
else {
|
|
$retriever_rule['data'][$setting] = $_POST['retriever_' . $setting];
|
|
}
|
|
}
|
|
foreach ($_POST as $k=>$v) {
|
|
if (preg_match("/retriever-(include|exclude)-(\d+)-(element|attribute|value)/", $k, $matches)) {
|
|
$retriever_rule['data'][$matches[1]][intval($matches[2])][$matches[3]] = $v;
|
|
}
|
|
}
|
|
// You've gotta have an element, even if it's just "*"
|
|
foreach ($retriever_rule['data']['include'] as $k=>$clause) {
|
|
if (!$clause['element']) {
|
|
unset($retriever_rule['data']['include'][$k]);
|
|
}
|
|
}
|
|
foreach ($retriever_rule['data']['exclude'] as $k=>$clause) {
|
|
if (!$clause['element']) {
|
|
unset($retriever_rule['data']['exclude'][$k]);
|
|
}
|
|
}
|
|
DBA::update('retriever_rule', ['data' => json_encode($retriever_rule['data'])], ['id' => intval($retriever_rule["id"])], ['data' => '']);
|
|
$a->page['content'] .= "<p><b>Settings Updated";
|
|
if (!empty($_POST["retriever_retrospective"])) {
|
|
apply_retrospective($a, $retriever_rule, $_POST["retriever_retrospective"]);
|
|
$a->page['content'] .= " and retrospectively applied to " . $_POST["retriever_retrospective"] . " posts";
|
|
}
|
|
$a->page['content'] .= ".</p></b>";
|
|
}
|
|
|
|
$template = Renderer::getMarkupTemplate('/rule-config.tpl', 'addon/retriever/');
|
|
$a->page['content'] .= Renderer::replaceMacros($template, array(
|
|
'$enable' => array(
|
|
'retriever_enable',
|
|
L10n::t('Enabled'),
|
|
$retriever_rule['data']['enable']),
|
|
'$modurl' => array(
|
|
'retriever_modurl',
|
|
L10n::t('Modify URL'),
|
|
$retriever_rule['data']['modurl'],
|
|
L10n::t("Modify each article's URL with regular expressions before retrieving.")),
|
|
'$pattern' => array(
|
|
'retriever_pattern',
|
|
L10n::t('URL Pattern'),
|
|
$retriever_rule['data']['pattern'],
|
|
L10n::t('Regular expression matching part of the URL to replace')),
|
|
'$replace' => array(
|
|
'retriever_replace',
|
|
L10n::t('URL Replace'),
|
|
$retriever_rule['data']['replace'],
|
|
L10n::t('Text to replace matching part of above regular expression')),
|
|
'$images' => array(
|
|
'retriever_images',
|
|
L10n::t('Download Images'),
|
|
$retriever_rule['data']['images']),
|
|
'$retrospective' => array(
|
|
'retriever_retrospective',
|
|
L10n::t('Retrospectively Apply'),
|
|
'0',
|
|
L10n::t('Reapply the rules to this number of posts')),
|
|
'storecookies' => array(
|
|
'retriever_storecookies',
|
|
L10n::t('Store cookies'),
|
|
$retriever_rule['data']['storecookies'],
|
|
L10n::t("Preserve cookie data across fetches.")),
|
|
'$cookiedata' => array(
|
|
'retriever_cookiedata',
|
|
L10n::t('Cookie Data'),
|
|
$retriever_rule['data']['cookiedata'],
|
|
L10n::t("Latest cookie data for this feed. Netscape cookie file format.")),
|
|
'$customxslt' => array(
|
|
'retriever_customxslt',
|
|
L10n::t('Custom XSLT'),
|
|
$retriever_rule['data']['customxslt'],
|
|
L10n::t("When standard rules aren't enough, apply custom XSLT to the article")),
|
|
'$title' => L10n::t('Retrieve Feed Content'),
|
|
'$help' => $a->getBaseUrl() . '/retriever/help',
|
|
'$help_t' => L10n::t('Get Help'),
|
|
'$submit_t' => L10n::t('Submit'),
|
|
'$submit' => L10n::t('Save Settings'),
|
|
'$id' => ($retriever_rule["id"] ? $retriever_rule["id"] : "create"),
|
|
'$tag_t' => L10n::t('Tag'),
|
|
'$attribute_t' => L10n::t('Attribute'),
|
|
'$value_t' => L10n::t('Value'),
|
|
'$add_t' => L10n::t('Add'),
|
|
'$remove_t' => L10n::t('Remove'),
|
|
'$include_t' => L10n::t('Include'),
|
|
'$include' => $retriever_rule['data']['include'],
|
|
'$exclude_t' => L10n::t('Exclude'),
|
|
'$exclude' => $retriever_rule['data']['exclude']));
|
|
return;
|
|
}
|
|
}
|
|
|
|
function retriever_contact_photo_menu($a, &$args) {
|
|
if (!$args) {
|
|
return;
|
|
}
|
|
if ($args["contact"]["network"] == "feed") {
|
|
$args["menu"][ 'retriever' ] = array(L10n::t('Retriever'), $a->getBaseUrl() . '/retriever/' . $args["contact"]['id']);
|
|
}
|
|
}
|
|
|
|
function retriever_post_remote_hook(&$a, &$item) {
|
|
// @@@ I believe this should either never have the id, or always should. This needs more investigation.
|
|
// @@@ and if it does not, does it have a content row?
|
|
Logger::debug('@@@ retriever_post_remote_hook uri ' . $item['uri'] . ' has id ' . array_key_exists('id', $item) . ' has uri-id ' . array_key_exists('uri-id', $item));
|
|
|
|
Logger::info('retriever_post_remote_hook: ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
|
|
|
|
$uri_id = ItemURI::getIdByURI($item['uri']); //@@@ why can't I get this from the item itself?
|
|
$retriever_rule = get_retriever_rule($item['contact-id'], $item["uid"], false);
|
|
if ($retriever_rule) {
|
|
retriever_on_item_insert($retriever_rule, $item);
|
|
}
|
|
else {
|
|
if (PConfig::get($item["uid"], 'retriever', 'oembed')) {
|
|
// Convert to HTML and back to take advantage of bbcode's resolution of oembeds.
|
|
$body = retriever_get_body($item);
|
|
$body = HTML::toBBCode(BBCode::convert($body));
|
|
retriever_set_body($item, $body);
|
|
}
|
|
if (PConfig::get($item["uid"], 'retriever', 'all_photos')) {
|
|
retrieve_images($item);
|
|
}
|
|
}
|
|
retriever_check_item_completed($item);
|
|
}
|
|
|
|
function retriever_plugin_settings(&$a,&$s) {
|
|
$all_photos = PConfig::get(local_user(), 'retriever', 'all_photos');
|
|
$oembed = PConfig::get(local_user(), 'retriever', 'oembed');
|
|
$template = Renderer::getMarkupTemplate('/settings.tpl', 'addon/retriever/');
|
|
$s .= Renderer::replaceMacros($template, array(
|
|
'$allphotos' => array(
|
|
'retriever_all_photos',
|
|
L10n::t('All Photos'),
|
|
$all_photos,
|
|
L10n::t('Check this to retrieve photos for all posts')),
|
|
'$oembed' => array(
|
|
'retriever_oembed',
|
|
L10n::t('Resolve OEmbed'),
|
|
$oembed,
|
|
L10n::t('Check this to attempt to retrieve embedded content for all posts - useful e.g. for Facebook posts')),
|
|
'$submit' => L10n::t('Save Settings'),
|
|
'$title' => L10n::t('Retriever Settings'),
|
|
'$help' => $a->getBaseUrl() . '/retriever/help'));
|
|
}
|
|
|
|
function retriever_plugin_settings_post($a,$post) {
|
|
if ($_POST['retriever_all_photos']) {
|
|
PConfig::set(local_user(), 'retriever', 'all_photos', $_POST['retriever_all_photos']);
|
|
}
|
|
else {
|
|
PConfig::del(local_user(), 'retriever', 'all_photos');
|
|
}
|
|
if ($_POST['retriever_oembed']) {
|
|
PConfig::set(local_user(), 'retriever', 'oembed', $_POST['retriever_oembed']);
|
|
}
|
|
else {
|
|
PConfig::del(local_user(), 'retriever', 'oembed');
|
|
}
|
|
}
|