friendica
/
friendica
mirror of https://github.com/friendica/friendica.git
1
2
Fork 1
Code Issues Releases 49 Wiki Activity
Browse Source

Speed up of OStatus transmission/Ignoring "created" with remote self

pull/3613/head
Michael 3 years ago
parent
390e34d587
commit
a80329b416
4 changed files with 100 additions and 85 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +8
    -5
      include/items.php
  2. +77
    -72
      include/ostatus.php
  3. +5
    -3
      include/poller.php
  4. +10
    -5
      include/pubsubpublish.php

+ 8
- 5
include/items.php View File

@ -623,11 +623,11 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
$arr['owner-name'] = ((x($arr, 'owner-name')) ? trim($arr['owner-name']) : '');
$arr['owner-link'] = ((x($arr, 'owner-link')) ? notags(trim($arr['owner-link'])) : '');
$arr['owner-avatar'] = ((x($arr, 'owner-avatar')) ? notags(trim($arr['owner-avatar'])) : '');
$arr['created'] = ((x($arr, 'created') !== false) ? datetime_convert('UTC','UTC', $arr['created']) : datetime_convert());
$arr['edited'] = ((x($arr, 'edited') !== false) ? datetime_convert('UTC','UTC', $arr['edited']) : datetime_convert());
$arr['commented'] = ((x($arr, 'commented') !== false) ? datetime_convert('UTC','UTC', $arr['commented']) : datetime_convert());
$arr['received'] = ((x($arr, 'received') !== false) ? datetime_convert('UTC','UTC', $arr['received']) : datetime_convert());
$arr['changed'] = ((x($arr, 'changed') !== false) ? datetime_convert('UTC','UTC', $arr['changed']) : datetime_convert());
$arr['received'] = ((x($arr, 'received') !== false) ? datetime_convert('UTC','UTC', $arr['received']) : datetime_convert());
$arr['created'] = ((x($arr, 'created') !== false) ? datetime_convert('UTC','UTC', $arr['created']) : $arr['received']);
$arr['edited'] = ((x($arr, 'edited') !== false) ? datetime_convert('UTC','UTC', $arr['edited']) : $arr['created']);
$arr['changed'] = ((x($arr, 'changed') !== false) ? datetime_convert('UTC','UTC', $arr['changed']) : $arr['created']);
$arr['commented'] = ((x($arr, 'commented') !== false) ? datetime_convert('UTC','UTC', $arr['commented']) : $arr['created']);
$arr['title'] = ((x($arr, 'title')) ? trim($arr['title']) : '');
$arr['location'] = ((x($arr, 'location')) ? trim($arr['location']) : '');
$arr['coord'] = ((x($arr, 'coord')) ? notags(trim($arr['coord'])) : '');
@ -1640,6 +1640,9 @@ function item_is_remote_self($contact, &$datarray) {
$datarray['author-name'] = $datarray['owner-name'];
$datarray['author-link'] = $datarray['owner-link'];
$datarray['author-avatar'] = $datarray['owner-avatar'];
unset($datarray['created']);
unset($datarray['edited']);
}
if ($contact['network'] != NETWORK_FEED) {


+ 77
- 72
include/ostatus.php View File

@ -21,6 +21,7 @@ require_once 'include/follow.php';
require_once 'include/api.php';
require_once 'mod/proxy.php';
require_once 'include/xml.php';
require_once 'include/cache.php';
/**
* @brief This class contain functions for the OStatus protocol
@ -195,7 +196,7 @@ class ostatus {
link_gcontact($gcid, $contact["uid"], $contact["id"]);
}
return($author);
return $author;
}
/**
@ -677,14 +678,14 @@ class ostatus {
if ((count($elements) == 4) && ($elements[2] == "post"))
return "http://".$server[0]."/notice/".$elements[3];
if ((count($conversation) != 2) || ($conversation[1] ==""))
if ((count($conversation) != 2) || ($conversation[1] =="")) {
return $href;
if ($elements[3] == "objectType=thread")
}
if ($elements[3] == "objectType=thread") {
return "http://".$server[0]."/conversation/".$conversation[1];
else
} else {
return "http://".$server[0]."/notice/".$conversation[1];
}
return $href;
}
@ -946,9 +947,9 @@ class ostatus {
STRAIGHT_JOIN `item` ON `item`.`parent` = `thritem`.`parent`
WHERE `term`.`uid` = %d AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`url` = '%s'",
intval($uid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($conversation_url));
if ($parents)
if ($parents) {
$parent = $parents[0];
elseif (count($item) > 0) {
} elseif (count($item) > 0) {
$parent = $item;
$parent["type"] = "remote";
$parent["verb"] = ACTIVITY_POST;
@ -956,9 +957,9 @@ class ostatus {
} else {
// Preset the parent
$r = q("SELECT `id` FROM `contact` WHERE `self` AND `uid`=%d", $uid);
if (!$r)
return(-2);
if (!dbm::is_result($r)) {
return -2;
}
$parent = array();
$parent["id"] = 0;
$parent["parent"] = 0;
@ -985,9 +986,9 @@ class ostatus {
} elseif (!$conv_arr["success"] && (substr($conv, 0, 8) == "https://")) {
$conv = str_replace("https://", "http://", $conv);
$conv_as = fetch_url($conv."?page=".$pageno);
} else
} else {
$conv_as = $conv_arr["body"];
}
$conv_as = str_replace(',"statusnet:notice_info":', ',"statusnet_notice_info":', $conv_as);
$conv_as = json_decode($conv_as);
@ -1015,9 +1016,10 @@ class ostatus {
self::store_conversation($item_id, $conversation_url);
}
return($item_stored);
} else
return(-3);
return $item_stored;
} else {
return -3;
}
}
$items = array_reverse($items);
@ -1153,7 +1155,7 @@ class ostatus {
}
if (is_array($single_conv->to))
foreach($single_conv->to AS $to)
foreach ($single_conv->to AS $to)
if ($importer["nurl"] == normalise_link($to->id))
$mention = true;
@ -1197,17 +1199,17 @@ class ostatus {
$arr['conversation-uri'] = $single_conv->status_net->conversation;
}
if (isset($single_conv->status_net->notice_info->source))
if (isset($single_conv->status_net->notice_info->source)) {
$arr["app"] = strip_tags($single_conv->status_net->notice_info->source);
elseif (isset($single_conv->statusnet->notice_info->source))
} elseif (isset($single_conv->statusnet->notice_info->source)) {
$arr["app"] = strip_tags($single_conv->statusnet->notice_info->source);
elseif (isset($single_conv->statusnet_notice_info->source))
} elseif (isset($single_conv->statusnet_notice_info->source)) {
$arr["app"] = strip_tags($single_conv->statusnet_notice_info->source);
elseif (isset($single_conv->provider->displayName))
} elseif (isset($single_conv->provider->displayName)) {
$arr["app"] = $single_conv->provider->displayName;
else
} else {
$arr["app"] = "OStatus";
}
$arr["source"] = json_encode($single_conv);
$arr["protocol"] = PROTOCOL_GS_CONVERSATION;
@ -1324,7 +1326,7 @@ class ostatus {
}
}
return($item_stored);
return $item_stored;
}
/**
@ -1367,16 +1369,16 @@ class ostatus {
// Skip if it isn't a pure repeated messages
// Does it start with a share?
if (strpos($body, "[share") > 0)
return("");
return "";
// Does it end with a share?
if (strlen($body) > (strrpos($body, "[/share]") + 8))
return("");
return "";
$attributes = preg_replace("/\[share(.*?)\]\s?(.*?)\s?\[\/share\]\s?/ism","$1",$body);
// Skip if there is no shared message in there
if ($body == $attributes)
return(false);
return false;
$guid = "";
preg_match("/guid='(.*?)'/ism", $attributes, $matches);
@ -1496,12 +1498,12 @@ class ostatus {
$hub = get_config('system','huburl');
$hubxml = '';
if(strlen($hub)) {
if (strlen($hub)) {
$hubs = explode(',', $hub);
if(count($hubs)) {
foreach($hubs as $h) {
if (count($hubs)) {
foreach ($hubs as $h) {
$h = trim($h);
if(! strlen($h))
if (! strlen($h))
continue;
if ($h === '[internal]')
$h = App::get_baseurl() . '/pubsubhubbub';
@ -1543,7 +1545,7 @@ class ostatus {
break;
}
if (($siteinfo["type"] != "photo") && isset($siteinfo["image"])) {
if (!Config::get('system', 'ostatus_not_attach_preview') && ($siteinfo["type"] != "photo") && isset($siteinfo["image"])) {
$imgdata = get_photo_info($siteinfo["image"]);
$attributes = array("rel" => "enclosure",
"href" => $siteinfo["image"],
@ -1586,9 +1588,9 @@ class ostatus {
private function add_author($doc, $owner) {
$r = q("SELECT `homepage`, `publish` FROM `profile` WHERE `uid` = %d AND `is-default` LIMIT 1", intval($owner["uid"]));
if ($r)
if (dbm::is_result($r)) {
$profile = $r[0];
}
$author = $doc->createElement("author");
xml::add_element($doc, $author, "id", $owner["url"]);
xml::add_element($doc, $author, "activity:object-type", ACTIVITY_OBJ_PERSON);
@ -1743,22 +1745,22 @@ class ostatus {
$r = q("SELECT * FROM `contact` WHERE `nurl` = '%s' AND `uid` IN (0, %d) ORDER BY `uid` DESC LIMIT 1",
dbesc(normalise_link($url)), intval($owner["uid"]));
if ($r) {
if (dbm::is_result($r)) {
$contact = $r[0];
$contact["uid"] = -1;
}
if (!$r) {
if (!dbm::is_result($r)) {
$r = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
dbesc(normalise_link($url)));
if ($r) {
if (dbm::is_result($r)) {
$contact = $r[0];
$contact["uid"] = -1;
$contact["success_update"] = $contact["updated"];
}
}
if (!$r)
if (!dbm::is_result($r))
$contact = owner;
if (!isset($contact["poll"])) {
@ -1797,11 +1799,11 @@ class ostatus {
$r = q("SELECT * FROM `item` WHERE `uid` = %d AND `guid` = '%s' AND NOT `private` AND `network` IN ('%s', '%s', '%s') LIMIT 1",
intval($owner["uid"]), dbesc($repeated_guid),
dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
if ($r)
if (dbm::is_result($r)) {
$repeated_item = $r[0];
else
} else {
return false;
}
$contact = self::contact_entry($repeated_item['author-link'], $owner);
$parent_item = (($item['thr-parent']) ? $item['thr-parent'] : $item['parent-uri']);
@ -2158,8 +2160,8 @@ class ostatus {
$tags = item_getfeedtags($item);
if(count($tags))
foreach($tags as $t)
if (count($tags))
foreach ($tags as $t)
if ($t[0] == "@")
$mentioned[$t[1]] = $t[1];
@ -2194,8 +2196,8 @@ class ostatus {
xml::add_element($doc, $entry, "mastodon:scope", "public");
}
if(count($tags))
foreach($tags as $t)
if (count($tags))
foreach ($tags as $t)
if ($t[0] != "@")
xml::add_element($doc, $entry, "category", "", array("term" => $t[2]));
@ -2227,19 +2229,33 @@ class ostatus {
*
* @return string XML feed
*/
public static function feed(App $a, $owner_nick, $last_update) {
public static function feed(App $a, $owner_nick, &$last_update) {
$stamp = microtime(true);
$cachekey = "ostatus:feed:".$owner_nick.":".$last_update;
$previous_created = $last_update;
$result = Cache::get($cachekey);
if (!is_null($result)) {
logger('Feed duration: '.number_format(microtime(true) - $stamp, 3).' - '.$owner_nick.' - '.$previous_created.' (cached)', LOGGER_DEBUG);
$last_update = $result['last_update'];
return $result['feed'];
}
$r = q("SELECT `contact`.*, `user`.`nickname`, `user`.`timezone`, `user`.`page-flags`
FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid`
WHERE `contact`.`self` AND `user`.`nickname` = '%s' LIMIT 1",
dbesc($owner_nick));
if (!$r)
if (!dbm::is_result($r)) {
return;
}
$owner = $r[0];
if (!strlen($last_update))
if (!strlen($last_update)) {
$last_update = 'now -30 days';
}
$check_date = datetime_convert('UTC','UTC',$last_update,'Y-m-d H:i:s');
$authorid = get_contact($owner["url"], 0);
@ -2255,28 +2271,6 @@ class ostatus {
intval($authorid), dbesc($check_date),
dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN));
/* 2016-10-23: The old query will be kept until we are sure that the query above is a good and fast replacement
$items = q("SELECT `item`.*, `item`.`id` AS `item_id` FROM `item`
STRAIGHT_JOIN `thread` ON `thread`.`iid` = `item`.`parent`
LEFT JOIN `item` AS `thritem` ON `thritem`.`uri`=`item`.`thr-parent` AND `thritem`.`uid`=`item`.`uid`
WHERE `item`.`uid` = %d AND `item`.`received` > '%s' AND NOT `item`.`private` AND NOT `item`.`deleted`
AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = ''
AND ((`item`.`wall` AND (`item`.`parent` = `item`.`id`))
OR (`item`.`network` = '%s' AND ((`thread`.`network` IN ('%s', '%s')) OR (`thritem`.`network` IN ('%s', '%s')))) AND `thread`.`mention`)
AND ((`item`.`owner-link` IN ('%s', '%s') AND (`item`.`parent` = `item`.`id`))
OR (`item`.`author-link` IN ('%s', '%s')))
ORDER BY `item`.`id` DESC
LIMIT 0, 300",
intval($owner["uid"]), dbesc($check_date), dbesc(NETWORK_DFRN),
//dbesc(NETWORK_OSTATUS), dbesc(NETWORK_OSTATUS),
//dbesc(NETWORK_OSTATUS), dbesc(NETWORK_OSTATUS),
dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN),
dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN),
dbesc($owner["nurl"]), dbesc(str_replace("http://", "https://", $owner["nurl"])),
dbesc($owner["nurl"]), dbesc(str_replace("http://", "https://", $owner["nurl"]))
);
*/
$doc = new DOMDocument('1.0', 'utf-8');
$doc->formatOutput = true;
@ -2288,9 +2282,20 @@ class ostatus {
}
$entry = self::entry($doc, $item, $owner);
$root->appendChild($entry);
if ($last_update < $item['created']) {
$last_update = $item['created'];
}
}
return(trim($doc->saveXML()));
$feeddata = trim($doc->saveXML());
$msg = array('feed' => $feeddata, 'last_update' => $last_update);
Cache::set($cachekey, $msg, CACHE_QUARTER_HOUR);
logger('Feed duration: '.number_format(microtime(true) - $stamp, 3).' - '.$owner_nick.' - '.$previous_created, LOGGER_DEBUG);
return $feeddata;
}
/**
@ -2314,6 +2319,6 @@ class ostatus {
$doc->appendChild($entry);
return(trim($doc->saveXML()));
return trim($doc->saveXML());
}
}

+ 5
- 3
include/poller.php View File

@ -273,7 +273,9 @@ function poller_exec_function($queue, $funcname, $argv) {
$argc = count($argv);
logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]);
$new_process_id = uniqid("wrk", true);
logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." ".$queue["parameter"]." - Process PID: ".$new_process_id);
$stamp = (float)microtime(true);
@ -295,7 +297,7 @@ function poller_exec_function($queue, $funcname, $argv) {
// For better logging create a new process id for every worker call
// But preserve the old one for the worker
$old_process_id = $a->process_id;
$a->process_id = uniqid("wrk", true);
$a->process_id = $new_process_id;
$a->queue = $queue;
$up_duration = number_format(microtime(true) - $poller_up_start, 3);
@ -330,7 +332,7 @@ function poller_exec_function($queue, $funcname, $argv) {
logger("Prio ".$queue["priority"].": ".$queue["parameter"]." - longer than 2 minutes (".round($duration/60, 3).")", LOGGER_DEBUG);
}
logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds.");
logger("Process ".$mypid." - Prio ".$queue["priority"]." - ID ".$queue["id"].": ".$funcname." - done in ".$duration." seconds. Process PID: ".$new_process_id);
// Write down the performance values into the log
if (Config::get("system", "profiler")) {


+ 10
- 5
include/pubsubpublish.php View File

@ -14,7 +14,7 @@ function pubsubpublish_run(&$argv, &$argc){
} else {
// We'll push to each subscriber that has push > 0,
// i.e. there has been an update (set in notifier.php).
$r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0");
$r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0 ORDER BY `last_update` DESC");
foreach ($r as $rr) {
logger("Publish feed to ".$rr["callback_url"], LOGGER_DEBUG);
@ -43,7 +43,13 @@ function handle_pubsubhubbub($id) {
logger("Generate feed of user ".$rr['nickname']." to ".$rr['callback_url']." - last updated ".$rr['last_update'], LOGGER_DEBUG);
$params = ostatus::feed($a, $rr['nickname'], $rr['last_update']);
$last_update = $rr['last_update'];
$params = ostatus::feed($a, $rr['nickname'], $last_update);
if (!$params) {
return;
}
$hmac_sig = hash_hmac("sha1", $params, $rr['secret']);
$headers = array("Content-type: application/atom+xml",
@ -60,10 +66,9 @@ function handle_pubsubhubbub($id) {
if ($ret >= 200 && $ret <= 299) {
logger('successfully pushed to '.$rr['callback_url']);
// set last_update to "now", and reset push=0
$date_now = datetime_convert('UTC','UTC','now','Y-m-d H:i:s');
// set last_update to the "created" date of the last item, and reset push=0
q("UPDATE `push_subscriber` SET `push` = 0, last_update = '%s' WHERE id = %d",
dbesc($date_now),
dbesc($last_update),
intval($rr['id']));
} else {


Write Preview
Loading…
Cancel
Save