MrPetovan/friendica
2
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

New fields for the term table, improved query for the tag search. Changed the cache handling for rendered bbcode.

Browse source
This commit is contained in:
Michael Vogel 2015-03-07 21:24:39 +01:00
parent 67d39770ed
commit 7c4a1a059d
11 changed files with 233 additions and 218 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
boot.php
View file

@ -18,7 +18,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica');
define ( 'FRIENDICA_CODENAME', 'Ginger');
define ( 'FRIENDICA_VERSION', '3.3.3' );
define ( 'DFRN_PROTOCOL_VERSION', '2.23' );
define ( 'DB_UPDATE_VERSION', 1180 );
define ( 'DB_UPDATE_VERSION', 1181 );
define ( 'EOL', "<br />\r\n" );
define ( 'ATOM_TIME', 'Y-m-d\TH:i:s\Z' );

7
include/dbstructure.php
View file

@ -776,6 +776,8 @@ function db_definition() {
"last-child" => array("type" => "tinyint(1) unsigned", "not null" => "1", "default" => "1"),
"mention" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"),
"network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
"rendered-hash" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
"rendered-html" => array("type" => "mediumtext", "not null" => "1"),
),
"indexes" => array(
"PRIMARY" => array("id"),
@ -1188,6 +1190,9 @@ function db_definition() {
"type" => array("type" => "tinyint(3) unsigned", "not null" => "1", "default" => "0"),
"term" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"guid" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
"received" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
"aid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"),
"uid" => array("type" => "int(10) unsigned", "not null" => "1", "default" => "0"),
),
@ -1196,7 +1201,7 @@ function db_definition() {
"oid_otype_type_term" => array("oid","otype","type","term"),
"uid_term_tid" => array("uid","term","tid"),
"type_term" => array("type","term"),
"uid_otype_type_term_tid" => array("uid","otype","type","term","tid"),
"uid_otype_type_term_created" => array("uid","otype","type","term","created"),
"otype_type_term_tid" => array("otype","type","term","tid"),
)
);

6
include/items.php
View file

@ -1346,6 +1346,9 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
return 0;
}
// Fill the cache field
put_item_in_cache($arr);
call_hooks('post_remote',$arr);
if(x($arr,'cancel')) {
@ -1478,9 +1481,6 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
// in it.
if (!$deleted AND !$dontcache) {
// Store the fresh generated item into the cache
put_item_in_cache($arr);
$r = q('SELECT * FROM `item` WHERE id = %d', intval($current_post));
if (count($r) == 1) {
call_hooks('post_remote_end', $r[0]);

7
include/network.php
View file

@ -870,13 +870,6 @@ function scale_external_images($srctext, $include_link = true, $scale_replace =
if(! $i)
return $srctext;
$cachefile = get_cachefile(hash("md5", $scaled));
if ($cachefile != '') {
$stamp1 = microtime(true);
file_put_contents($cachefile, $i);
$a->save_timestamp($stamp1, "file");
}
// guess mimetype from headers or filename
$type = guess_image_type($mtch[1],true);

7
include/ostatus_conversation.php
View file

@ -52,7 +52,7 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
$a->last_ostatus_conversation_url = $conversation_url;
$messages = q("SELECT `uid`, `parent`, `created` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
$messages = q("SELECT `uid`, `parent`, `created`, `received`, `guid` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
if (!$messages)
return;
$message = $messages[0];
@ -62,8 +62,9 @@ function complete_conversation($itemid, $conversation_url, $only_add_conversatio
intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION));
if (!$conversation) {
$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`) VALUES (%d, %d, %d, %d, '%s', '%s')",
intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($message["created"]), dbesc($conversation_url));
$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `created`, `received`, `guid`) VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')",
intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION),
dbesc($message["created"]), dbesc($conversation_url), dbesc($message["created"]), dbesc($message["received"]), dbesc($message["guid"]));
logger('complete_conversation: Storing conversation url '.$conversation_url.' for id '.$itemid);
}

23
include/tags.php
View file

@ -9,7 +9,7 @@ function create_tags_from_item($itemid) {
$searchpath = $a->get_baseurl()."/search?tag=";
$messages = q("SELECT `guid`, `uid`, `id`, `edited`, `deleted`, `title`, `body`, `tag`, `parent` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
$messages = q("SELECT `guid`, `uid`, `id`, `edited`, `deleted`, `created`, `received`, `title`, `body`, `tag`, `parent` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
if (!$messages)
return;
@ -69,8 +69,10 @@ function create_tags_from_item($itemid) {
$term = $tag;
}
$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`) VALUES (%d, %d, %d, %d, '%s', '%s')",
intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type), dbesc($term), dbesc($link));
$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`, `guid`, `created`, `received`)
VALUES (%d, %d, %d, %d, '%s', '%s', '%s', '%s', '%s')",
intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type),
dbesc($term), dbesc($link), dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"]));
// Search for mentions
if ((substr($tag, 0, 1) == '@') AND (strpos($link, $profile_base_friendica) OR strpos($link, $profile_base_diaspora))) {
@ -96,10 +98,17 @@ function create_tags_from_itemuri($itemuri, $uid) {
}
function update_items() {
//$messages = q("SELECT `id` FROM `item` where tag !='' ORDER BY `created` DESC limit 10");
$messages = q("SELECT `id` FROM `item` where tag !=''");
global $db;
foreach ($messages as $message)
create_tags_from_item($message["id"]);
$messages = $db->q("SELECT `oid`,`item`.`guid`, `item`.`created`, `item`.`received` FROM `term` INNER JOIN `item` ON `item`.`id`=`term`.`oid` WHERE `term`.`otype` = 1 AND `term`.`guid` = ''", true);
logger("fetched messages: ".count($messages));
while ($message = $db->qfetch()) {
q("UPDATE `term` SET `guid` = '%s', `created` = '%s', `received` = '%s' WHERE `otype` = %d AND `oid` = %d",
dbesc($message["guid"]), dbesc($message["created"]), dbesc($message["received"]),
intval(TERM_OBJ_POST), intval($message["oid"]));
}
$db->qclose();
}
?>

22
include/tagupdate.php Normal file
View file

@ -0,0 +1,22 @@
<?php
require_once("boot.php");
require_once("include/tags.php");
global $a, $db;
if(is_null($a))
$a = new App;
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
}
load_config('config');
load_config('system');
update_items();
killme();
?>

50
include/text.php
View file

@ -1296,16 +1296,26 @@ function redir_private_images($a, &$item) {
}}
function put_item_in_cache($item) {
$cachefile = get_cachefile(urlencode($item["guid"])."-".hash("md5", $item['body']));
function put_item_in_cache(&$item, $update = false) {
if (($item["rendered-hash"] != hash("md5", $item["body"])) OR ($item["rendered-hash"] == "") OR
($item["rendered-html"] == "") OR get_config("system", "ignore_cache")) {
// The function "redir_private_images" changes the body.
// I'm not sure if we should store it permanently, so we save the old value.
$body = $item["body"];
if (($cachefile != '') AND !file_exists($cachefile)) {
$s = prepare_text($item['body']);
$a = get_app();
$stamp1 = microtime(true);
file_put_contents($cachefile, $s);
$a->save_timestamp($stamp1, "file");
logger('put item '.$item["guid"].' into cachefile '.$cachefile);
redir_private_images($a, $item);
$item["rendered-html"] = prepare_text($item["body"]);
$item["rendered-hash"] = hash("md5", $item["body"]);
$item["body"] = $body;
if ($update AND ($item["id"] != 0)) {
q("UPDATE `item` SET `rendered-html` = '%s', `rendered-hash` = '%s' WHERE `id` = %d",
dbesc($item["rendered-html"]), dbesc($item["rendered-hash"]), intval($item["id"]));
}
}
}
@ -1359,28 +1369,8 @@ function prepare_body(&$item,$attach = false, $preview = false) {
$item['hashtags'] = $hashtags;
$item['mentions'] = $mentions;
$cachefile = get_cachefile(urlencode($item["guid"])."-".hash("md5", $item['body']));
if (($cachefile != '')) {
if (file_exists($cachefile)) {
$stamp1 = microtime(true);
$s = file_get_contents($cachefile);
$a->save_timestamp($stamp1, "file");
} else {
redir_private_images($a, $item);
$s = prepare_text($item['body']);
$stamp1 = microtime(true);
file_put_contents($cachefile, $s);
$a->save_timestamp($stamp1, "file");
logger('prepare_body: put item '.$item["id"].' into cachefile '.$cachefile);
}
} else {
redir_private_images($a, $item);
$s = prepare_text($item['body']);
}
put_item_in_cache($item, true);
$s = $item["rendered-html"];
require_once("mod/proxy.php");
$s = proxy_parse_html($s);

42
mod/item.php
View file

@ -746,14 +746,18 @@ function item_post(&$a) {
killme();
}
// Fill the cache field
put_item_in_cache($datarray);
if($orig_post) {
$r = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `tag` = '%s', `attach` = '%s', `file` = '%s', `edited` = '%s', `changed` = '%s' WHERE `id` = %d AND `uid` = %d",
$r = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `tag` = '%s', `attach` = '%s', `file` = '%s', `rendered-html` = '%s', `rendered-hash` = '%s', `edited` = '%s', `changed` = '%s' WHERE `id` = %d AND `uid` = %d",
dbesc($datarray['title']),
dbesc($datarray['body']),
dbesc($datarray['tag']),
dbesc($datarray['attach']),
dbesc($datarray['file']),
dbesc($datarray['rendered-html']),
dbesc($datarray['rendered-hash']),
dbesc(datetime_convert()),
dbesc(datetime_convert()),
intval($post_id),
@ -778,10 +782,10 @@ function item_post(&$a) {
$post_id = 0;
$r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`,`owner-name`,`owner-link`,`owner-avatar`,
`author-name`, `author-link`, `author-avatar`, `created`, `edited`, `commented`, `received`, `changed`, `uri`, `thr-parent`, `title`, `body`, `app`, `location`, `coord`,
`tag`, `inform`, `verb`, `object-type`, `postopts`, `allow_cid`, `allow_gid`, `deny_cid`, `deny_gid`, `private`, `pubmail`, `attach`, `bookmark`,`origin`, `moderated`, `file` )
VALUES( '%s', '%s', %d, '%s', %d, %d, '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s', %d, %d, %d, '%s' )",
$r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`,`owner-name`,`owner-link`,`owner-avatar`, `author-name`, `author-link`, `author-avatar`,
`created`, `edited`, `commented`, `received`, `changed`, `uri`, `thr-parent`, `title`, `body`, `app`, `location`, `coord`, `tag`, `inform`, `verb`, `object-type`, `postopts`,
`allow_cid`, `allow_gid`, `deny_cid`, `deny_gid`, `private`, `pubmail`, `attach`, `bookmark`,`origin`, `moderated`, `file`, `rendered-html`, `rendered-hash`)
VALUES( '%s', '%s', %d, '%s', %d, %d, '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s', %d, %d, %d, '%s', '%s', '%s')",
dbesc($datarray['guid']),
dbesc($datarray['extid']),
intval($datarray['uid']),
@ -823,21 +827,29 @@ function item_post(&$a) {
intval($datarray['bookmark']),
intval($datarray['origin']),
intval($datarray['moderated']),
dbesc($datarray['file'])
dbesc($datarray['file']),
dbesc($datarray['rendered-html']),
dbesc($datarray['rendered-hash'])
);
$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' LIMIT 1",
dbesc($datarray['uri']));
if(count($r)) {
if(!count($r)) {
logger('mod_item: unable to retrieve post that was just stored.');
notice( t('System error. Post not saved.') . EOL);
goaway($a->get_baseurl() . "/" . $return_path );
// NOTREACHED
}
$post_id = $r[0]['id'];
logger('mod_item: saved item ' . $post_id);
$datarray["id"] = $post_id;
$datarray["plink"] = $a->get_baseurl().'/display/'.urlencode($datarray["guid"]);
// update filetags in pconfig
file_tag_update_pconfig($uid,$categories_old,$categories_new,'category');
// Store the fresh generated item into the cache
put_item_in_cache($datarray);
if($parent) {
// This item is the last leaf and gets the comment box, clear any ancestors
@ -930,13 +942,6 @@ function item_post(&$a) {
);
update_thread($parent_item['id']);
}
}
else {
logger('mod_item: unable to retrieve post that was just stored.');
notice( t('System error. Post not saved.') . EOL);
goaway($a->get_baseurl() . "/" . $return_path );
// NOTREACHED
}
// update the commented timestamp on the parent
@ -948,9 +953,6 @@ function item_post(&$a) {
if ($post_id != $parent)
update_thread($parent);
$datarray['id'] = $post_id;
$datarray['plink'] = $a->get_baseurl().'/display/'.urlencode($datarray['guid']);
call_hooks('post_local_end', $datarray);
if(strlen($emailcc) && $profile_uid == local_user()) {

77
mod/search.php
View file

@ -127,66 +127,48 @@ function search_content(&$a) {
if (get_config('system','only_tag_search'))
$tag = true;
if($tag) {
$sql_extra = "";
$sql_table = sprintf("`item` INNER JOIN (SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` IN (%d, 0)) AS `term` ON `item`.`id` = `term`.`oid` ",
dbesc(protect_sprintf($search)), intval(TERM_OBJ_POST), intval(TERM_HASHTAG), intval(local_user()));
$sql_order = "`item`.`id`";
} else {
if (get_config('system','use_fulltext_engine')) {
$sql_extra = sprintf(" AND MATCH (`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode) ", dbesc(protect_sprintf($search)));
} else {
$sql_extra = sprintf(" AND `item`.`body` REGEXP '%s' ", dbesc(protect_sprintf(preg_quote($search))));
}
$sql_table = "`item`";
$sql_order = "`item`.`id`";
//$sql_order = "`item`.`received`";
}
// Here is the way permissions work in the search module...
// Only public posts can be shown
// OR your own posts if you are a logged in member
// No items will be shown if the member has a blocked profile wall.
if(get_config('system', 'old_pager')) {
$r = q("SELECT distinct(`item`.`uri`) as `total`
FROM $sql_table INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
AND ((`item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' AND `item`.`private` = 0 AND `item`.`uid` = 0)
OR (`item`.`uid` = %d))
$sql_extra ",
intval(local_user())
);
if($tag) {
logger("Start tag search for '".$search."'");
if(count($r))
$a->set_pager_total(count($r));
$r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`,
`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`,
`contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`,
`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`
FROM `term`
INNER JOIN `item` ON `item`.`id`=`term`.`oid`
INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`
WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
AND `term`.`uid` IN (%d,0) AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`term` = '%s' AND `term`.`guid` != ''
GROUP BY `term`.`guid` ORDER BY term.created DESC LIMIT %d , %d ",
intval(local_user()), intval(TERM_OBJ_POST), intval(TERM_HASHTAG), dbesc(protect_sprintf($search)),
intval($a->pager['start']), intval($a->pager['itemspage']));
} else {
logger("Start fulltext search for '".$search."'");
if(! count($r)) {
info( t('No results.') . EOL);
return $o;
}
if (get_config('system','use_fulltext_engine')) {
$sql_extra = sprintf(" AND MATCH (`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode) ", dbesc(protect_sprintf($search)));
} else {
$sql_extra = sprintf(" AND `item`.`body` REGEXP '%s' ", dbesc(protect_sprintf(preg_quote($search))));
}
$r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`,
`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`,
`contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`,
`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`
FROM $sql_table INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
AND ((`item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' AND `item`.`private` = 0 AND `item`.`uid`=0)
OR `item`.`uid` = %d)
FROM `item`
INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id` AND NOT `contact`.`blocked` AND NOT `contact`.`pending`
WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
AND (`item`.`uid` = 0 OR (`item`.`uid` = %s AND (`item`.`private` OR NOT `item`.`network` IN ('%s', '%s', '%s'))))
$sql_extra
GROUP BY `item`.`uri`
ORDER BY $sql_order DESC LIMIT %d , %d ",
intval(local_user()),
intval($a->pager['start']),
intval($a->pager['itemspage'])
);
GROUP BY `item`.`uri` ORDER BY `item`.`id` DESC LIMIT %d , %d ",
intval(local_user()), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA),
intval($a->pager['start']), intval($a->pager['itemspage']));
}
if(! count($r)) {
info( t('No results.') . EOL);
@ -199,13 +181,16 @@ function search_content(&$a) {
else
$o .= '<h2>Search results for: ' . $search . '</h2>';
logger("Start Conversation");
$o .= conversation($a,$r,'search',false);
logger("Start Pager");
if(!get_config('system', 'old_pager')) {
$o .= alt_pager($a,count($r));
} else {
$o .= paginate($a);
}
logger("Done");
return $o;
}

10
update.php
View file

@ -1,6 +1,6 @@
<?php
define( 'UPDATE_VERSION' , 1180 );
define( 'UPDATE_VERSION' , 1181 );
/**
*
@ -1640,3 +1640,11 @@ function update_1178() {
return UPDATE_SUCCESS;
}
function update_1180() {
// Fill the new fields in the term table.
proc_run('php',"include/tagupdate.php");
return UPDATE_SUCCESS;
}