From c4ba035ee314295c5a6a62ed49554f09a50c86f4 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 14 Jan 2016 21:56:37 +0100 Subject: [PATCH 1/9] Add some analysing stuff, rendered value is written now more often --- boot.php | 28 +++++++++++++++++++++++++++- include/conversation.php | 10 +++++----- include/cron.php | 2 +- include/diaspora.php | 6 +++++- include/items.php | 5 +++-- include/text.php | 8 ++++++-- include/threads.php | 8 +++++--- 7 files changed, 52 insertions(+), 15 deletions(-) diff --git a/boot.php b/boot.php index 7d8c7040d..ae98e9240 100644 --- a/boot.php +++ b/boot.php @@ -467,6 +467,7 @@ class App { public $is_tablet; public $is_friendica_app; public $performance = array(); + public $callstack = array(); public $nav_sel; @@ -552,7 +553,15 @@ class App { $this->performance["rendering"] = 0; $this->performance["parser"] = 0; $this->performance["marktime"] = 0; - $this->performance["markstart"] = microtime(true); + + $this->performance["file"] = 0; + $this->performance["file"] = 0; + + $this->callstack["database"] = array(); + $this->callstack["network"] = array(); + $this->callstack["file"] = array(); + $this->callstack["rendering"] = array(); + $this->callstack["parser"] = array(); $this->config = array(); $this->page = array(); @@ -1016,6 +1025,23 @@ class App { $this->performance[$value] += (float)$duration; $this->performance["marktime"] += (float)$duration; + + // Trace the different functions with their timestamps + $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 5); + + array_shift($trace); + + $function = array(); + foreach ($trace AS $func) + $function[] = $func["function"]; + + $function = implode(", ", $function); + + //$last = array_pop($trace); + //$function = $last["function"]; + + $this->callstack[$value][$function] += (float)$duration; + } function mark_timestamp($mark) { diff --git a/include/conversation.php b/include/conversation.php index 7eae1e052..d8c5e4461 100644 --- a/include/conversation.php +++ b/include/conversation.php @@ -315,11 +315,11 @@ function localize_item(&$item){ } // add zrl's to public images - $photo_pattern = "/\[url=(.*?)\/photos\/(.*?)\/image\/(.*?)\]\[img(.*?)\]h(.*?)\[\/img\]\[\/url\]/is"; - if(preg_match($photo_pattern,$item['body'])) { - $photo_replace = '[url=' . zrl('$1' . '/photos/' . '$2' . '/image/' . '$3' ,true) . '][img' . '$4' . ']h' . '$5' . '[/img][/url]'; - $item['body'] = bb_tag_preg_replace($photo_pattern, $photo_replace, 'url', $item['body']); - } +// $photo_pattern = "/\[url=(.*?)\/photos\/(.*?)\/image\/(.*?)\]\[img(.*?)\]h(.*?)\[\/img\]\[\/url\]/is"; +// if(preg_match($photo_pattern,$item['body'])) { +// $photo_replace = '[url=' . zrl('$1' . '/photos/' . '$2' . '/image/' . '$3' ,true) . '][img' . '$4' . ']h' . '$5' . '[/img][/url]'; +// $item['body'] = bb_tag_preg_replace($photo_pattern, $photo_replace, 'url', $item['body']); +// } // add sparkle links to appropriate permalinks diff --git a/include/cron.php b/include/cron.php index 09d141702..ed7edc699 100644 --- a/include/cron.php +++ b/include/cron.php @@ -230,7 +230,7 @@ function cron_run(&$argv, &$argc){ // Repair missing Diaspora settings $r = q("SELECT `id`, `url` FROM `contact` - WHERE `uid` > 0 AND `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') + WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); if ($r) { foreach ($r AS $contact) { diff --git a/include/diaspora.php b/include/diaspora.php index 1cf8897af..f03486fc2 100644 --- a/include/diaspora.php +++ b/include/diaspora.php @@ -1973,11 +1973,15 @@ function diaspora_photo($importer,$xml,$msg,$attempt=1) { array($remote_photo_name, 'scaled_full_' . $remote_photo_name)); if(strpos($parent_item['body'],$link_text) === false) { + + $parent_item['body'] = $link_text . $parent_item['body']; + $r = q("UPDATE `item` SET `body` = '%s', `visible` = 1 WHERE `id` = %d AND `uid` = %d", - dbesc($link_text . $parent_item['body']), + dbesc($parent_item['body']), intval($parent_item['id']), intval($parent_item['uid']) ); + put_item_in_cache($parent_item, true); update_thread($parent_item['id']); } diff --git a/include/items.php b/include/items.php index 549027671..62526d448 100644 --- a/include/items.php +++ b/include/items.php @@ -955,8 +955,9 @@ function add_page_info_data($data) { $a = get_app(); $hashtags = "\n"; foreach ($data["keywords"] AS $keyword) { - $hashtag = str_replace(array(" ", "+", "/", ".", "#", "'"), - array("","", "", "", "", ""), $keyword); + /// @todo make a positive list of allowed characters + $hashtag = str_replace(array(" ", "+", "/", ".", "#", "'", "’", "`", "(", ")", "„", "“"), + array("","", "", "", "", "", "", "", "", "", "", ""), $keyword); $hashtags .= "#[url=".$a->get_baseurl()."/search?tag=".rawurlencode($hashtag)."]".$hashtag."[/url] "; } } diff --git a/include/text.php b/include/text.php index 1b03c39d2..3f4fe07d6 100644 --- a/include/text.php +++ b/include/text.php @@ -20,10 +20,10 @@ function replace_macros($s,$r) { $stamp1 = microtime(true); $a = get_app(); - + // pass $baseurl to all templates $r['$baseurl'] = z_root(); - + $t = $a->template_engine(); try { @@ -1415,9 +1415,13 @@ function prepare_body(&$item,$attach = false, $preview = false) { $item['hashtags'] = $hashtags; $item['mentions'] = $mentions; + $test = $item["rendered-html"]; put_item_in_cache($item, true); $s = $item["rendered-html"]; + //if ($test != $s) + // $s .= "
*********************************
".$test; + $prep_arr = array('item' => $item, 'html' => $s, 'preview' => $preview); call_hooks('prepare_body', $prep_arr); $s = $prep_arr['html']; diff --git a/include/threads.php b/include/threads.php index dddcc4cdd..e542295d7 100644 --- a/include/threads.php +++ b/include/threads.php @@ -112,7 +112,7 @@ function update_thread_uri($itemuri, $uid) { function update_thread($itemid, $setmention = false) { $items = q("SELECT `uid`, `guid`, `title`, `body`, `created`, `edited`, `commented`, `received`, `changed`, `wall`, `private`, `pubmail`, `moderated`, `visible`, `spam`, `starred`, `bookmark`, `contact-id`, `gcontact-id`, - `deleted`, `origin`, `forum_mode`, `network` FROM `item` WHERE `id` = %d AND (`parent` = %d OR `parent` = 0) LIMIT 1", intval($itemid), intval($itemid)); + `deleted`, `origin`, `forum_mode`, `network`, `rendered-html`, `rendered-hash` FROM `item` WHERE `id` = %d AND (`parent` = %d OR `parent` = 0) LIMIT 1", intval($itemid), intval($itemid)); if (!$items) return; @@ -125,7 +125,7 @@ function update_thread($itemid, $setmention = false) { $sql = ""; foreach ($item AS $field => $data) - if (!in_array($field, array("guid", "title", "body"))) { + if (!in_array($field, array("guid", "title", "body", "rendered-html", "rendered-hash"))) { if ($sql != "") $sql .= ", "; @@ -142,9 +142,11 @@ function update_thread($itemid, $setmention = false) { if (!$items) return; - $result = q("UPDATE `item` SET `title` = '%s', `body` = '%s' WHERE `id` = %d", + $result = q("UPDATE `item` SET `title` = '%s', `body` = '%s', `rendered-html` = '%s', `rendered-hash` = '%s' WHERE `id` = %d", dbesc($item["title"]), dbesc($item["body"]), + dbesc($item["rendered-html"]), + dbesc($item["rendered-hash"]), intval($items[0]["id"]) ); logger("Updating public shadow for post ".$items[0]["id"]." - guid ".$item["guid"]." Result: ".print_r($result, true), LOGGER_DEBUG); From d5e1f33506dbb83d333e7ecd5e776c44eca35cb7 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 14 Jan 2016 23:59:51 +0100 Subject: [PATCH 2/9] OEmbed and parse_url are now cached in dedicated tables --- boot.php | 2 +- include/dbstructure.php | 20 ++++++++++++++++++++ include/items.php | 7 +------ include/oembed.php | 18 +++++++++++++++--- mod/parse_url.php | 13 +++++++++++-- update.php | 2 +- 6 files changed, 49 insertions(+), 13 deletions(-) diff --git a/boot.php b/boot.php index ae98e9240..049dc9110 100644 --- a/boot.php +++ b/boot.php @@ -36,7 +36,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Asparagus'); define ( 'FRIENDICA_VERSION', '3.5-dev' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1192 ); +define ( 'DB_UPDATE_VERSION', 1193 ); /** * @brief Constant with a HTML line break. diff --git a/include/dbstructure.php b/include/dbstructure.php index 7fed50e74..35bacd7f8 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -1021,6 +1021,26 @@ function db_definition() { "receiver-uid" => array("receiver-uid"), ) ); + $database["oembed"] = array( + "fields" => array( + "url" => array("type" => "varchar(255)", "not null" => "1", "primary" => "1"), + "content" => array("type" => "text", "not null" => "1"), + ), + "indexes" => array( + "PRIMARY" => array("url"), + ) + ); + $database["parsed_url"] = array( + "fields" => array( + "url" => array("type" => "varchar(255)", "not null" => "1", "primary" => "1"), + "guessing" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0", "primary" => "1"), + "oembed" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0", "primary" => "1"), + "content" => array("type" => "text", "not null" => "1"), + ), + "indexes" => array( + "PRIMARY" => array("url", "guessing", "oembed"), + ) + ); $database["pconfig"] = array( "fields" => array( "id" => array("type" => "int(11)", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), diff --git a/include/items.php b/include/items.php index 62526d448..a74ca3b36 100644 --- a/include/items.php +++ b/include/items.php @@ -968,12 +968,7 @@ function add_page_info_data($data) { function query_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "") { require_once("mod/parse_url.php"); - $data = Cache::get("parse_url:".$url); - if (is_null($data)){ - $data = parseurl_getsiteinfo($url, true); - Cache::set("parse_url:".$url,serialize($data), CACHE_DAY); - } else - $data = unserialize($data); + $data = parseurl_getsiteinfo_cached($url, true); if ($photo != "") $data["images"][0]["src"] = $photo; diff --git a/include/oembed.php b/include/oembed.php index c848a4580..511306458 100755 --- a/include/oembed.php +++ b/include/oembed.php @@ -13,7 +13,13 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){ $a = get_app(); - $txt = Cache::get($a->videowidth . $embedurl); + $r = q("SELECT * FROM `oembed` WHERE `url` = '%s'", + dbesc(normalise_link($embedurl))); + + if ($r) + $txt = $r[0]["content"]; + else + $txt = Cache::get($a->videowidth . $embedurl); // These media files should now be caught in bbcode.php // left here as a fallback in case this is called from another source @@ -66,8 +72,14 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){ if ($txt[0]!="{") $txt='{"type":"error"}'; - else //save in cache + else { //save in cache + $j = json_decode($txt); + if ($j->type != "error") + q("INSERT INTO `oembed` (`url`, `content`) VALUES ('%s', '%s')", + dbesc(normalise_link($embedurl)), dbesc($txt)); + Cache::set($a->videowidth . $embedurl,$txt, CACHE_DAY); + } } $j = json_decode($txt); @@ -85,7 +97,7 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){ // If fetching information doesn't work, then improve via internal functions if (($j->type == "error") OR ($no_rich_type AND ($j->type == "rich"))) { require_once("mod/parse_url.php"); - $data = parseurl_getsiteinfo($embedurl, true, false); + $data = parseurl_getsiteinfo_cached($embedurl, true, false); $j->type = $data["type"]; if ($j->type == "photo") { diff --git a/mod/parse_url.php b/mod/parse_url.php index 28869b4c1..8aa0c17ee 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -56,7 +56,15 @@ function completeurl($url, $scheme) { function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = true) { - $data = Cache::get("parse_url:".$no_guessing.":".$do_oembed.":".$url); + if ($url == "") + return false; + + $r = q("SELECT * FROM `parsed_url` WHERE `url` = '%s' AND `guessing` = %d AND `oembed` = %d", + dbesc(normalise_link($url)), intval(!$no_guessing), intval($do_oembed)); + + if ($r) + $data = $r[0]["content"]; + if (!is_null($data)) { $data = unserialize($data); return $data; @@ -64,7 +72,8 @@ function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = tr $data = parseurl_getsiteinfo($url, $no_guessing, $do_oembed); - Cache::set("parse_url:".$no_guessing.":".$do_oembed.":".$url,serialize($data), CACHE_DAY); + q("INSERT INTO `parsed_url` (`url`, `guessing`, `oembed`, `content`) VALUES ('%s', %d, %d, '%s')", + dbesc(normalise_link($url)), intval(!$no_guessing), intval($do_oembed), dbesc(serialize($data))); return $data; } diff --git a/update.php b/update.php index 197e78de5..a825c82e9 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ Date: Fri, 15 Jan 2016 00:58:57 +0100 Subject: [PATCH 3/9] Add the creation date to the cache tables --- include/dbstructure.php | 4 ++++ include/oembed.php | 4 ++-- mod/parse_url.php | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 35bacd7f8..43dfbd7b3 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -1025,9 +1025,11 @@ function db_definition() { "fields" => array( "url" => array("type" => "varchar(255)", "not null" => "1", "primary" => "1"), "content" => array("type" => "text", "not null" => "1"), + "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), ), "indexes" => array( "PRIMARY" => array("url"), + "created" => array("created"), ) ); $database["parsed_url"] = array( @@ -1036,9 +1038,11 @@ function db_definition() { "guessing" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0", "primary" => "1"), "oembed" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0", "primary" => "1"), "content" => array("type" => "text", "not null" => "1"), + "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), ), "indexes" => array( "PRIMARY" => array("url", "guessing", "oembed"), + "created" => array("created"), ) ); $database["pconfig"] = array( diff --git a/include/oembed.php b/include/oembed.php index 511306458..b1770f689 100755 --- a/include/oembed.php +++ b/include/oembed.php @@ -75,8 +75,8 @@ function oembed_fetch_url($embedurl, $no_rich_type = false){ else { //save in cache $j = json_decode($txt); if ($j->type != "error") - q("INSERT INTO `oembed` (`url`, `content`) VALUES ('%s', '%s')", - dbesc(normalise_link($embedurl)), dbesc($txt)); + q("INSERT INTO `oembed` (`url`, `content`, `created`) VALUES ('%s', '%s', '%s')", + dbesc(normalise_link($embedurl)), dbesc($txt), dbesc(datetime_convert())); Cache::set($a->videowidth . $embedurl,$txt, CACHE_DAY); } diff --git a/mod/parse_url.php b/mod/parse_url.php index 8aa0c17ee..1ca5dc1da 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -72,8 +72,8 @@ function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = tr $data = parseurl_getsiteinfo($url, $no_guessing, $do_oembed); - q("INSERT INTO `parsed_url` (`url`, `guessing`, `oembed`, `content`) VALUES ('%s', %d, %d, '%s')", - dbesc(normalise_link($url)), intval(!$no_guessing), intval($do_oembed), dbesc(serialize($data))); + q("INSERT INTO `parsed_url` (`url`, `guessing`, `oembed`, `content`, `created`) VALUES ('%s', %d, %d, '%s', '%s')", + dbesc(normalise_link($url)), intval(!$no_guessing), intval($do_oembed), dbesc(serialize($data)), dbesc(datetime_convert())); return $data; } From 90a8d6f0f5dddfc8eb99e53ea06ea58010dd6379 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 15 Jan 2016 23:27:25 +0100 Subject: [PATCH 4/9] Several performance improvements --- boot.php | 5 + database.sql | 26 ++++- doc/database.md | 2 + doc/database/db_oembed.md | 10 ++ doc/database/db_parsed_url.md | 12 ++ include/Contact.php | 28 ++--- include/conversation.php | 8 +- include/cron.php | 199 +++++++++++++++++++--------------- include/text.php | 10 +- index.php | 1 + mod/community.php | 4 + view/theme/vier/style.php | 48 ++++++-- 12 files changed, 236 insertions(+), 117 deletions(-) create mode 100644 doc/database/db_oembed.md create mode 100644 doc/database/db_parsed_url.md diff --git a/boot.php b/boot.php index 049dc9110..1b1c6a84d 100644 --- a/boot.php +++ b/boot.php @@ -912,6 +912,10 @@ class App { } function get_cached_avatar_image($avatar_image){ + return $avatar_image; + + // The following code is deactivated. It doesn't seem to make any sense and it slows down the system. + /* if($this->cached_profile_image[$avatar_image]) return $this->cached_profile_image[$avatar_image]; @@ -931,6 +935,7 @@ class App { } } return $this->cached_profile_image[$avatar_image]; + */ } diff --git a/database.sql b/database.sql index df5ec74e2..70b315ea2 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 3.5-dev (Asparagus) --- DB_UPDATE_VERSION 1192 +-- DB_UPDATE_VERSION 1193 -- ------------------------------------------ @@ -685,6 +685,30 @@ CREATE TABLE IF NOT EXISTS `notify-threads` ( INDEX `receiver-uid` (`receiver-uid`) ) DEFAULT CHARSET=utf8; +-- +-- TABLE oembed +-- +CREATE TABLE IF NOT EXISTS `oembed` ( + `url` varchar(255) NOT NULL, + `content` text NOT NULL, + `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', + PRIMARY KEY(`url`), + INDEX `created` (`created`) +) DEFAULT CHARSET=utf8; + +-- +-- TABLE parsed_url +-- +CREATE TABLE IF NOT EXISTS `parsed_url` ( + `url` varchar(255) NOT NULL, + `guessing` tinyint(1) NOT NULL DEFAULT 0, + `oembed` tinyint(1) NOT NULL DEFAULT 0, + `content` text NOT NULL, + `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', + PRIMARY KEY(`url`,`guessing`,`oembed`), + INDEX `created` (`created`) +) DEFAULT CHARSET=utf8; + -- -- TABLE pconfig -- diff --git a/doc/database.md b/doc/database.md index 2ef77c6da..a0b28f4e8 100644 --- a/doc/database.md +++ b/doc/database.md @@ -38,6 +38,8 @@ Database Tables | [manage](help/database/db_manage) | table of accounts that can "su" each other | | [notify](help/database/db_notify) | notifications | | [notify-threads](help/database/db_notify-threads) | | +| [oembed](help/database/db_oembed) | cache for OEmbed queries | +| [parsed_url](help/database/db_parsed_url) | cache for "parse_url" queries | | [pconfig](help/database/db_pconfig) | personal (per user) configuration storage | | [photo](help/database/db_photo) | photo storage | | [poll](help/database/db_poll) | data for polls | diff --git a/doc/database/db_oembed.md b/doc/database/db_oembed.md new file mode 100644 index 000000000..5e994eca3 --- /dev/null +++ b/doc/database/db_oembed.md @@ -0,0 +1,10 @@ +Table oembed +============ + +| Field | Description | Type | Null | Key | Default | Extra | +| ------------ | ---------------------------------- | ------------ | ---- | --- | ------------------- | ----- | +| url | page url | varchar(255) | NO | PRI | NULL | | +| content | OEmbed data of the page | text | NO | | NULL | | +| created | datetime of creation | datetime | NO | MUL | 0000-00-00 00:00:00 | | + +Return to [database documentation](help/database) diff --git a/doc/database/db_parsed_url.md b/doc/database/db_parsed_url.md new file mode 100644 index 000000000..ada42c2ea --- /dev/null +++ b/doc/database/db_parsed_url.md @@ -0,0 +1,12 @@ +Table parsed_url +================ + +| Field | Description | Type | Null | Key | Default | Extra | +| ------------ | ---------------------------------- | ------------ | ---- | --- | ------------------- | ----- | +| url | page url | varchar(255) | NO | PRI | NULL | | +| guessing | is the "guessing" mode active? | tinyint(1) | NO | PRI | 0 | | +| oembed | is the data the result of oembed? | tinyint(1) | NO | PRI | 0 | | +| content | page data | text | NO | | NULL | | +| created | datetime of creation | datetime | NO | MUL | 0000-00-00 00:00:00 | | + +Return to [database documentation](help/database) diff --git a/include/Contact.php b/include/Contact.php index c10bb9b79..93d6237cb 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -211,40 +211,40 @@ function get_contact_details_by_url($url, $uid = -1) { $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d AND `network` = '%s'", dbesc(normalise_link($url)), intval($uid), dbesc($profile["network"])); - if (!count($r)) + if (!count($r) AND !isset($profile)) $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d", dbesc(normalise_link($url)), intval($uid)); - if (!count($r)) + if (!count($r) AND !isset($profile)) $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = 0", dbesc(normalise_link($url))); if ($r) { - if (isset($r[0]["url"]) AND $r[0]["url"]) + if (!isset($profile["url"]) AND $r[0]["url"]) $profile["url"] = $r[0]["url"]; - if (isset($r[0]["name"]) AND $r[0]["name"]) + if (!isset($profile["name"]) AND $r[0]["name"]) $profile["name"] = $r[0]["name"]; - if (isset($r[0]["nick"]) AND $r[0]["nick"] AND ($profile["nick"] == "")) + if (!isset($profile["nick"]) AND $r[0]["nick"]) $profile["nick"] = $r[0]["nick"]; - if (isset($r[0]["addr"]) AND $r[0]["addr"] AND ($profile["addr"] == "")) + if (!isset($profile["addr"]) AND $r[0]["addr"]) $profile["addr"] = $r[0]["addr"]; - if (isset($r[0]["photo"]) AND $r[0]["photo"]) + if (!isset($profile["photo"]) AND $r[0]["photo"]) $profile["photo"] = $r[0]["photo"]; - if (isset($r[0]["location"]) AND $r[0]["location"]) + if (!isset($profile["location"]) AND $r[0]["location"]) $profile["location"] = $r[0]["location"]; - if (isset($r[0]["about"]) AND $r[0]["about"]) + if (!isset($profile["about"]) AND $r[0]["about"]) $profile["about"] = $r[0]["about"]; - if (isset($r[0]["keywords"]) AND $r[0]["keywords"]) + if (!isset($profile["keywords"]) AND $r[0]["keywords"]) $profile["keywords"] = $r[0]["keywords"]; - if (isset($r[0]["gender"]) AND $r[0]["gender"]) + if (!isset($profile["gender"]) AND $r[0]["gender"]) $profile["gender"] = $r[0]["gender"]; if (isset($r[0]["forum"]) OR isset($r[0]["prv"])) $profile["community"] = ($r[0]["forum"] OR $r[0]["prv"]); - if (isset($r[0]["network"]) AND $r[0]["network"]) + if (!isset($profile["network"]) AND $r[0]["network"]) $profile["network"] = $r[0]["network"]; - if (isset($r[0]["addr"]) AND $r[0]["addr"]) + if (!isset($profile["addr"]) AND $r[0]["addr"]) $profile["addr"] = $r[0]["addr"]; - if (isset($r[0]["bd"]) AND $r[0]["bd"]) + if (!isset($profile["bd"]) AND $r[0]["bd"]) $profile["bd"] = $r[0]["bd"]; if ($r[0]["uid"] == 0) $profile["cid"] = 0; diff --git a/include/conversation.php b/include/conversation.php index d8c5e4461..5268c9e29 100644 --- a/include/conversation.php +++ b/include/conversation.php @@ -811,16 +811,16 @@ function best_link_url($item,&$sparkle,$ssl_state = false) { if((local_user()) && (local_user() == $item['uid'])) { if(isset($a->contacts) && x($a->contacts,$clean_url)) { if($a->contacts[$clean_url]['network'] === NETWORK_DFRN) { - $best_url = $a->get_baseurl($ssl_state) . '/redir/' . $a->contacts[$clean_url]['id']; + $best_url = 'redir/'.$a->contacts[$clean_url]['id']; $sparkle = true; } else $best_url = $a->contacts[$clean_url]['url']; } } elseif (local_user()) { - $r = q("SELECT `id`, `network` FROM `contact` WHERE `network` = '%s' AND `uid` = %d AND `nurl` = '%s'", + $r = q("SELECT `id` FROM `contact` WHERE `network` = '%s' AND `uid` = %d AND `nurl` = '%s' LIMIT 1", dbesc(NETWORK_DFRN), intval(local_user()), dbesc(normalise_link($clean_url))); if ($r) { - $best_url = $a->get_baseurl($ssl_state).'/redir/'.$r[0]['id']; + $best_url = 'redir/'.$r[0]['id']; $sparkle = true; } } @@ -876,7 +876,7 @@ function item_photo_menu($item){ if(local_user() && local_user() == $item['uid'] && link_compare($item['url'],$item['author-link'])) { $cid = $item['contact-id']; } else { - $r = q("SELECT `id`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' ORDER BY `uid` DESC LIMIT 1", + $r = q("SELECT `id`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' LIMIT 1", intval(local_user()), dbesc(normalise_link($item['author-link']))); if ($r) { $cid = $r[0]["id"]; diff --git a/include/cron.php b/include/cron.php index ed7edc699..bf3243cb6 100644 --- a/include/cron.php +++ b/include/cron.php @@ -158,93 +158,11 @@ function cron_run(&$argv, &$argc){ proc_run('php','include/expire.php'); } - $last = get_config('system','cache_last_cleared'); + // Clear cache entries + cron_clear_cache($a); - if($last) { - $next = $last + (3600); // Once per hour - $clear_cache = ($next <= time()); - } else - $clear_cache = true; - - if ($clear_cache) { - // clear old cache - Cache::clear(); - - // clear old item cache files - clear_cache(); - - // clear cache for photos - clear_cache($a->get_basepath(), $a->get_basepath()."/photo"); - - // clear smarty cache - clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled"); - - // clear cache for image proxy - if (!get_config("system", "proxy_disabled")) { - clear_cache($a->get_basepath(), $a->get_basepath()."/proxy"); - - $cachetime = get_config('system','proxy_cache_time'); - if (!$cachetime) $cachetime = PROXY_DEFAULT_TIME; - - q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime); - } - - // Maximum table size in megabyte - $max_tablesize = intval(get_config('system','optimize_max_tablesize')) * 1000000; - if ($max_tablesize == 0) - $max_tablesize = 100 * 1000000; // Default are 100 MB - - // Minimum fragmentation level in percent - $fragmentation_level = intval(get_config('system','optimize_fragmentation')) / 100; - if ($fragmentation_level == 0) - $fragmentation_level = 0.3; // Default value is 30% - - // Optimize some tables that need to be optimized - $r = q("SHOW TABLE STATUS"); - foreach($r as $table) { - - // Don't optimize tables that are too large - if ($table["Data_length"] > $max_tablesize) - continue; - - // Don't optimize empty tables - if ($table["Data_length"] == 0) - continue; - - // Calculate fragmentation - $fragmentation = $table["Data_free"] / $table["Data_length"]; - - logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG); - - // Don't optimize tables that needn't to be optimized - if ($fragmentation < $fragmentation_level) - continue; - - // So optimize it - logger("Optimize Table ".$table["Name"], LOGGER_DEBUG); - q("OPTIMIZE TABLE `%s`", dbesc($table["Name"])); - } - - set_config('system','cache_last_cleared', time()); - } - - // Repair missing Diaspora settings - $r = q("SELECT `id`, `url` FROM `contact` - WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') - ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); - if ($r) { - foreach ($r AS $contact) { - if (poco_reachable($contact["url"])) { - $data = probe_url($contact["url"]); - if ($data["network"] == NETWORK_DIASPORA) { - logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG); - q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", - dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]), - intval($contact["id"])); - } - } - } - } + // Repair missing Diaspora values in contacts + cron_repair_diaspora($a); $manual_id = 0; $generation = 0; @@ -390,6 +308,115 @@ function cron_run(&$argv, &$argc){ return; } +/** + * @brief Clear cache entries + * + * @param App $a + */ +function cron_clear_cache(&$a) { + + $last = get_config('system','cache_last_cleared'); + + if($last) { + $next = $last + (3600); // Once per hour + $clear_cache = ($next <= time()); + } else + $clear_cache = true; + + if (!$clear_cache) + return; + + // clear old cache + Cache::clear(); + + // clear old item cache files + clear_cache(); + + // clear cache for photos + clear_cache($a->get_basepath(), $a->get_basepath()."/photo"); + + // clear smarty cache + clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled"); + + // clear cache for image proxy + if (!get_config("system", "proxy_disabled")) { + clear_cache($a->get_basepath(), $a->get_basepath()."/proxy"); + + $cachetime = get_config('system','proxy_cache_time'); + if (!$cachetime) $cachetime = PROXY_DEFAULT_TIME; + + q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime); + } + + // Delete the cached OEmbed entries that are older than one year + q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 1 YEAR"); + + // Delete the cached "parse_url" entries that are older than one year + q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 1 YEAR"); + + // Maximum table size in megabyte + $max_tablesize = intval(get_config('system','optimize_max_tablesize')) * 1000000; + if ($max_tablesize == 0) + $max_tablesize = 100 * 1000000; // Default are 100 MB + + // Minimum fragmentation level in percent + $fragmentation_level = intval(get_config('system','optimize_fragmentation')) / 100; + if ($fragmentation_level == 0) + $fragmentation_level = 0.3; // Default value is 30% + + // Optimize some tables that need to be optimized + $r = q("SHOW TABLE STATUS"); + foreach($r as $table) { + + // Don't optimize tables that are too large + if ($table["Data_length"] > $max_tablesize) + continue; + + // Don't optimize empty tables + if ($table["Data_length"] == 0) + continue; + + // Calculate fragmentation + $fragmentation = $table["Data_free"] / $table["Data_length"]; + + logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG); + + // Don't optimize tables that needn't to be optimized + if ($fragmentation < $fragmentation_level) + continue; + + // So optimize it + logger("Optimize Table ".$table["Name"], LOGGER_DEBUG); + q("OPTIMIZE TABLE `%s`", dbesc($table["Name"])); + } + + set_config('system','cache_last_cleared', time()); +} + +/** + * @brief Repair missing values in Diaspora contacts + * + * @param App $a + */ +function cron_repair_diaspora(&$a) { + $r = q("SELECT `id`, `url` FROM `contact` + WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') + ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); + if ($r) { + foreach ($r AS $contact) { + if (poco_reachable($contact["url"])) { + $data = probe_url($contact["url"]); + if ($data["network"] == NETWORK_DIASPORA) { + logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG); + q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", + dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]), + intval($contact["id"])); + } + } + } + } +} + if (array_search(__file__,get_included_files())===0){ cron_run($_SERVER["argv"],$_SERVER["argc"]); killme(); diff --git a/include/text.php b/include/text.php index 3f4fe07d6..7e0aec97e 100644 --- a/include/text.php +++ b/include/text.php @@ -1416,7 +1416,15 @@ function prepare_body(&$item,$attach = false, $preview = false) { $item['mentions'] = $mentions; $test = $item["rendered-html"]; - put_item_in_cache($item, true); + + // Update the cached values if there is no "zrl=..." on the links + $update = (!local_user() and !remote_user() and ($item["uid"] == 0)); + + // Or update it if the current viewer is the intented viewer + if (($item["uid"] == local_user()) AND ($item["uid"] != 0)) + $update = true; + + put_item_in_cache($item, $update); $s = $item["rendered-html"]; //if ($test != $s) diff --git a/index.php b/index.php index 89ed05846..bf926d1fe 100644 --- a/index.php +++ b/index.php @@ -556,6 +556,7 @@ EOT; $page = $a->page; $profile = $a->profile; +header("X-Friendica-Version: ".FRIENDICA_VERSION); header("Content-type: text/html; charset=utf-8"); diff --git a/mod/community.php b/mod/community.php index 86bd18c86..b6d72a355 100644 --- a/mod/community.php +++ b/mod/community.php @@ -14,6 +14,10 @@ function community_content(&$a, $update = 0) { $o = ''; + // Currently the community page isn't able to handle update requests + if ($update) + return; + if((get_config('system','block_public')) && (! local_user()) && (! remote_user())) { notice( t('Public access denied.') . EOL); return; diff --git a/view/theme/vier/style.php b/view/theme/vier/style.php index 72731a9be..3d3c77674 100644 --- a/view/theme/vier/style.php +++ b/view/theme/vier/style.php @@ -1,12 +1,7 @@ $modified) + $modified = $stylemodified; + +$modified = gmdate('r', $modified); + +$etag = md5($stylecss); + +// Only send the CSS file if it was changed +header('Cache-Control: public'); +header('ETag: "'.$etag.'"'); +header('Last-Modified: '.$modified); + +if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE']) || isset($_SERVER['HTTP_IF_NONE_MATCH'])) { + + $cached_modified = gmdate('r', strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE'])); + $cached_etag = str_replace(array('"', "-gzip"), array('', ''), + stripslashes($_SERVER['HTTP_IF_NONE_MATCH'])); + + if (($cached_modified == $modified) AND ($cached_etag == $etag)) { + header('HTTP/1.1 304 Not Modified'); + exit(); + } +} echo $stylecss; From 092c2e54333b3cecd49f034623886c7fd8c8981a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 15 Jan 2016 23:32:13 +0100 Subject: [PATCH 5/9] Removed some test code --- boot.php | 3 --- include/conversation.php | 10 +++++----- include/text.php | 5 ----- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/boot.php b/boot.php index 1b1c6a84d..4f0bec62b 100644 --- a/boot.php +++ b/boot.php @@ -1042,9 +1042,6 @@ class App { $function = implode(", ", $function); - //$last = array_pop($trace); - //$function = $last["function"]; - $this->callstack[$value][$function] += (float)$duration; } diff --git a/include/conversation.php b/include/conversation.php index 5268c9e29..6c33be84f 100644 --- a/include/conversation.php +++ b/include/conversation.php @@ -315,11 +315,11 @@ function localize_item(&$item){ } // add zrl's to public images -// $photo_pattern = "/\[url=(.*?)\/photos\/(.*?)\/image\/(.*?)\]\[img(.*?)\]h(.*?)\[\/img\]\[\/url\]/is"; -// if(preg_match($photo_pattern,$item['body'])) { -// $photo_replace = '[url=' . zrl('$1' . '/photos/' . '$2' . '/image/' . '$3' ,true) . '][img' . '$4' . ']h' . '$5' . '[/img][/url]'; -// $item['body'] = bb_tag_preg_replace($photo_pattern, $photo_replace, 'url', $item['body']); -// } + $photo_pattern = "/\[url=(.*?)\/photos\/(.*?)\/image\/(.*?)\]\[img(.*?)\]h(.*?)\[\/img\]\[\/url\]/is"; + if(preg_match($photo_pattern,$item['body'])) { + $photo_replace = '[url=' . zrl('$1' . '/photos/' . '$2' . '/image/' . '$3' ,true) . '][img' . '$4' . ']h' . '$5' . '[/img][/url]'; + $item['body'] = bb_tag_preg_replace($photo_pattern, $photo_replace, 'url', $item['body']); + } // add sparkle links to appropriate permalinks diff --git a/include/text.php b/include/text.php index 7e0aec97e..200e2c9bc 100644 --- a/include/text.php +++ b/include/text.php @@ -1415,8 +1415,6 @@ function prepare_body(&$item,$attach = false, $preview = false) { $item['hashtags'] = $hashtags; $item['mentions'] = $mentions; - $test = $item["rendered-html"]; - // Update the cached values if there is no "zrl=..." on the links $update = (!local_user() and !remote_user() and ($item["uid"] == 0)); @@ -1427,9 +1425,6 @@ function prepare_body(&$item,$attach = false, $preview = false) { put_item_in_cache($item, $update); $s = $item["rendered-html"]; - //if ($test != $s) - // $s .= "
*********************************
".$test; - $prep_arr = array('item' => $item, 'html' => $s, 'preview' => $preview); call_hooks('prepare_body', $prep_arr); $s = $prep_arr['html']; From 571d1544e4ca30b3c4b591322b53bf8d62caddea Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 16 Jan 2016 08:58:22 +0100 Subject: [PATCH 6/9] I guess it is enough to set a variable to zero once ... --- boot.php | 3 --- 1 file changed, 3 deletions(-) diff --git a/boot.php b/boot.php index 4f0bec62b..963f5c454 100644 --- a/boot.php +++ b/boot.php @@ -554,9 +554,6 @@ class App { $this->performance["parser"] = 0; $this->performance["marktime"] = 0; - $this->performance["file"] = 0; - $this->performance["file"] = 0; - $this->callstack["database"] = array(); $this->callstack["network"] = array(); $this->callstack["file"] = array(); From 5769c08cbc7cac1881c7919695a8737f846fda2e Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 16 Jan 2016 09:16:51 +0100 Subject: [PATCH 7/9] This line vanished accidentally ... --- boot.php | 1 + 1 file changed, 1 insertion(+) diff --git a/boot.php b/boot.php index 963f5c454..2242ba476 100644 --- a/boot.php +++ b/boot.php @@ -553,6 +553,7 @@ class App { $this->performance["rendering"] = 0; $this->performance["parser"] = 0; $this->performance["marktime"] = 0; + $this->performance["markstart"] = microtime(true); $this->callstack["database"] = array(); $this->callstack["network"] = array(); From 22752039178e3d61844e6c1dafc8f1abd06b09cc Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 16 Jan 2016 12:32:11 +0100 Subject: [PATCH 8/9] Just some improved queries --- include/acl_selectors.php | 4 ++-- include/config.php | 4 ++-- include/text.php | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/acl_selectors.php b/include/acl_selectors.php index d5730a93a..69181b735 100644 --- a/include/acl_selectors.php +++ b/include/acl_selectors.php @@ -20,7 +20,7 @@ function group_select($selname,$selclass,$preselected = false,$size = 4) { $o .= "