From 90a8d6f0f5dddfc8eb99e53ea06ea58010dd6379 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 15 Jan 2016 23:27:25 +0100 Subject: [PATCH] Several performance improvements --- boot.php | 5 + database.sql | 26 ++++- doc/database.md | 2 + doc/database/db_oembed.md | 10 ++ doc/database/db_parsed_url.md | 12 ++ include/Contact.php | 28 ++--- include/conversation.php | 8 +- include/cron.php | 199 +++++++++++++++++++--------------- include/text.php | 10 +- index.php | 1 + mod/community.php | 4 + view/theme/vier/style.php | 48 ++++++-- 12 files changed, 236 insertions(+), 117 deletions(-) create mode 100644 doc/database/db_oembed.md create mode 100644 doc/database/db_parsed_url.md diff --git a/boot.php b/boot.php index 049dc9110..1b1c6a84d 100644 --- a/boot.php +++ b/boot.php @@ -912,6 +912,10 @@ class App { } function get_cached_avatar_image($avatar_image){ + return $avatar_image; + + // The following code is deactivated. It doesn't seem to make any sense and it slows down the system. + /* if($this->cached_profile_image[$avatar_image]) return $this->cached_profile_image[$avatar_image]; @@ -931,6 +935,7 @@ class App { } } return $this->cached_profile_image[$avatar_image]; + */ } diff --git a/database.sql b/database.sql index df5ec74e2..70b315ea2 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 3.5-dev (Asparagus) --- DB_UPDATE_VERSION 1192 +-- DB_UPDATE_VERSION 1193 -- ------------------------------------------ @@ -685,6 +685,30 @@ CREATE TABLE IF NOT EXISTS `notify-threads` ( INDEX `receiver-uid` (`receiver-uid`) ) DEFAULT CHARSET=utf8; +-- +-- TABLE oembed +-- +CREATE TABLE IF NOT EXISTS `oembed` ( + `url` varchar(255) NOT NULL, + `content` text NOT NULL, + `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', + PRIMARY KEY(`url`), + INDEX `created` (`created`) +) DEFAULT CHARSET=utf8; + +-- +-- TABLE parsed_url +-- +CREATE TABLE IF NOT EXISTS `parsed_url` ( + `url` varchar(255) NOT NULL, + `guessing` tinyint(1) NOT NULL DEFAULT 0, + `oembed` tinyint(1) NOT NULL DEFAULT 0, + `content` text NOT NULL, + `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', + PRIMARY KEY(`url`,`guessing`,`oembed`), + INDEX `created` (`created`) +) DEFAULT CHARSET=utf8; + -- -- TABLE pconfig -- diff --git a/doc/database.md b/doc/database.md index 2ef77c6da..a0b28f4e8 100644 --- a/doc/database.md +++ b/doc/database.md @@ -38,6 +38,8 @@ Database Tables | [manage](help/database/db_manage) | table of accounts that can "su" each other | | [notify](help/database/db_notify) | notifications | | [notify-threads](help/database/db_notify-threads) | | +| [oembed](help/database/db_oembed) | cache for OEmbed queries | +| [parsed_url](help/database/db_parsed_url) | cache for "parse_url" queries | | [pconfig](help/database/db_pconfig) | personal (per user) configuration storage | | [photo](help/database/db_photo) | photo storage | | [poll](help/database/db_poll) | data for polls | diff --git a/doc/database/db_oembed.md b/doc/database/db_oembed.md new file mode 100644 index 000000000..5e994eca3 --- /dev/null +++ b/doc/database/db_oembed.md @@ -0,0 +1,10 @@ +Table oembed +============ + +| Field | Description | Type | Null | Key | Default | Extra | +| ------------ | ---------------------------------- | ------------ | ---- | --- | ------------------- | ----- | +| url | page url | varchar(255) | NO | PRI | NULL | | +| content | OEmbed data of the page | text | NO | | NULL | | +| created | datetime of creation | datetime | NO | MUL | 0000-00-00 00:00:00 | | + +Return to [database documentation](help/database) diff --git a/doc/database/db_parsed_url.md b/doc/database/db_parsed_url.md new file mode 100644 index 000000000..ada42c2ea --- /dev/null +++ b/doc/database/db_parsed_url.md @@ -0,0 +1,12 @@ +Table parsed_url +================ + +| Field | Description | Type | Null | Key | Default | Extra | +| ------------ | ---------------------------------- | ------------ | ---- | --- | ------------------- | ----- | +| url | page url | varchar(255) | NO | PRI | NULL | | +| guessing | is the "guessing" mode active? | tinyint(1) | NO | PRI | 0 | | +| oembed | is the data the result of oembed? | tinyint(1) | NO | PRI | 0 | | +| content | page data | text | NO | | NULL | | +| created | datetime of creation | datetime | NO | MUL | 0000-00-00 00:00:00 | | + +Return to [database documentation](help/database) diff --git a/include/Contact.php b/include/Contact.php index c10bb9b79..93d6237cb 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -211,40 +211,40 @@ function get_contact_details_by_url($url, $uid = -1) { $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d AND `network` = '%s'", dbesc(normalise_link($url)), intval($uid), dbesc($profile["network"])); - if (!count($r)) + if (!count($r) AND !isset($profile)) $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d", dbesc(normalise_link($url)), intval($uid)); - if (!count($r)) + if (!count($r) AND !isset($profile)) $r = q("SELECT `id`, `uid`, `url`, `network`, `name`, `nick`, `addr`, `location`, `about`, `keywords`, `gender`, `photo`, `addr`, `forum`, `prv`, `bd` FROM `contact` WHERE `nurl` = '%s' AND `uid` = 0", dbesc(normalise_link($url))); if ($r) { - if (isset($r[0]["url"]) AND $r[0]["url"]) + if (!isset($profile["url"]) AND $r[0]["url"]) $profile["url"] = $r[0]["url"]; - if (isset($r[0]["name"]) AND $r[0]["name"]) + if (!isset($profile["name"]) AND $r[0]["name"]) $profile["name"] = $r[0]["name"]; - if (isset($r[0]["nick"]) AND $r[0]["nick"] AND ($profile["nick"] == "")) + if (!isset($profile["nick"]) AND $r[0]["nick"]) $profile["nick"] = $r[0]["nick"]; - if (isset($r[0]["addr"]) AND $r[0]["addr"] AND ($profile["addr"] == "")) + if (!isset($profile["addr"]) AND $r[0]["addr"]) $profile["addr"] = $r[0]["addr"]; - if (isset($r[0]["photo"]) AND $r[0]["photo"]) + if (!isset($profile["photo"]) AND $r[0]["photo"]) $profile["photo"] = $r[0]["photo"]; - if (isset($r[0]["location"]) AND $r[0]["location"]) + if (!isset($profile["location"]) AND $r[0]["location"]) $profile["location"] = $r[0]["location"]; - if (isset($r[0]["about"]) AND $r[0]["about"]) + if (!isset($profile["about"]) AND $r[0]["about"]) $profile["about"] = $r[0]["about"]; - if (isset($r[0]["keywords"]) AND $r[0]["keywords"]) + if (!isset($profile["keywords"]) AND $r[0]["keywords"]) $profile["keywords"] = $r[0]["keywords"]; - if (isset($r[0]["gender"]) AND $r[0]["gender"]) + if (!isset($profile["gender"]) AND $r[0]["gender"]) $profile["gender"] = $r[0]["gender"]; if (isset($r[0]["forum"]) OR isset($r[0]["prv"])) $profile["community"] = ($r[0]["forum"] OR $r[0]["prv"]); - if (isset($r[0]["network"]) AND $r[0]["network"]) + if (!isset($profile["network"]) AND $r[0]["network"]) $profile["network"] = $r[0]["network"]; - if (isset($r[0]["addr"]) AND $r[0]["addr"]) + if (!isset($profile["addr"]) AND $r[0]["addr"]) $profile["addr"] = $r[0]["addr"]; - if (isset($r[0]["bd"]) AND $r[0]["bd"]) + if (!isset($profile["bd"]) AND $r[0]["bd"]) $profile["bd"] = $r[0]["bd"]; if ($r[0]["uid"] == 0) $profile["cid"] = 0; diff --git a/include/conversation.php b/include/conversation.php index d8c5e4461..5268c9e29 100644 --- a/include/conversation.php +++ b/include/conversation.php @@ -811,16 +811,16 @@ function best_link_url($item,&$sparkle,$ssl_state = false) { if((local_user()) && (local_user() == $item['uid'])) { if(isset($a->contacts) && x($a->contacts,$clean_url)) { if($a->contacts[$clean_url]['network'] === NETWORK_DFRN) { - $best_url = $a->get_baseurl($ssl_state) . '/redir/' . $a->contacts[$clean_url]['id']; + $best_url = 'redir/'.$a->contacts[$clean_url]['id']; $sparkle = true; } else $best_url = $a->contacts[$clean_url]['url']; } } elseif (local_user()) { - $r = q("SELECT `id`, `network` FROM `contact` WHERE `network` = '%s' AND `uid` = %d AND `nurl` = '%s'", + $r = q("SELECT `id` FROM `contact` WHERE `network` = '%s' AND `uid` = %d AND `nurl` = '%s' LIMIT 1", dbesc(NETWORK_DFRN), intval(local_user()), dbesc(normalise_link($clean_url))); if ($r) { - $best_url = $a->get_baseurl($ssl_state).'/redir/'.$r[0]['id']; + $best_url = 'redir/'.$r[0]['id']; $sparkle = true; } } @@ -876,7 +876,7 @@ function item_photo_menu($item){ if(local_user() && local_user() == $item['uid'] && link_compare($item['url'],$item['author-link'])) { $cid = $item['contact-id']; } else { - $r = q("SELECT `id`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' ORDER BY `uid` DESC LIMIT 1", + $r = q("SELECT `id`, `network` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' LIMIT 1", intval(local_user()), dbesc(normalise_link($item['author-link']))); if ($r) { $cid = $r[0]["id"]; diff --git a/include/cron.php b/include/cron.php index ed7edc699..bf3243cb6 100644 --- a/include/cron.php +++ b/include/cron.php @@ -158,93 +158,11 @@ function cron_run(&$argv, &$argc){ proc_run('php','include/expire.php'); } - $last = get_config('system','cache_last_cleared'); + // Clear cache entries + cron_clear_cache($a); - if($last) { - $next = $last + (3600); // Once per hour - $clear_cache = ($next <= time()); - } else - $clear_cache = true; - - if ($clear_cache) { - // clear old cache - Cache::clear(); - - // clear old item cache files - clear_cache(); - - // clear cache for photos - clear_cache($a->get_basepath(), $a->get_basepath()."/photo"); - - // clear smarty cache - clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled"); - - // clear cache for image proxy - if (!get_config("system", "proxy_disabled")) { - clear_cache($a->get_basepath(), $a->get_basepath()."/proxy"); - - $cachetime = get_config('system','proxy_cache_time'); - if (!$cachetime) $cachetime = PROXY_DEFAULT_TIME; - - q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime); - } - - // Maximum table size in megabyte - $max_tablesize = intval(get_config('system','optimize_max_tablesize')) * 1000000; - if ($max_tablesize == 0) - $max_tablesize = 100 * 1000000; // Default are 100 MB - - // Minimum fragmentation level in percent - $fragmentation_level = intval(get_config('system','optimize_fragmentation')) / 100; - if ($fragmentation_level == 0) - $fragmentation_level = 0.3; // Default value is 30% - - // Optimize some tables that need to be optimized - $r = q("SHOW TABLE STATUS"); - foreach($r as $table) { - - // Don't optimize tables that are too large - if ($table["Data_length"] > $max_tablesize) - continue; - - // Don't optimize empty tables - if ($table["Data_length"] == 0) - continue; - - // Calculate fragmentation - $fragmentation = $table["Data_free"] / $table["Data_length"]; - - logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG); - - // Don't optimize tables that needn't to be optimized - if ($fragmentation < $fragmentation_level) - continue; - - // So optimize it - logger("Optimize Table ".$table["Name"], LOGGER_DEBUG); - q("OPTIMIZE TABLE `%s`", dbesc($table["Name"])); - } - - set_config('system','cache_last_cleared', time()); - } - - // Repair missing Diaspora settings - $r = q("SELECT `id`, `url` FROM `contact` - WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') - ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); - if ($r) { - foreach ($r AS $contact) { - if (poco_reachable($contact["url"])) { - $data = probe_url($contact["url"]); - if ($data["network"] == NETWORK_DIASPORA) { - logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG); - q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", - dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]), - intval($contact["id"])); - } - } - } - } + // Repair missing Diaspora values in contacts + cron_repair_diaspora($a); $manual_id = 0; $generation = 0; @@ -390,6 +308,115 @@ function cron_run(&$argv, &$argc){ return; } +/** + * @brief Clear cache entries + * + * @param App $a + */ +function cron_clear_cache(&$a) { + + $last = get_config('system','cache_last_cleared'); + + if($last) { + $next = $last + (3600); // Once per hour + $clear_cache = ($next <= time()); + } else + $clear_cache = true; + + if (!$clear_cache) + return; + + // clear old cache + Cache::clear(); + + // clear old item cache files + clear_cache(); + + // clear cache for photos + clear_cache($a->get_basepath(), $a->get_basepath()."/photo"); + + // clear smarty cache + clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled"); + + // clear cache for image proxy + if (!get_config("system", "proxy_disabled")) { + clear_cache($a->get_basepath(), $a->get_basepath()."/proxy"); + + $cachetime = get_config('system','proxy_cache_time'); + if (!$cachetime) $cachetime = PROXY_DEFAULT_TIME; + + q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime); + } + + // Delete the cached OEmbed entries that are older than one year + q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 1 YEAR"); + + // Delete the cached "parse_url" entries that are older than one year + q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 1 YEAR"); + + // Maximum table size in megabyte + $max_tablesize = intval(get_config('system','optimize_max_tablesize')) * 1000000; + if ($max_tablesize == 0) + $max_tablesize = 100 * 1000000; // Default are 100 MB + + // Minimum fragmentation level in percent + $fragmentation_level = intval(get_config('system','optimize_fragmentation')) / 100; + if ($fragmentation_level == 0) + $fragmentation_level = 0.3; // Default value is 30% + + // Optimize some tables that need to be optimized + $r = q("SHOW TABLE STATUS"); + foreach($r as $table) { + + // Don't optimize tables that are too large + if ($table["Data_length"] > $max_tablesize) + continue; + + // Don't optimize empty tables + if ($table["Data_length"] == 0) + continue; + + // Calculate fragmentation + $fragmentation = $table["Data_free"] / $table["Data_length"]; + + logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG); + + // Don't optimize tables that needn't to be optimized + if ($fragmentation < $fragmentation_level) + continue; + + // So optimize it + logger("Optimize Table ".$table["Name"], LOGGER_DEBUG); + q("OPTIMIZE TABLE `%s`", dbesc($table["Name"])); + } + + set_config('system','cache_last_cleared', time()); +} + +/** + * @brief Repair missing values in Diaspora contacts + * + * @param App $a + */ +function cron_repair_diaspora(&$a) { + $r = q("SELECT `id`, `url` FROM `contact` + WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') + ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); + if ($r) { + foreach ($r AS $contact) { + if (poco_reachable($contact["url"])) { + $data = probe_url($contact["url"]); + if ($data["network"] == NETWORK_DIASPORA) { + logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG); + q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", + dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]), + intval($contact["id"])); + } + } + } + } +} + if (array_search(__file__,get_included_files())===0){ cron_run($_SERVER["argv"],$_SERVER["argc"]); killme(); diff --git a/include/text.php b/include/text.php index 3f4fe07d6..7e0aec97e 100644 --- a/include/text.php +++ b/include/text.php @@ -1416,7 +1416,15 @@ function prepare_body(&$item,$attach = false, $preview = false) { $item['mentions'] = $mentions; $test = $item["rendered-html"]; - put_item_in_cache($item, true); + + // Update the cached values if there is no "zrl=..." on the links + $update = (!local_user() and !remote_user() and ($item["uid"] == 0)); + + // Or update it if the current viewer is the intented viewer + if (($item["uid"] == local_user()) AND ($item["uid"] != 0)) + $update = true; + + put_item_in_cache($item, $update); $s = $item["rendered-html"]; //if ($test != $s) diff --git a/index.php b/index.php index 89ed05846..bf926d1fe 100644 --- a/index.php +++ b/index.php @@ -556,6 +556,7 @@ EOT; $page = $a->page; $profile = $a->profile; +header("X-Friendica-Version: ".FRIENDICA_VERSION); header("Content-type: text/html; charset=utf-8"); diff --git a/mod/community.php b/mod/community.php index 86bd18c86..b6d72a355 100644 --- a/mod/community.php +++ b/mod/community.php @@ -14,6 +14,10 @@ function community_content(&$a, $update = 0) { $o = ''; + // Currently the community page isn't able to handle update requests + if ($update) + return; + if((get_config('system','block_public')) && (! local_user()) && (! remote_user())) { notice( t('Public access denied.') . EOL); return; diff --git a/view/theme/vier/style.php b/view/theme/vier/style.php index 72731a9be..3d3c77674 100644 --- a/view/theme/vier/style.php +++ b/view/theme/vier/style.php @@ -1,12 +1,7 @@ $modified) + $modified = $stylemodified; + +$modified = gmdate('r', $modified); + +$etag = md5($stylecss); + +// Only send the CSS file if it was changed +header('Cache-Control: public'); +header('ETag: "'.$etag.'"'); +header('Last-Modified: '.$modified); + +if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE']) || isset($_SERVER['HTTP_IF_NONE_MATCH'])) { + + $cached_modified = gmdate('r', strtotime($_SERVER['HTTP_IF_MODIFIED_SINCE'])); + $cached_etag = str_replace(array('"', "-gzip"), array('', ''), + stripslashes($_SERVER['HTTP_IF_NONE_MATCH'])); + + if (($cached_modified == $modified) AND ($cached_etag == $etag)) { + header('HTTP/1.1 304 Not Modified'); + exit(); + } +} echo $stylecss;