From e6148f4c1cb0bc06e27f9e3d98f69f2722d02bed Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 12 Nov 2016 13:17:28 +0000 Subject: [PATCH 1/2] Performance improvements when storing items (via API) --- boot.php | 61 ++++++++++++++++++++++------------ include/Photo.php | 7 +--- include/api.php | 11 +++--- include/create_shadowentry.php | 40 ++++++++++++++++++++++ include/threads.php | 11 ++++++ mod/item.php | 21 ++++-------- 6 files changed, 102 insertions(+), 49 deletions(-) create mode 100644 include/create_shadowentry.php diff --git a/boot.php b/boot.php index eb91b26baf..74609b8968 100644 --- a/boot.php +++ b/boot.php @@ -1897,11 +1897,12 @@ function get_max_import_size() { * @brief Wrap calls to proc_close(proc_open()) and call hook * so plugins can take part in process :) * - * @param (string|integer) $cmd program to run or priority + * @param (string|integer|array) $cmd program to run, priority or parameter array * * next args are passed as $cmd command line * e.g.: proc_run("ls","-la","/tmp"); * or: proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id); + * or: proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id); * * @note $cmd and string args are surrounded with "" * @@ -1912,24 +1913,31 @@ function proc_run($cmd){ $a = get_app(); - $args = func_get_args(); + $proc_args = func_get_args(); - $newargs = array(); - if (!count($args)) + $args = array(); + if (!count($proc_args)) { return; - - // expand any arrays - - foreach($args as $arg) { - if(is_array($arg)) { - foreach($arg as $n) { - $newargs[] = $n; - } - } else - $newargs[] = $arg; } - $args = $newargs; + // Preserve the first parameter + // It could contain a command, the priority or an parameter array + // If we use the parameter array we have to protect it from the following function + $run_parameter = array_shift($proc_args); + + // expand any arrays + foreach ($proc_args as $arg) { + if (is_array($arg)) { + foreach ($arg as $n) { + $args[] = $n; + } + } else { + $args[] = $arg; + } + } + + // Now we add the run parameters back to the array + array_unshift($args, $run_parameter); $arr = array('args' => $args, 'run_cmd' => true); @@ -1937,16 +1945,24 @@ function proc_run($cmd){ if (!$arr['run_cmd'] OR !count($args)) return; - if (!get_config("system", "worker") OR - (($args[0] != 'php') AND !is_int($args[0]))) { + if (!get_config("system", "worker") OR (is_string($run_parameter) AND ($run_parameter != 'php'))) { $a->proc_run($args); return; } - if (is_int($args[0])) - $priority = $args[0]; - else - $priority = PRIORITY_MEDIUM; + $priority = PRIORITY_MEDIUM; + $dont_fork = get_config("system", "worker_dont_fork"); + + if (is_int($run_parameter)) { + $priority = $run_parameter; + } elseif (is_array($run_parameter)) { + if (isset($run_parameter['priority'])) { + $priority = $run_parameter['priority']; + } + if (isset($run_parameter['dont_fork'])) { + $dont_fork = $run_parameter['dont_fork']; + } + } $argv = $args; array_shift($argv); @@ -1963,8 +1979,9 @@ function proc_run($cmd){ intval($priority)); // Should we quit and wait for the poller to be called as a cronjob? - if (get_config("system", "worker_dont_fork")) + if ($dont_fork) { return; + } // Checking number of workers $workers = q("SELECT COUNT(*) AS `workers` FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'"); diff --git a/include/Photo.php b/include/Photo.php index 6a7cc3e52c..014cca7d25 100644 --- a/include/Photo.php +++ b/include/Photo.php @@ -707,11 +707,6 @@ class Photo { ); } - // Update the cached values - if ($album != 'Contact Photos') { - photo_albums($uid, true); - } - return $r; } } @@ -872,7 +867,7 @@ function get_photo_info($url) { $data = Cache::get($url); - if (is_null($data) OR !$data) { + if (is_null($data) OR !$data OR !is_array($data)) { $img_str = fetch_url($url, true, $redirects, 4); $filesize = strlen($img_str); diff --git a/include/api.php b/include/api.php index 350c6dd07c..123a4b9cfe 100644 --- a/include/api.php +++ b/include/api.php @@ -281,16 +281,15 @@ logger("API call duration: ".round($duration, 2)."\t".$a->query_string, LOGGER_DEBUG); if (get_config("system", "profiler")) { - logger(sprintf("Database: %s/%s, Network: %s, Rendering: %s, Session: %s, I/O: %s, Other: %s, Total: %s", + $duration = microtime(true)-$a->performance["start"]; + + logger(parse_url($a->query_string, PHP_URL_PATH).": ".sprintf("Database: %s/%s, Network: %s, I/O: %s, Other: %s, Total: %s", round($a->performance["database"] - $a->performance["database_write"], 3), round($a->performance["database_write"], 3), round($a->performance["network"], 2), - round($a->performance["rendering"], 2), - round($a->performance["parser"], 2), round($a->performance["file"], 2), - round($duration - $a->performance["database"] - - $a->performance["network"] - $a->performance["rendering"] - - $a->performance["parser"] - $a->performance["file"], 2), + round($duration - ($a->performance["database"] + $a->performance["network"] + + $a->performance["file"]), 2), round($duration, 2)), LOGGER_DEBUG); diff --git a/include/create_shadowentry.php b/include/create_shadowentry.php new file mode 100644 index 0000000000..4027024f99 --- /dev/null +++ b/include/create_shadowentry.php @@ -0,0 +1,40 @@ + diff --git a/include/threads.php b/include/threads.php index 3d9b656ea3..df0f8ee1de 100644 --- a/include/threads.php +++ b/include/threads.php @@ -123,8 +123,19 @@ function add_shadow_thread($itemid) { function add_shadow_entry($itemid) { $items = q("SELECT * FROM `item` WHERE `id` = %d", intval($itemid)); + + if (!dbm::is_result($items)) { + return; + } + $item = $items[0]; + // Is it a toplevel post? + if ($item['id'] == $item['parent']) { + add_shadow_thread($itemid); + return; + } + // Is this a shadow entry? if ($item['uid'] == 0) return; diff --git a/mod/item.php b/mod/item.php index 23bc2d0bb0..29a2101355 100644 --- a/mod/item.php +++ b/mod/item.php @@ -1020,22 +1020,13 @@ function item_post(&$a) { create_tags_from_item($post_id); create_files_from_item($post_id); - // Insert an item entry for UID=0 for global entries - if ($post_id != $parent) { - add_shadow_thread($post_id); - } else { - add_shadow_entry($post_id); - } - - // This is a real juggling act on shared hosting services which kill your processes - // e.g. dreamhost. We used to start delivery to our native delivery agents in the background - // and then run our plugin delivery from the foreground. We're now doing plugin delivery first, - // because as soon as you start loading up a bunch of remote delivey processes, *this* page is - // likely to get killed off. If you end up looking at an /item URL and a blank page, - // it's very likely the delivery got killed before all your friends could be notified. - // Currently the only realistic fixes are to use a reliable server - which precludes shared hosting, - // or cut back on plugins which do remote deliveries. + // Insert an item entry for UID=0 for global entries. + // We now do it in the background to save some time. + // This is important in interactive environments like the frontend or the API. + // We don't fork a new process since this is done anyway with the following command + proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id); + // Call the background process that is delivering the item to the receivers proc_run(PRIORITY_HIGH, "include/notifier.php", $notify_type, $post_id); logger('post_complete'); From d24e3a6a92af5ddb2b2f11140d3fc4baeb683c6f Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 12 Nov 2016 17:20:26 +0000 Subject: [PATCH 2/2] Removed "&" --- include/create_shadowentry.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/create_shadowentry.php b/include/create_shadowentry.php index 4027024f99..f06a0dd1bf 100644 --- a/include/create_shadowentry.php +++ b/include/create_shadowentry.php @@ -8,7 +8,7 @@ require_once("boot.php"); require_once("include/threads.php"); -function create_shadowentry_run(&$argv, &$argc) { +function create_shadowentry_run($argv, $argc) { global $a, $db; if (is_null($a))