diff --git a/bin/daemon.php b/bin/daemon.php index 047bf71be7..9051e0e6c3 100755 --- a/bin/daemon.php +++ b/bin/daemon.php @@ -144,7 +144,7 @@ if (!$foreground) { file_put_contents($pidfile, $pid); // We lose the database connection upon forking - Factory\DBFactory::init($a->getConfigCache(), $a->getProfiler(), $_SERVER); + $a->getDatabase()->reconnect(); } Config::set('system', 'worker_daemon_mode', true); diff --git a/include/dba.php b/include/dba.php index 2d26a94720..c56f63a84b 100644 --- a/include/dba.php +++ b/include/dba.php @@ -18,7 +18,7 @@ function q($sql) { $args = func_get_args(); unset($args[0]); - if (!DBA::$connected) { + if (!DBA::connected()) { return false; } diff --git a/include/text.php b/include/text.php index e4227cd7d0..7bed8b4994 100644 --- a/include/text.php +++ b/include/text.php @@ -140,16 +140,12 @@ function redir_private_images($a, &$item) * @brief Given a text string, convert from bbcode to html and add smilie icons. * * @param string $text String with bbcode. - * @return string Formattet HTML. + * @return string Formatted HTML * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ -function prepare_text($text) { - if (stristr($text, '[nosmile]')) { - $s = BBCode::convert($text); - } else { - $s = Smilies::replace(BBCode::convert($text)); - } - +function prepare_text($text) +{ + $s = BBCode::convert($text); return trim($s); } diff --git a/mod/message.php b/mod/message.php index 3ff84a1e66..fe4429e000 100644 --- a/mod/message.php +++ b/mod/message.php @@ -384,7 +384,7 @@ function message_content(App $a) $from_name_e = $message['from-name']; $subject_e = $message['title']; - $body_e = Smilies::replace(BBCode::convert($message['body'])); + $body_e = BBCode::convert($message['body']); $to_name_e = $message['name']; $contact = Contact::getDetailsByURL($message['from-url']); diff --git a/src/App.php b/src/App.php index f59194077c..7c2a003c79 100644 --- a/src/App.php +++ b/src/App.php @@ -12,6 +12,7 @@ use Friendica\Core\Config\Cache\IConfigCache; use Friendica\Core\Config\Configuration; use Friendica\Core\Hook; use Friendica\Core\Theme; +use Friendica\Database\Database; use Friendica\Database\DBA; use Friendica\Model\Profile; use Friendica\Network\HTTPException; @@ -122,6 +123,11 @@ class App */ private $profiler; + /** + * @var Database The Friendica database connection + */ + private $database; + /** * Returns the current config cache of this node * @@ -193,6 +199,14 @@ class App return $this->router; } + /** + * @return Database + */ + public function getDatabase() + { + return $this->database; + } + /** * Register a stylesheet file path to be included in the tag of every page. * Inclusion is done in App->initHead(). @@ -232,6 +246,7 @@ class App /** * @brief App constructor. * + * @param Database $database The Friendica Database * @param Configuration $config The Configuration * @param App\Mode $mode The mode of this Friendica app * @param App\Router $router The router of this Friendica app @@ -242,10 +257,11 @@ class App * * @throws Exception if the Basepath is not usable */ - public function __construct(Configuration $config, App\Mode $mode, App\Router $router, BaseURL $baseURL, LoggerInterface $logger, Profiler $profiler, $isBackend = true) + public function __construct(Database $database, Configuration $config, App\Mode $mode, App\Router $router, BaseURL $baseURL, LoggerInterface $logger, Profiler $profiler, $isBackend = true) { BaseObject::setApp($this); + $this->database = $database; $this->config = $config; $this->mode = $mode; $this->router = $router; diff --git a/src/Content/Smilies.php b/src/Content/Smilies.php index 57d14633ac..2bf232d090 100644 --- a/src/Content/Smilies.php +++ b/src/Content/Smilies.php @@ -267,17 +267,18 @@ class Smilies * @return string HTML Output * * @throws \Friendica\Network\HTTPException\InternalServerErrorException - * @todo : Rework because it doesn't work correctly */ private static function pregHeart($x) { if (strlen($x[1]) == 1) { return $x[0]; } + $t = ''; for ($cnt = 0; $cnt < strlen($x[1]); $cnt ++) { - $t .= '<3'; + $t .= '❤'; } + $r = str_replace($x[0], $t, $x[0]); return $r; } diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index e08d60579d..da09e13dd9 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1395,6 +1395,7 @@ class BBCode extends BaseObject // This is actually executed in Item::prepareBody() + $nosmile = strpos($text, '[nosmile]') !== false; $text = str_replace('[nosmile]', '', $text); // Check for font change text @@ -1572,7 +1573,7 @@ class BBCode extends BaseObject } // Replace non graphical smilies for external posts - if ($simple_html) { + if (!$nosmile && !$for_plaintext) { $text = Smilies::replace($text); } diff --git a/src/Database/DBA.php b/src/Database/DBA.php index 72769dca9b..6e9bc89be1 100644 --- a/src/Database/DBA.php +++ b/src/Database/DBA.php @@ -2,17 +2,11 @@ namespace Friendica\Database; -use Friendica\Core\Config\Cache\IConfigCache; -use Friendica\Core\System; -use Friendica\Util\DateTimeFormat; -use Friendica\Util\Profiler; use mysqli; use mysqli_result; use mysqli_stmt; use PDO; -use PDOException; use PDOStatement; -use Psr\Log\LoggerInterface; /** * @class MySQL database class @@ -30,133 +24,19 @@ class DBA */ const NULL_DATETIME = '0001-01-01 00:00:00'; - public static $connected = false; + /** + * @var Database + */ + private static $database; - /** - * @var IConfigCache - */ - private static $configCache; - /** - * @var Profiler - */ - private static $profiler; - /** - * @var LoggerInterface - */ - private static $logger; - private static $server_info = ''; - /** @var PDO|mysqli */ - private static $connection; - private static $driver; - private static $error = false; - private static $errorno = 0; - private static $affected_rows = 0; - private static $in_transaction = false; - private static $in_retrial = false; - private static $relation = []; - private static $db_serveraddr = ''; - private static $db_user = ''; - private static $db_pass = ''; - private static $db_name = ''; - private static $db_charset = ''; - - public static function connect(IConfigCache $configCache, Profiler $profiler, LoggerInterface $logger, $serveraddr, $user, $pass, $db, $charset = null) + public static function init(Database $database) { - if (!is_null(self::$connection) && self::connected()) { - return true; - } - - // We are storing these values for being able to perform a reconnect - self::$configCache = $configCache; - self::$profiler = $profiler; - self::$logger = $logger; - self::$db_serveraddr = $serveraddr; - self::$db_user = $user; - self::$db_pass = $pass; - self::$db_name = $db; - self::$db_charset = $charset; - - $port = 0; - $serveraddr = trim($serveraddr); - - $serverdata = explode(':', $serveraddr); - $server = $serverdata[0]; - - if (count($serverdata) > 1) { - $port = trim($serverdata[1]); - } - - $server = trim($server); - $user = trim($user); - $pass = trim($pass); - $db = trim($db); - $charset = trim($charset); - - if (!(strlen($server) && strlen($user))) { - return false; - } - - if (class_exists('\PDO') && in_array('mysql', PDO::getAvailableDrivers())) { - self::$driver = 'pdo'; - $connect = "mysql:host=".$server.";dbname=".$db; - - if ($port > 0) { - $connect .= ";port=".$port; - } - - if ($charset) { - $connect .= ";charset=".$charset; - } - - try { - self::$connection = @new PDO($connect, $user, $pass); - self::$connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); - self::$connected = true; - } catch (PDOException $e) { - /// @TODO At least log exception, don't ignore it! - } - } - - if (!self::$connected && class_exists('\mysqli')) { - self::$driver = 'mysqli'; - - if ($port > 0) { - self::$connection = @new mysqli($server, $user, $pass, $db, $port); - } else { - self::$connection = @new mysqli($server, $user, $pass, $db); - } - - if (!mysqli_connect_errno()) { - self::$connected = true; - - if ($charset) { - self::$connection->set_charset($charset); - } - } - } - - // No suitable SQL driver was found. - if (!self::$connected) { - self::$driver = null; - self::$connection = null; - } - - return self::$connected; + self::$database = $database; } - /** - * Sets the logger for DBA - * - * @note this is necessary because if we want to load the logger configuration - * from the DB, but there's an error, we would print out an exception. - * So the logger gets updated after the logger configuration can be retrieved - * from the database - * - * @param LoggerInterface $logger - */ - public static function setLogger(LoggerInterface $logger) + public static function connect() { - self::$logger = $logger; + return self::$database->connect(); } /** @@ -164,29 +44,15 @@ class DBA */ public static function disconnect() { - if (is_null(self::$connection)) { - return; - } - - switch (self::$driver) { - case 'pdo': - self::$connection = null; - break; - case 'mysqli': - self::$connection->close(); - self::$connection = null; - break; - } + self::$database->disconnect(); } /** * Perform a reconnect of an existing database connection */ - public static function reconnect() { - self::disconnect(); - - $ret = self::connect(self::$configCache, self::$profiler, self::$logger, self::$db_serveraddr, self::$db_user, self::$db_pass, self::$db_name, self::$db_charset); - return $ret; + public static function reconnect() + { + return self::$database->reconnect(); } /** @@ -195,7 +61,7 @@ class DBA */ public static function getConnection() { - return self::$connection; + return self::$database->getConnection(); } /** @@ -206,18 +72,9 @@ class DBA * * @return string */ - public static function serverInfo() { - if (self::$server_info == '') { - switch (self::$driver) { - case 'pdo': - self::$server_info = self::$connection->getAttribute(PDO::ATTR_SERVER_VERSION); - break; - case 'mysqli': - self::$server_info = self::$connection->server_info; - break; - } - } - return self::$server_info; + public static function serverInfo() + { + return self::$database->serverInfo(); } /** @@ -226,116 +83,19 @@ class DBA * @return string * @throws \Exception */ - public static function databaseName() { - $ret = self::p("SELECT DATABASE() AS `db`"); - $data = self::toArray($ret); - return $data[0]['db']; - } - - /** - * @brief Analyze a database query and log this if some conditions are met. - * - * @param string $query The database query that will be analyzed - * @throws \Exception - */ - private static function logIndex($query) { - - if (!self::$configCache->get('system', 'db_log_index')) { - return; - } - - // Don't explain an explain statement - if (strtolower(substr($query, 0, 7)) == "explain") { - return; - } - - // Only do the explain on "select", "update" and "delete" - if (!in_array(strtolower(substr($query, 0, 6)), ["select", "update", "delete"])) { - return; - } - - $r = self::p("EXPLAIN ".$query); - if (!self::isResult($r)) { - return; - } - - $watchlist = explode(',', self::$configCache->get('system', 'db_log_index_watch')); - $blacklist = explode(',', self::$configCache->get('system', 'db_log_index_blacklist')); - - while ($row = self::fetch($r)) { - if ((intval(self::$configCache->get('system', 'db_loglimit_index')) > 0)) { - $log = (in_array($row['key'], $watchlist) && - ($row['rows'] >= intval(self::$configCache->get('system', 'db_loglimit_index')))); - } else { - $log = false; - } - - if ((intval(self::$configCache->get('system', 'db_loglimit_index_high')) > 0) && ($row['rows'] >= intval(self::$configCache->get('system', 'db_loglimit_index_high')))) { - $log = true; - } - - if (in_array($row['key'], $blacklist) || ($row['key'] == "")) { - $log = false; - } - - if ($log) { - $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); - @file_put_contents(self::$configCache->get('system', 'db_log_index'), DateTimeFormat::utcNow()."\t". - $row['key']."\t".$row['rows']."\t".$row['Extra']."\t". - basename($backtrace[1]["file"])."\t". - $backtrace[1]["line"]."\t".$backtrace[2]["function"]."\t". - substr($query, 0, 2000)."\n", FILE_APPEND); - } - } - } - - /** - * Removes every not whitelisted character from the identifier string - * - * @param string $identifier - * - * @return string sanitized identifier - * @throws \Exception - */ - private static function sanitizeIdentifier($identifier) + public static function databaseName() { - return preg_replace('/[^A-Za-z0-9_\-]+/', '', $identifier); + return self::$database->databaseName(); } - public static function escape($str) { - if (self::$connected) { - switch (self::$driver) { - case 'pdo': - return substr(@self::$connection->quote($str, PDO::PARAM_STR), 1, -1); - - case 'mysqli': - return @self::$connection->real_escape_string($str); - } - } else { - return str_replace("'", "\\'", $str); - } + public static function escape($str) + { + return self::$database->escape($str); } - public static function connected() { - $connected = false; - - if (is_null(self::$connection)) { - return false; - } - - switch (self::$driver) { - case 'pdo': - $r = self::p("SELECT 1"); - if (self::isResult($r)) { - $row = self::toArray($r); - $connected = ($row[0]['1'] == '1'); - } - break; - case 'mysqli': - $connected = self::$connection->ping(); - break; - } - return $connected; + public static function connected() + { + return self::$database->connected(); } /** @@ -349,13 +109,9 @@ class DBA * @param string $sql An SQL string without the values * @return string The input SQL string modified if necessary. */ - public static function anyValueFallback($sql) { - $server_info = self::serverInfo(); - if (version_compare($server_info, '5.7.5', '<') || - (stripos($server_info, 'MariaDB') !== false)) { - $sql = str_ireplace('ANY_VALUE(', 'MIN(', $sql); - } - return $sql; + public static function anyValueFallback($sql) + { + return self::$database->anyValueFallback($sql); } /** @@ -367,7 +123,8 @@ class DBA * @param string $sql An SQL string without the values * @return string The input SQL string modified if necessary. */ - public static function cleanQuery($sql) { + public static function cleanQuery($sql) + { $search = ["\t", "\n", "\r", " "]; $replace = [' ', ' ', ' ', ' ']; do { @@ -378,38 +135,13 @@ class DBA return $sql; } - - /** - * @brief Replaces the ? placeholders with the parameters in the $args array - * - * @param string $sql SQL query - * @param array $args The parameters that are to replace the ? placeholders - * @return string The replaced SQL query - */ - private static function replaceParameters($sql, $args) { - $offset = 0; - foreach ($args AS $param => $value) { - if (is_int($args[$param]) || is_float($args[$param])) { - $replace = intval($args[$param]); - } else { - $replace = "'".self::escape($args[$param])."'"; - } - - $pos = strpos($sql, '?', $offset); - if ($pos !== false) { - $sql = substr_replace($sql, $replace, $pos, 1); - } - $offset = $pos + strlen($replace); - } - return $sql; - } - /** * @brief Convert parameter array to an universal form * @param array $args Parameter array * @return array universalized parameter array */ - private static function getParam($args) { + public static function getParam($args) + { unset($args[0]); // When the second function parameter is an array then use this as the parameter array @@ -431,227 +163,11 @@ class DBA * @return bool|object statement object or result object * @throws \Exception */ - public static function p($sql) { - - $stamp1 = microtime(true); - + public static function p($sql) + { $params = self::getParam(func_get_args()); - // Renumber the array keys to be sure that they fit - $i = 0; - $args = []; - foreach ($params AS $param) { - // Avoid problems with some MySQL servers and boolean values. See issue #3645 - if (is_bool($param)) { - $param = (int)$param; - } - $args[++$i] = $param; - } - - if (!self::$connected) { - return false; - } - - if ((substr_count($sql, '?') != count($args)) && (count($args) > 0)) { - // Question: Should we continue or stop the query here? - self::$logger->warning('Query parameters mismatch.', ['query' => $sql, 'args' => $args, 'callstack' => System::callstack()]); - } - - $sql = self::cleanQuery($sql); - $sql = self::anyValueFallback($sql); - - $orig_sql = $sql; - - if (self::$configCache->get('system', 'db_callstack') !== null) { - $sql = "/*".System::callstack()." */ ".$sql; - } - - self::$error = ''; - self::$errorno = 0; - self::$affected_rows = 0; - - // We have to make some things different if this function is called from "e" - $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 2); - - if (isset($trace[1])) { - $called_from = $trace[1]; - } else { - // We use just something that is defined to avoid warnings - $called_from = $trace[0]; - } - // We are having an own error logging in the function "e" - $called_from_e = ($called_from['function'] == 'e'); - - switch (self::$driver) { - case 'pdo': - // If there are no arguments we use "query" - if (count($args) == 0) { - if (!$retval = self::$connection->query($sql)) { - $errorInfo = self::$connection->errorInfo(); - self::$error = $errorInfo[2]; - self::$errorno = $errorInfo[1]; - $retval = false; - break; - } - self::$affected_rows = $retval->rowCount(); - break; - } - - /** @var $stmt mysqli_stmt|PDOStatement */ - if (!$stmt = self::$connection->prepare($sql)) { - $errorInfo = self::$connection->errorInfo(); - self::$error = $errorInfo[2]; - self::$errorno = $errorInfo[1]; - $retval = false; - break; - } - - foreach ($args AS $param => $value) { - if (is_int($args[$param])) { - $data_type = PDO::PARAM_INT; - } else { - $data_type = PDO::PARAM_STR; - } - $stmt->bindParam($param, $args[$param], $data_type); - } - - if (!$stmt->execute()) { - $errorInfo = $stmt->errorInfo(); - self::$error = $errorInfo[2]; - self::$errorno = $errorInfo[1]; - $retval = false; - } else { - $retval = $stmt; - self::$affected_rows = $retval->rowCount(); - } - break; - case 'mysqli': - // There are SQL statements that cannot be executed with a prepared statement - $parts = explode(' ', $orig_sql); - $command = strtolower($parts[0]); - $can_be_prepared = in_array($command, ['select', 'update', 'insert', 'delete']); - - // The fallback routine is called as well when there are no arguments - if (!$can_be_prepared || (count($args) == 0)) { - $retval = self::$connection->query(self::replaceParameters($sql, $args)); - if (self::$connection->errno) { - self::$error = self::$connection->error; - self::$errorno = self::$connection->errno; - $retval = false; - } else { - if (isset($retval->num_rows)) { - self::$affected_rows = $retval->num_rows; - } else { - self::$affected_rows = self::$connection->affected_rows; - } - } - break; - } - - $stmt = self::$connection->stmt_init(); - - if (!$stmt->prepare($sql)) { - self::$error = $stmt->error; - self::$errorno = $stmt->errno; - $retval = false; - break; - } - - $param_types = ''; - $values = []; - foreach ($args AS $param => $value) { - if (is_int($args[$param])) { - $param_types .= 'i'; - } elseif (is_float($args[$param])) { - $param_types .= 'd'; - } elseif (is_string($args[$param])) { - $param_types .= 's'; - } else { - $param_types .= 'b'; - } - $values[] = &$args[$param]; - } - - if (count($values) > 0) { - array_unshift($values, $param_types); - call_user_func_array([$stmt, 'bind_param'], $values); - } - - if (!$stmt->execute()) { - self::$error = self::$connection->error; - self::$errorno = self::$connection->errno; - $retval = false; - } else { - $stmt->store_result(); - $retval = $stmt; - self::$affected_rows = $retval->affected_rows; - } - break; - } - - // We are having an own error logging in the function "e" - if ((self::$errorno != 0) && !$called_from_e) { - // We have to preserve the error code, somewhere in the logging it get lost - $error = self::$error; - $errorno = self::$errorno; - - self::$logger->error('DB Error', [ - 'code' => self::$errorno, - 'error' => self::$error, - 'callstack' => System::callstack(8), - 'params' => self::replaceParameters($sql, $args), - ]); - - // On a lost connection we try to reconnect - but only once. - if ($errorno == 2006) { - if (self::$in_retrial || !self::reconnect()) { - // It doesn't make sense to continue when the database connection was lost - if (self::$in_retrial) { - self::$logger->notice('Giving up retrial because of database error', [ - 'code' => self::$errorno, - 'error' => self::$error, - ]); - } else { - self::$logger->notice('Couldn\'t reconnect after database error', [ - 'code' => self::$errorno, - 'error' => self::$error, - ]); - } - exit(1); - } else { - // We try it again - self::$logger->notice('Reconnected after database error', [ - 'code' => self::$errorno, - 'error' => self::$error, - ]); - self::$in_retrial = true; - $ret = self::p($sql, $args); - self::$in_retrial = false; - return $ret; - } - } - - self::$error = $error; - self::$errorno = $errorno; - } - - self::$profiler->saveTimestamp($stamp1, 'database', System::callstack()); - - if (self::$configCache->get('system', 'db_log')) { - $stamp2 = microtime(true); - $duration = (float)($stamp2 - $stamp1); - - if (($duration > self::$configCache->get('system', 'db_loglimit'))) { - $duration = round($duration, 3); - $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); - - @file_put_contents(self::$configCache->get('system', 'db_log'), DateTimeFormat::utcNow()."\t".$duration."\t". - basename($backtrace[1]["file"])."\t". - $backtrace[1]["line"]."\t".$backtrace[2]["function"]."\t". - substr(self::replaceParameters($sql, $args), 0, 2000)."\n", FILE_APPEND); - } - } - return $retval; + return self::$database->p($sql, $params); } /** @@ -665,57 +181,9 @@ class DBA */ public static function e($sql) { - $stamp = microtime(true); - $params = self::getParam(func_get_args()); - // In a case of a deadlock we are repeating the query 20 times - $timeout = 20; - - do { - $stmt = self::p($sql, $params); - - if (is_bool($stmt)) { - $retval = $stmt; - } elseif (is_object($stmt)) { - $retval = true; - } else { - $retval = false; - } - - self::close($stmt); - - } while ((self::$errorno == 1213) && (--$timeout > 0)); - - if (self::$errorno != 0) { - // We have to preserve the error code, somewhere in the logging it get lost - $error = self::$error; - $errorno = self::$errorno; - - self::$logger->error('DB Error', [ - 'code' => self::$errorno, - 'error' => self::$error, - 'callstack' => System::callstack(8), - 'params' => self::replaceParameters($sql, $params), - ]); - - // On a lost connection we simply quit. - // A reconnect like in self::p could be dangerous with modifications - if ($errorno == 2006) { - self::$logger->notice('Giving up because of database error', [ - 'code' => self::$errorno, - 'error' => self::$error, - ]); - exit(1); - } - - self::$error = $error; - self::$errorno = $errorno; - } - - self::$profiler->saveTimestamp($stamp, "database_write", System::callstack()); - - return $retval; + return self::$database->e($sql, $params); } /** @@ -727,34 +195,9 @@ class DBA * @return boolean Are there rows for that condition? * @throws \Exception */ - public static function exists($table, $condition) { - if (empty($table)) { - return false; - } - - $fields = []; - - if (empty($condition)) { - return DBStructure::existsTable($table); - } - - reset($condition); - $first_key = key($condition); - if (!is_int($first_key)) { - $fields = [$first_key]; - } - - $stmt = self::select($table, $fields, $condition, ['limit' => 1]); - - if (is_bool($stmt)) { - $retval = $stmt; - } else { - $retval = (self::numRows($stmt) > 0); - } - - self::close($stmt); - - return $retval; + public static function exists($table, $condition) + { + return self::$database->exists($table, $condition); } /** @@ -767,20 +210,11 @@ class DBA * @return array first row of query * @throws \Exception */ - public static function fetchFirst($sql) { + public static function fetchFirst($sql) + { $params = self::getParam(func_get_args()); - $stmt = self::p($sql, $params); - - if (is_bool($stmt)) { - $retval = $stmt; - } else { - $retval = self::fetch($stmt); - } - - self::close($stmt); - - return $retval; + return self::$database->fetchFirst($sql, $params); } /** @@ -788,8 +222,9 @@ class DBA * * @return int Number of rows */ - public static function affectedRows() { - return self::$affected_rows; + public static function affectedRows() + { + return self::$database->affectedRows(); } /** @@ -798,17 +233,9 @@ class DBA * @param object Statement object * @return int Number of columns */ - public static function columnCount($stmt) { - if (!is_object($stmt)) { - return 0; - } - switch (self::$driver) { - case 'pdo': - return $stmt->columnCount(); - case 'mysqli': - return $stmt->field_count; - } - return 0; + public static function columnCount($stmt) + { + return self::$database->columnCount($stmt); } /** * @brief Returns the number of rows of a statement @@ -816,17 +243,9 @@ class DBA * @param PDOStatement|mysqli_result|mysqli_stmt Statement object * @return int Number of rows */ - public static function numRows($stmt) { - if (!is_object($stmt)) { - return 0; - } - switch (self::$driver) { - case 'pdo': - return $stmt->rowCount(); - case 'mysqli': - return $stmt->num_rows; - } - return 0; + public static function numRows($stmt) + { + return self::$database->numRows($stmt); } /** @@ -835,79 +254,9 @@ class DBA * @param mixed $stmt statement object * @return array current row */ - public static function fetch($stmt) { - - $stamp1 = microtime(true); - - $columns = []; - - if (!is_object($stmt)) { - return false; - } - - switch (self::$driver) { - case 'pdo': - $columns = $stmt->fetch(PDO::FETCH_ASSOC); - break; - case 'mysqli': - if (get_class($stmt) == 'mysqli_result') { - $columns = $stmt->fetch_assoc(); - break; - } - - // This code works, but is slow - - // Bind the result to a result array - $cols = []; - - $cols_num = []; - for ($x = 0; $x < $stmt->field_count; $x++) { - $cols[] = &$cols_num[$x]; - } - - call_user_func_array([$stmt, 'bind_result'], $cols); - - if (!$stmt->fetch()) { - return false; - } - - // The slow part: - // We need to get the field names for the array keys - // It seems that there is no better way to do this. - $result = $stmt->result_metadata(); - $fields = $result->fetch_fields(); - - foreach ($cols_num AS $param => $col) { - $columns[$fields[$param]->name] = $col; - } - } - - self::$profiler->saveTimestamp($stamp1, 'database', System::callstack()); - - return $columns; - } - - /** - * @brief Insert a row into a table - * - * @param string/array $table Table name - * - * @return string formatted and sanitzed table name - * @throws \Exception - */ - public static function formatTableName($table) + public static function fetch($stmt) { - if (is_string($table)) { - return "`" . self::sanitizeIdentifier($table) . "`"; - } - - if (!is_array($table)) { - return ''; - } - - $scheme = key($table); - - return "`" . self::sanitizeIdentifier($scheme) . "`.`" . self::sanitizeIdentifier($table[$scheme]) . "`"; + return self::$database->fetch($stmt); } /** @@ -920,24 +269,9 @@ class DBA * @return boolean was the insert successful? * @throws \Exception */ - public static function insert($table, $param, $on_duplicate_update = false) { - - if (empty($table) || empty($param)) { - self::$logger->info('Table and fields have to be set'); - return false; - } - - $sql = "INSERT INTO " . self::formatTableName($table) . " (`".implode("`, `", array_keys($param))."`) VALUES (". - substr(str_repeat("?, ", count($param)), 0, -2).")"; - - if ($on_duplicate_update) { - $sql .= " ON DUPLICATE KEY UPDATE `".implode("` = ?, `", array_keys($param))."` = ?"; - - $values = array_values($param); - $param = array_merge_recursive($values, $values); - } - - return self::e($sql, $param); + public static function insert($table, $param, $on_duplicate_update = false) + { + return self::$database->insert($table, $param, $on_duplicate_update); } /** @@ -945,16 +279,9 @@ class DBA * * @return integer Last inserted id */ - public static function lastInsertId() { - switch (self::$driver) { - case 'pdo': - $id = self::$connection->lastInsertId(); - break; - case 'mysqli': - $id = self::$connection->insert_id; - break; - } - return $id; + public static function lastInsertId() + { + return self::$database->lastInsertId(); } /** @@ -967,31 +294,9 @@ class DBA * @return boolean was the lock successful? * @throws \Exception */ - public static function lock($table) { - // See here: https://dev.mysql.com/doc/refman/5.7/en/lock-tables-and-transactions.html - if (self::$driver == 'pdo') { - self::e("SET autocommit=0"); - self::$connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, true); - } else { - self::$connection->autocommit(false); - } - - $success = self::e("LOCK TABLES " . self::formatTableName($table) ." WRITE"); - - if (self::$driver == 'pdo') { - self::$connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); - } - - if (!$success) { - if (self::$driver == 'pdo') { - self::e("SET autocommit=1"); - } else { - self::$connection->autocommit(true); - } - } else { - self::$in_transaction = true; - } - return $success; + public static function lock($table) + { + return self::$database->lock($table); } /** @@ -1000,25 +305,9 @@ class DBA * @return boolean was the unlock successful? * @throws \Exception */ - public static function unlock() { - // See here: https://dev.mysql.com/doc/refman/5.7/en/lock-tables-and-transactions.html - self::performCommit(); - - if (self::$driver == 'pdo') { - self::$connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, true); - } - - $success = self::e("UNLOCK TABLES"); - - if (self::$driver == 'pdo') { - self::$connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); - self::e("SET autocommit=1"); - } else { - self::$connection->autocommit(true); - } - - self::$in_transaction = false; - return $success; + public static function unlock() + { + return self::$database->unlock(); } /** @@ -1026,44 +315,9 @@ class DBA * * @return boolean Was the command executed successfully? */ - public static function transaction() { - if (!self::performCommit()) { - return false; - } - - switch (self::$driver) { - case 'pdo': - if (!self::$connection->inTransaction() && !self::$connection->beginTransaction()) { - return false; - } - break; - - case 'mysqli': - if (!self::$connection->begin_transaction()) { - return false; - } - break; - } - - self::$in_transaction = true; - return true; - } - - private static function performCommit() + public static function transaction() { - switch (self::$driver) { - case 'pdo': - if (!self::$connection->inTransaction()) { - return true; - } - - return self::$connection->commit(); - - case 'mysqli': - return self::$connection->commit(); - } - - return true; + return self::$database->transaction(); } /** @@ -1071,12 +325,9 @@ class DBA * * @return boolean Was the command executed successfully? */ - public static function commit() { - if (!self::performCommit()) { - return false; - } - self::$in_transaction = false; - return true; + public static function commit() + { + return self::$database->commit(); } /** @@ -1084,45 +335,9 @@ class DBA * * @return boolean Was the command executed successfully? */ - public static function rollback() { - $ret = false; - - switch (self::$driver) { - case 'pdo': - if (!self::$connection->inTransaction()) { - $ret = true; - break; - } - $ret = self::$connection->rollBack(); - break; - - case 'mysqli': - $ret = self::$connection->rollback(); - break; - } - self::$in_transaction = false; - return $ret; - } - - /** - * @brief Build the array with the table relations - * - * The array is build from the database definitions in DBStructure.php - * - * This process must only be started once, since the value is cached. - */ - private static function buildRelationData() { - $definition = DBStructure::definition(self::$configCache->get('system', 'basepath')); - - foreach ($definition AS $table => $structure) { - foreach ($structure['fields'] AS $field => $field_struct) { - if (isset($field_struct['relation'])) { - foreach ($field_struct['relation'] AS $rel_table => $rel_field) { - self::$relation[$rel_table][$rel_field][$table][] = $field; - } - } - } - } + public static function rollback() + { + return self::$database->rollback(); } /** @@ -1133,145 +348,13 @@ class DBA * @param array $options * - cascade: If true we delete records in other tables that depend on the one we're deleting through * relations (default: true) - * @param array $callstack Internal use: prevent endless loops * * @return boolean was the delete successful? * @throws \Exception */ - public static function delete($table, array $conditions, array $options = [], array &$callstack = []) + public static function delete($table, array $conditions, array $options = []) { - if (empty($table) || empty($conditions)) { - self::$logger->info('Table and conditions have to be set'); - return false; - } - - $commands = []; - - // Create a key for the loop prevention - $key = $table . ':' . json_encode($conditions); - - // We quit when this key already exists in the callstack. - if (isset($callstack[$key])) { - return $commands; - } - - $callstack[$key] = true; - - $table = self::sanitizeIdentifier($table); - - $commands[$key] = ['table' => $table, 'conditions' => $conditions]; - - // Don't use "defaults" here, since it would set "false" to "true" - if (isset($options['cascade'])) { - $cascade = $options['cascade']; - } else { - $cascade = true; - } - - // To speed up the whole process we cache the table relations - if ($cascade && count(self::$relation) == 0) { - self::buildRelationData(); - } - - // Is there a relation entry for the table? - if ($cascade && isset(self::$relation[$table])) { - // We only allow a simple "one field" relation. - $field = array_keys(self::$relation[$table])[0]; - $rel_def = array_values(self::$relation[$table])[0]; - - // Create a key for preventing double queries - $qkey = $field . '-' . $table . ':' . json_encode($conditions); - - // When the search field is the relation field, we don't need to fetch the rows - // This is useful when the leading record is already deleted in the frontend but the rest is done in the backend - if ((count($conditions) == 1) && ($field == array_keys($conditions)[0])) { - foreach ($rel_def AS $rel_table => $rel_fields) { - foreach ($rel_fields AS $rel_field) { - self::delete($rel_table, [$rel_field => array_values($conditions)[0]], $options, $callstack); - } - } - // We quit when this key already exists in the callstack. - } elseif (!isset($callstack[$qkey])) { - $callstack[$qkey] = true; - - // Fetch all rows that are to be deleted - $data = self::select($table, [$field], $conditions); - - while ($row = self::fetch($data)) { - self::delete($table, [$field => $row[$field]], $options, $callstack); - } - - self::close($data); - - // Since we had split the delete command we don't need the original command anymore - unset($commands[$key]); - } - } - - // Now we finalize the process - $do_transaction = !self::$in_transaction; - - if ($do_transaction) { - self::transaction(); - } - - $compacted = []; - $counter = []; - - foreach ($commands AS $command) { - $conditions = $command['conditions']; - reset($conditions); - $first_key = key($conditions); - - $condition_string = self::buildCondition($conditions); - - if ((count($command['conditions']) > 1) || is_int($first_key)) { - $sql = "DELETE FROM `" . $command['table'] . "`" . $condition_string; - self::$logger->debug(self::replaceParameters($sql, $conditions)); - - if (!self::e($sql, $conditions)) { - if ($do_transaction) { - self::rollback(); - } - return false; - } - } else { - $key_table = $command['table']; - $key_condition = array_keys($command['conditions'])[0]; - $value = array_values($command['conditions'])[0]; - - // Split the SQL queries in chunks of 100 values - // We do the $i stuff here to make the code better readable - $i = isset($counter[$key_table][$key_condition]) ? $counter[$key_table][$key_condition] : 0; - if (isset($compacted[$key_table][$key_condition][$i]) && count($compacted[$key_table][$key_condition][$i]) > 100) { - ++$i; - } - - $compacted[$key_table][$key_condition][$i][$value] = $value; - $counter[$key_table][$key_condition] = $i; - } - } - foreach ($compacted AS $table => $values) { - foreach ($values AS $field => $field_value_list) { - foreach ($field_value_list AS $field_values) { - $sql = "DELETE FROM `" . $table . "` WHERE `" . $field . "` IN (" . - substr(str_repeat("?, ", count($field_values)), 0, -2) . ");"; - - self::$logger->debug(self::replaceParameters($sql, $field_values)); - - if (!self::e($sql, $field_values)) { - if ($do_transaction) { - self::rollback(); - } - return false; - } - } - } - } - if ($do_transaction) { - self::commit(); - } - return true; + return self::$database->delete($table, $conditions, $options); } /** @@ -1303,53 +386,9 @@ class DBA * @return boolean was the update successfull? * @throws \Exception */ - public static function update($table, $fields, $condition, $old_fields = []) { - - if (empty($table) || empty($fields) || empty($condition)) { - self::$logger->info('Table, fields and condition have to be set'); - return false; - } - - $condition_string = self::buildCondition($condition); - - if (is_bool($old_fields)) { - $do_insert = $old_fields; - - $old_fields = self::selectFirst($table, [], $condition); - - if (is_bool($old_fields)) { - if ($do_insert) { - $values = array_merge($condition, $fields); - return self::insert($table, $values, $do_insert); - } - $old_fields = []; - } - } - - $do_update = (count($old_fields) == 0); - - foreach ($old_fields AS $fieldname => $content) { - if (isset($fields[$fieldname])) { - if (($fields[$fieldname] == $content) && !is_null($content)) { - unset($fields[$fieldname]); - } else { - $do_update = true; - } - } - } - - if (!$do_update || (count($fields) == 0)) { - return true; - } - - $sql = "UPDATE ". self::formatTableName($table) . " SET `". - implode("` = ?, `", array_keys($fields))."` = ?".$condition_string; - - $params1 = array_values($fields); - $params2 = array_values($condition); - $params = array_merge_recursive($params1, $params2); - - return self::e($sql, $params); + public static function update($table, $fields, $condition, $old_fields = []) + { + return self::$database->update($table, $fields, $condition, $old_fields); } /** @@ -1366,16 +405,7 @@ class DBA */ public static function selectFirst($table, array $fields = [], array $condition = [], $params = []) { - $params['limit'] = 1; - $result = self::select($table, $fields, $condition, $params); - - if (is_bool($result)) { - return $result; - } else { - $row = self::fetch($result); - self::close($result); - return $row; - } + return self::$database->selectFirst($table, $fields, $condition, $params); } /** @@ -1403,25 +433,7 @@ class DBA */ public static function select($table, array $fields = [], array $condition = [], array $params = []) { - if (empty($table)) { - return false; - } - - if (count($fields) > 0) { - $select_fields = "`" . implode("`, `", array_values($fields)) . "`"; - } else { - $select_fields = "*"; - } - - $condition_string = self::buildCondition($condition); - - $param_string = self::buildParameter($params); - - $sql = "SELECT " . $select_fields . " FROM " . self::formatTableName($table) . $condition_string . $param_string; - - $result = self::p($sql, $condition); - - return $result; + return self::$database->select($table, $fields, $condition, $params); } /** @@ -1444,17 +456,7 @@ class DBA */ public static function count($table, array $condition = []) { - if (empty($table)) { - return false; - } - - $condition_string = self::buildCondition($condition); - - $sql = "SELECT COUNT(*) AS `count` FROM " . self::formatTableName($table) . $condition_string; - - $row = self::fetchFirst($sql, $condition); - - return $row['count']; + return self::$database->count($table, $condition); } /** @@ -1584,19 +586,9 @@ class DBA * @param bool $do_close * @return array Data array */ - public static function toArray($stmt, $do_close = true) { - if (is_bool($stmt)) { - return $stmt; - } - - $data = []; - while ($row = self::fetch($stmt)) { - $data[] = $row; - } - if ($do_close) { - self::close($stmt); - } - return $data; + public static function toArray($stmt, $do_close = true) + { + return self::$database->toArray($stmt, $do_close); } /** @@ -1604,8 +596,9 @@ class DBA * * @return string Error number (0 if no error) */ - public static function errorNo() { - return self::$errorno; + public static function errorNo() + { + return self::$database->errorNo(); } /** @@ -1613,8 +606,9 @@ class DBA * * @return string Error message ('' if no error) */ - public static function errorMessage() { - return self::$error; + public static function errorMessage() + { + return self::$database->errorMessage(); } /** @@ -1623,37 +617,9 @@ class DBA * @param object $stmt statement object * @return boolean was the close successful? */ - public static function close($stmt) { - - $stamp1 = microtime(true); - - if (!is_object($stmt)) { - return false; - } - - switch (self::$driver) { - case 'pdo': - $ret = $stmt->closeCursor(); - break; - case 'mysqli': - // MySQLi offers both a mysqli_stmt and a mysqli_result class. - // We should be careful not to assume the object type of $stmt - // because DBA::p() has been able to return both types. - if ($stmt instanceof mysqli_stmt) { - $stmt->free_result(); - $ret = $stmt->close(); - } elseif ($stmt instanceof mysqli_result) { - $stmt->free(); - $ret = true; - } else { - $ret = false; - } - break; - } - - self::$profiler->saveTimestamp($stamp1, 'database', System::callstack()); - - return $ret; + public static function close($stmt) + { + return self::$database->close($stmt); } /** @@ -1666,29 +632,7 @@ class DBA */ public static function processlist() { - $ret = self::p("SHOW PROCESSLIST"); - $data = self::toArray($ret); - - $processes = 0; - $states = []; - foreach ($data as $process) { - $state = trim($process["State"]); - - // Filter out all non blocking processes - if (!in_array($state, ["", "init", "statistics", "updating"])) { - ++$states[$state]; - ++$processes; - } - } - - $statelist = ""; - foreach ($states as $state => $usage) { - if ($statelist != "") { - $statelist .= ", "; - } - $statelist .= $state.": ".$usage; - } - return(["list" => $statelist, "amount" => $processes]); + return self::$database->processlist(); } /** @@ -1700,44 +644,7 @@ class DBA */ public static function isResult($array) { - // It could be a return value from an update statement - if (is_bool($array)) { - return $array; - } - - if (is_object($array)) { - return self::numRows($array) > 0; - } - - return (is_array($array) && (count($array) > 0)); - } - - /** - * @brief Callback function for "esc_array" - * - * @param mixed $value Array value - * @param string $key Array key - * @param boolean $add_quotation add quotation marks for string values - * @return void - */ - private static function escapeArrayCallback(&$value, $key, $add_quotation) - { - if (!$add_quotation) { - if (is_bool($value)) { - $value = ($value ? '1' : '0'); - } else { - $value = self::escape($value); - } - return; - } - - if (is_bool($value)) { - $value = ($value ? 'true' : 'false'); - } elseif (is_float($value) || is_integer($value)) { - $value = (string) $value; - } else { - $value = "'" . self::escape($value) . "'"; - } + return self::$database->isResult($array); } /** @@ -1749,6 +656,6 @@ class DBA */ public static function escapeArray(&$arr, $add_quotation = false) { - array_walk($arr, 'self::escapeArrayCallback', $add_quotation); + return self::$database->escapeArray($arr, $add_quotation); } } diff --git a/src/Database/Database.php b/src/Database/Database.php new file mode 100644 index 0000000000..38406b6bad --- /dev/null +++ b/src/Database/Database.php @@ -0,0 +1,1644 @@ +configCache = $configCache; + $this->profiler = $profiler; + $this->logger = $logger; + $this->db_serveraddr = $serveraddr; + $this->db_user = $user; + $this->db_pass = $pass; + $this->db_name = $db; + $this->db_charset = $charset; + + $this->connect(); + + DBA::init($this); + } + + public function connect() + { + if (!is_null($this->connection) && $this->connected()) { + return true; + } + + $port = 0; + $serveraddr = trim($this->db_serveraddr); + + $serverdata = explode(':', $serveraddr); + $server = $serverdata[0]; + + if (count($serverdata) > 1) { + $port = trim($serverdata[1]); + } + + $server = trim($server); + $user = trim($this->db_user); + $pass = trim($this->db_pass); + $db = trim($this->db_name); + $charset = trim($this->db_charset); + + if (!(strlen($server) && strlen($user))) { + return false; + } + + if (class_exists('\PDO') && in_array('mysql', PDO::getAvailableDrivers())) { + $this->driver = 'pdo'; + $connect = "mysql:host=" . $server . ";dbname=" . $db; + + if ($port > 0) { + $connect .= ";port=" . $port; + } + + if ($charset) { + $connect .= ";charset=" . $charset; + } + + try { + $this->connection = @new PDO($connect, $user, $pass); + $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); + $this->connected = true; + } catch (PDOException $e) { + /// @TODO At least log exception, don't ignore it! + } + } + + if (!$this->connected && class_exists('\mysqli')) { + $this->driver = 'mysqli'; + + if ($port > 0) { + $this->connection = @new mysqli($server, $user, $pass, $db, $port); + } else { + $this->connection = @new mysqli($server, $user, $pass, $db); + } + + if (!mysqli_connect_errno()) { + $this->connected = true; + + if ($charset) { + $this->connection->set_charset($charset); + } + } + } + + // No suitable SQL driver was found. + if (!$this->connected) { + $this->driver = null; + $this->connection = null; + } + + return $this->connected; + } + + /** + * Sets the logger for DBA + * + * @note this is necessary because if we want to load the logger configuration + * from the DB, but there's an error, we would print out an exception. + * So the logger gets updated after the logger configuration can be retrieved + * from the database + * + * @param LoggerInterface $logger + */ + public function setLogger(LoggerInterface $logger) + { + $this->logger = $logger; + } + + /** + * Disconnects the current database connection + */ + public function disconnect() + { + if (is_null($this->connection)) { + return; + } + + switch ($this->driver) { + case 'pdo': + $this->connection = null; + break; + case 'mysqli': + $this->connection->close(); + $this->connection = null; + break; + } + } + + /** + * Perform a reconnect of an existing database connection + */ + public function reconnect() + { + $this->disconnect(); + return $this->connect(); + } + + /** + * Return the database object. + * + * @return PDO|mysqli + */ + public function getConnection() + { + return $this->connection; + } + + /** + * @brief Returns the MySQL server version string + * + * This function discriminate between the deprecated mysql API and the current + * object-oriented mysqli API. Example of returned string: 5.5.46-0+deb8u1 + * + * @return string + */ + public function serverInfo() + { + if ($this->server_info == '') { + switch ($this->driver) { + case 'pdo': + $this->server_info = $this->connection->getAttribute(PDO::ATTR_SERVER_VERSION); + break; + case 'mysqli': + $this->server_info = $this->connection->server_info; + break; + } + } + return $this->server_info; + } + + /** + * @brief Returns the selected database name + * + * @return string + * @throws \Exception + */ + public function databaseName() + { + $ret = $this->p("SELECT DATABASE() AS `db`"); + $data = $this->toArray($ret); + return $data[0]['db']; + } + + /** + * @brief Analyze a database query and log this if some conditions are met. + * + * @param string $query The database query that will be analyzed + * + * @throws \Exception + */ + private function logIndex($query) + { + + if (!$this->configCache->get('system', 'db_log_index')) { + return; + } + + // Don't explain an explain statement + if (strtolower(substr($query, 0, 7)) == "explain") { + return; + } + + // Only do the explain on "select", "update" and "delete" + if (!in_array(strtolower(substr($query, 0, 6)), ["select", "update", "delete"])) { + return; + } + + $r = $this->p("EXPLAIN " . $query); + if (!$this->isResult($r)) { + return; + } + + $watchlist = explode(',', $this->configCache->get('system', 'db_log_index_watch')); + $blacklist = explode(',', $this->configCache->get('system', 'db_log_index_blacklist')); + + while ($row = $this->fetch($r)) { + if ((intval($this->configCache->get('system', 'db_loglimit_index')) > 0)) { + $log = (in_array($row['key'], $watchlist) && + ($row['rows'] >= intval($this->configCache->get('system', 'db_loglimit_index')))); + } else { + $log = false; + } + + if ((intval($this->configCache->get('system', 'db_loglimit_index_high')) > 0) && ($row['rows'] >= intval($this->configCache->get('system', 'db_loglimit_index_high')))) { + $log = true; + } + + if (in_array($row['key'], $blacklist) || ($row['key'] == "")) { + $log = false; + } + + if ($log) { + $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); + @file_put_contents($this->configCache->get('system', 'db_log_index'), DateTimeFormat::utcNow() . "\t" . + $row['key'] . "\t" . $row['rows'] . "\t" . $row['Extra'] . "\t" . + basename($backtrace[1]["file"]) . "\t" . + $backtrace[1]["line"] . "\t" . $backtrace[2]["function"] . "\t" . + substr($query, 0, 2000) . "\n", FILE_APPEND); + } + } + } + + /** + * Removes every not whitelisted character from the identifier string + * + * @param string $identifier + * + * @return string sanitized identifier + * @throws \Exception + */ + private function sanitizeIdentifier($identifier) + { + return preg_replace('/[^A-Za-z0-9_\-]+/', '', $identifier); + } + + public function escape($str) + { + if ($this->connected) { + switch ($this->driver) { + case 'pdo': + return substr(@$this->connection->quote($str, PDO::PARAM_STR), 1, -1); + + case 'mysqli': + return @$this->connection->real_escape_string($str); + } + } else { + return str_replace("'", "\\'", $str); + } + } + + public function connected() + { + $connected = false; + + if (is_null($this->connection)) { + return false; + } + + switch ($this->driver) { + case 'pdo': + $r = $this->p("SELECT 1"); + if ($this->isResult($r)) { + $row = $this->toArray($r); + $connected = ($row[0]['1'] == '1'); + } + break; + case 'mysqli': + $connected = $this->connection->ping(); + break; + } + return $connected; + } + + /** + * @brief Replaces ANY_VALUE() function by MIN() function, + * if the database server does not support ANY_VALUE(). + * + * Considerations for Standard SQL, or MySQL with ONLY_FULL_GROUP_BY (default since 5.7.5). + * ANY_VALUE() is available from MySQL 5.7.5 https://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html + * A standard fall-back is to use MIN(). + * + * @param string $sql An SQL string without the values + * + * @return string The input SQL string modified if necessary. + */ + public function anyValueFallback($sql) + { + $server_info = $this->serverInfo(); + if (version_compare($server_info, '5.7.5', '<') || + (stripos($server_info, 'MariaDB') !== false)) { + $sql = str_ireplace('ANY_VALUE(', 'MIN(', $sql); + } + return $sql; + } + + /** + * @brief Replaces the ? placeholders with the parameters in the $args array + * + * @param string $sql SQL query + * @param array $args The parameters that are to replace the ? placeholders + * + * @return string The replaced SQL query + */ + private function replaceParameters($sql, $args) + { + $offset = 0; + foreach ($args AS $param => $value) { + if (is_int($args[$param]) || is_float($args[$param])) { + $replace = intval($args[$param]); + } else { + $replace = "'" . $this->escape($args[$param]) . "'"; + } + + $pos = strpos($sql, '?', $offset); + if ($pos !== false) { + $sql = substr_replace($sql, $replace, $pos, 1); + } + $offset = $pos + strlen($replace); + } + return $sql; + } + + /** + * @brief Executes a prepared statement that returns data + * @usage Example: $r = p("SELECT * FROM `item` WHERE `guid` = ?", $guid); + * + * Please only use it with complicated queries. + * For all regular queries please use DBA::select or DBA::exists + * + * @param string $sql SQL statement + * + * @return bool|object statement object or result object + * @throws \Exception + */ + public function p($sql) + { + + $stamp1 = microtime(true); + + $params = DBA::getParam(func_get_args()); + + // Renumber the array keys to be sure that they fit + $i = 0; + $args = []; + foreach ($params AS $param) { + // Avoid problems with some MySQL servers and boolean values. See issue #3645 + if (is_bool($param)) { + $param = (int)$param; + } + $args[++$i] = $param; + } + + if (!$this->connected) { + return false; + } + + if ((substr_count($sql, '?') != count($args)) && (count($args) > 0)) { + // Question: Should we continue or stop the query here? + $this->logger->warning('Query parameters mismatch.', ['query' => $sql, 'args' => $args, 'callstack' => System::callstack()]); + } + + $sql = DBA::cleanQuery($sql); + $sql = $this->anyValueFallback($sql); + + $orig_sql = $sql; + + if ($this->configCache->get('system', 'db_callstack') !== null) { + $sql = "/*" . System::callstack() . " */ " . $sql; + } + + $this->error = ''; + $this->errorno = 0; + $this->affected_rows = 0; + + // We have to make some things different if this function is called from "e" + $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 2); + + if (isset($trace[1])) { + $called_from = $trace[1]; + } else { + // We use just something that is defined to avoid warnings + $called_from = $trace[0]; + } + // We are having an own error logging in the function "e" + $called_from_e = ($called_from['function'] == 'e'); + + switch ($this->driver) { + case 'pdo': + // If there are no arguments we use "query" + if (count($args) == 0) { + if (!$retval = $this->connection->query($sql)) { + $errorInfo = $this->connection->errorInfo(); + $this->error = $errorInfo[2]; + $this->errorno = $errorInfo[1]; + $retval = false; + break; + } + $this->affected_rows = $retval->rowCount(); + break; + } + + /** @var $stmt mysqli_stmt|PDOStatement */ + if (!$stmt = $this->connection->prepare($sql)) { + $errorInfo = $this->connection->errorInfo(); + $this->error = $errorInfo[2]; + $this->errorno = $errorInfo[1]; + $retval = false; + break; + } + + foreach ($args AS $param => $value) { + if (is_int($args[$param])) { + $data_type = PDO::PARAM_INT; + } else { + $data_type = PDO::PARAM_STR; + } + $stmt->bindParam($param, $args[$param], $data_type); + } + + if (!$stmt->execute()) { + $errorInfo = $stmt->errorInfo(); + $this->error = $errorInfo[2]; + $this->errorno = $errorInfo[1]; + $retval = false; + } else { + $retval = $stmt; + $this->affected_rows = $retval->rowCount(); + } + break; + case 'mysqli': + // There are SQL statements that cannot be executed with a prepared statement + $parts = explode(' ', $orig_sql); + $command = strtolower($parts[0]); + $can_be_prepared = in_array($command, ['select', 'update', 'insert', 'delete']); + + // The fallback routine is called as well when there are no arguments + if (!$can_be_prepared || (count($args) == 0)) { + $retval = $this->connection->query($this->replaceParameters($sql, $args)); + if ($this->connection->errno) { + $this->error = $this->connection->error; + $this->errorno = $this->connection->errno; + $retval = false; + } else { + if (isset($retval->num_rows)) { + $this->affected_rows = $retval->num_rows; + } else { + $this->affected_rows = $this->connection->affected_rows; + } + } + break; + } + + $stmt = $this->connection->stmt_init(); + + if (!$stmt->prepare($sql)) { + $this->error = $stmt->error; + $this->errorno = $stmt->errno; + $retval = false; + break; + } + + $param_types = ''; + $values = []; + foreach ($args AS $param => $value) { + if (is_int($args[$param])) { + $param_types .= 'i'; + } elseif (is_float($args[$param])) { + $param_types .= 'd'; + } elseif (is_string($args[$param])) { + $param_types .= 's'; + } else { + $param_types .= 'b'; + } + $values[] = &$args[$param]; + } + + if (count($values) > 0) { + array_unshift($values, $param_types); + call_user_func_array([$stmt, 'bind_param'], $values); + } + + if (!$stmt->execute()) { + $this->error = $this->connection->error; + $this->errorno = $this->connection->errno; + $retval = false; + } else { + $stmt->store_result(); + $retval = $stmt; + $this->affected_rows = $retval->affected_rows; + } + break; + } + + // We are having an own error logging in the function "e" + if (($this->errorno != 0) && !$called_from_e) { + // We have to preserve the error code, somewhere in the logging it get lost + $error = $this->error; + $errorno = $this->errorno; + + $this->logger->error('DB Error', [ + 'code' => $this->errorno, + 'error' => $this->error, + 'callstack' => System::callstack(8), + 'params' => $this->replaceParameters($sql, $args), + ]); + + // On a lost connection we try to reconnect - but only once. + if ($errorno == 2006) { + if ($this->in_retrial || !$this->reconnect()) { + // It doesn't make sense to continue when the database connection was lost + if ($this->in_retrial) { + $this->logger->notice('Giving up retrial because of database error', [ + 'code' => $this->errorno, + 'error' => $this->error, + ]); + } else { + $this->logger->notice('Couldn\'t reconnect after database error', [ + 'code' => $this->errorno, + 'error' => $this->error, + ]); + } + exit(1); + } else { + // We try it again + $this->logger->notice('Reconnected after database error', [ + 'code' => $this->errorno, + 'error' => $this->error, + ]); + $this->in_retrial = true; + $ret = $this->p($sql, $args); + $this->in_retrial = false; + return $ret; + } + } + + $this->error = $error; + $this->errorno = $errorno; + } + + $this->profiler->saveTimestamp($stamp1, 'database', System::callstack()); + + if ($this->configCache->get('system', 'db_log')) { + $stamp2 = microtime(true); + $duration = (float)($stamp2 - $stamp1); + + if (($duration > $this->configCache->get('system', 'db_loglimit'))) { + $duration = round($duration, 3); + $backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS); + + @file_put_contents($this->configCache->get('system', 'db_log'), DateTimeFormat::utcNow() . "\t" . $duration . "\t" . + basename($backtrace[1]["file"]) . "\t" . + $backtrace[1]["line"] . "\t" . $backtrace[2]["function"] . "\t" . + substr($this->replaceParameters($sql, $args), 0, 2000) . "\n", FILE_APPEND); + } + } + return $retval; + } + + /** + * @brief Executes a prepared statement like UPDATE or INSERT that doesn't return data + * + * Please use DBA::delete, DBA::insert, DBA::update, ... instead + * + * @param string $sql SQL statement + * + * @return boolean Was the query successfull? False is returned only if an error occurred + * @throws \Exception + */ + public function e($sql) + { + + $stamp = microtime(true); + + $params = DBA::getParam(func_get_args()); + + // In a case of a deadlock we are repeating the query 20 times + $timeout = 20; + + do { + $stmt = $this->p($sql, $params); + + if (is_bool($stmt)) { + $retval = $stmt; + } elseif (is_object($stmt)) { + $retval = true; + } else { + $retval = false; + } + + $this->close($stmt); + + } while (($this->errorno == 1213) && (--$timeout > 0)); + + if ($this->errorno != 0) { + // We have to preserve the error code, somewhere in the logging it get lost + $error = $this->error; + $errorno = $this->errorno; + + $this->logger->error('DB Error', [ + 'code' => $this->errorno, + 'error' => $this->error, + 'callstack' => System::callstack(8), + 'params' => $this->replaceParameters($sql, $params), + ]); + + // On a lost connection we simply quit. + // A reconnect like in $this->p could be dangerous with modifications + if ($errorno == 2006) { + $this->logger->notice('Giving up because of database error', [ + 'code' => $this->errorno, + 'error' => $this->error, + ]); + exit(1); + } + + $this->error = $error; + $this->errorno = $errorno; + } + + $this->profiler->saveTimestamp($stamp, "database_write", System::callstack()); + + return $retval; + } + + /** + * @brief Check if data exists + * + * @param string $table Table name + * @param array $condition array of fields for condition + * + * @return boolean Are there rows for that condition? + * @throws \Exception + */ + public function exists($table, $condition) + { + if (empty($table)) { + return false; + } + + $fields = []; + + if (empty($condition)) { + return DBStructure::existsTable($table); + } + + reset($condition); + $first_key = key($condition); + if (!is_int($first_key)) { + $fields = [$first_key]; + } + + $stmt = $this->select($table, $fields, $condition, ['limit' => 1]); + + if (is_bool($stmt)) { + $retval = $stmt; + } else { + $retval = ($this->numRows($stmt) > 0); + } + + $this->close($stmt); + + return $retval; + } + + /** + * Fetches the first row + * + * Please use DBA::selectFirst or DBA::exists whenever this is possible. + * + * @brief Fetches the first row + * + * @param string $sql SQL statement + * + * @return array first row of query + * @throws \Exception + */ + public function fetchFirst($sql) + { + $params = DBA::getParam(func_get_args()); + + $stmt = $this->p($sql, $params); + + if (is_bool($stmt)) { + $retval = $stmt; + } else { + $retval = $this->fetch($stmt); + } + + $this->close($stmt); + + return $retval; + } + + /** + * @brief Returns the number of affected rows of the last statement + * + * @return int Number of rows + */ + public function affectedRows() + { + return $this->affected_rows; + } + + /** + * @brief Returns the number of columns of a statement + * + * @param object Statement object + * + * @return int Number of columns + */ + public function columnCount($stmt) + { + if (!is_object($stmt)) { + return 0; + } + switch ($this->driver) { + case 'pdo': + return $stmt->columnCount(); + case 'mysqli': + return $stmt->field_count; + } + return 0; + } + + /** + * @brief Returns the number of rows of a statement + * + * @param PDOStatement|mysqli_result|mysqli_stmt Statement object + * + * @return int Number of rows + */ + public function numRows($stmt) + { + if (!is_object($stmt)) { + return 0; + } + switch ($this->driver) { + case 'pdo': + return $stmt->rowCount(); + case 'mysqli': + return $stmt->num_rows; + } + return 0; + } + + /** + * @brief Fetch a single row + * + * @param mixed $stmt statement object + * + * @return array current row + */ + public function fetch($stmt) + { + + $stamp1 = microtime(true); + + $columns = []; + + if (!is_object($stmt)) { + return false; + } + + switch ($this->driver) { + case 'pdo': + $columns = $stmt->fetch(PDO::FETCH_ASSOC); + break; + case 'mysqli': + if (get_class($stmt) == 'mysqli_result') { + $columns = $stmt->fetch_assoc(); + break; + } + + // This code works, but is slow + + // Bind the result to a result array + $cols = []; + + $cols_num = []; + for ($x = 0; $x < $stmt->field_count; $x++) { + $cols[] = &$cols_num[$x]; + } + + call_user_func_array([$stmt, 'bind_result'], $cols); + + if (!$stmt->fetch()) { + return false; + } + + // The slow part: + // We need to get the field names for the array keys + // It seems that there is no better way to do this. + $result = $stmt->result_metadata(); + $fields = $result->fetch_fields(); + + foreach ($cols_num AS $param => $col) { + $columns[$fields[$param]->name] = $col; + } + } + + $this->profiler->saveTimestamp($stamp1, 'database', System::callstack()); + + return $columns; + } + + /** + * @brief Insert a row into a table + * + * @param string/array $table Table name + * + * @return string formatted and sanitzed table name + * @throws \Exception + */ + public function formatTableName($table) + { + if (is_string($table)) { + return "`" . $this->sanitizeIdentifier($table) . "`"; + } + + if (!is_array($table)) { + return ''; + } + + $scheme = key($table); + + return "`" . $this->sanitizeIdentifier($scheme) . "`.`" . $this->sanitizeIdentifier($table[$scheme]) . "`"; + } + + /** + * @brief Insert a row into a table + * + * @param string $table Table name + * @param array $param parameter array + * @param bool $on_duplicate_update Do an update on a duplicate entry + * + * @return boolean was the insert successful? + * @throws \Exception + */ + public function insert($table, $param, $on_duplicate_update = false) + { + + if (empty($table) || empty($param)) { + $this->logger->info('Table and fields have to be set'); + return false; + } + + $sql = "INSERT INTO " . $this->formatTableName($table) . " (`" . implode("`, `", array_keys($param)) . "`) VALUES (" . + substr(str_repeat("?, ", count($param)), 0, -2) . ")"; + + if ($on_duplicate_update) { + $sql .= " ON DUPLICATE KEY UPDATE `" . implode("` = ?, `", array_keys($param)) . "` = ?"; + + $values = array_values($param); + $param = array_merge_recursive($values, $values); + } + + return $this->e($sql, $param); + } + + /** + * @brief Fetch the id of the last insert command + * + * @return integer Last inserted id + */ + public function lastInsertId() + { + switch ($this->driver) { + case 'pdo': + $id = $this->connection->lastInsertId(); + break; + case 'mysqli': + $id = $this->connection->insert_id; + break; + } + return $id; + } + + /** + * @brief Locks a table for exclusive write access + * + * This function can be extended in the future to accept a table array as well. + * + * @param string $table Table name + * + * @return boolean was the lock successful? + * @throws \Exception + */ + public function lock($table) + { + // See here: https://dev.mysql.com/doc/refman/5.7/en/lock-tables-and-transactions.html + if ($this->driver == 'pdo') { + $this->e("SET autocommit=0"); + $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, true); + } else { + $this->connection->autocommit(false); + } + + $success = $this->e("LOCK TABLES " . $this->formatTableName($table) . " WRITE"); + + if ($this->driver == 'pdo') { + $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); + } + + if (!$success) { + if ($this->driver == 'pdo') { + $this->e("SET autocommit=1"); + } else { + $this->connection->autocommit(true); + } + } else { + $this->in_transaction = true; + } + return $success; + } + + /** + * @brief Unlocks all locked tables + * + * @return boolean was the unlock successful? + * @throws \Exception + */ + public function unlock() + { + // See here: https://dev.mysql.com/doc/refman/5.7/en/lock-tables-and-transactions.html + $this->performCommit(); + + if ($this->driver == 'pdo') { + $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, true); + } + + $success = $this->e("UNLOCK TABLES"); + + if ($this->driver == 'pdo') { + $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); + $this->e("SET autocommit=1"); + } else { + $this->connection->autocommit(true); + } + + $this->in_transaction = false; + return $success; + } + + /** + * @brief Starts a transaction + * + * @return boolean Was the command executed successfully? + */ + public function transaction() + { + if (!$this->performCommit()) { + return false; + } + + switch ($this->driver) { + case 'pdo': + if (!$this->connection->inTransaction() && !$this->connection->beginTransaction()) { + return false; + } + break; + + case 'mysqli': + if (!$this->connection->begin_transaction()) { + return false; + } + break; + } + + $this->in_transaction = true; + return true; + } + + private function performCommit() + { + switch ($this->driver) { + case 'pdo': + if (!$this->connection->inTransaction()) { + return true; + } + + return $this->connection->commit(); + + case 'mysqli': + return $this->connection->commit(); + } + + return true; + } + + /** + * @brief Does a commit + * + * @return boolean Was the command executed successfully? + */ + public function commit() + { + if (!$this->performCommit()) { + return false; + } + $this->in_transaction = false; + return true; + } + + /** + * @brief Does a rollback + * + * @return boolean Was the command executed successfully? + */ + public function rollback() + { + $ret = false; + + switch ($this->driver) { + case 'pdo': + if (!$this->connection->inTransaction()) { + $ret = true; + break; + } + $ret = $this->connection->rollBack(); + break; + + case 'mysqli': + $ret = $this->connection->rollback(); + break; + } + $this->in_transaction = false; + return $ret; + } + + /** + * @brief Build the array with the table relations + * + * The array is build from the database definitions in DBStructure.php + * + * This process must only be started once, since the value is cached. + */ + private function buildRelationData() + { + $definition = DBStructure::definition($this->configCache->get('system', 'basepath')); + + foreach ($definition AS $table => $structure) { + foreach ($structure['fields'] AS $field => $field_struct) { + if (isset($field_struct['relation'])) { + foreach ($field_struct['relation'] AS $rel_table => $rel_field) { + $this->relation[$rel_table][$rel_field][$table][] = $field; + } + } + } + } + } + + /** + * @brief Delete a row from a table + * + * @param string $table Table name + * @param array $conditions Field condition(s) + * @param array $options + * - cascade: If true we delete records in other tables that depend on the one we're deleting through + * relations (default: true) + * @param array $callstack Internal use: prevent endless loops + * + * @return boolean was the delete successful? + * @throws \Exception + */ + public function delete($table, array $conditions, array $options = [], array &$callstack = []) + { + if (empty($table) || empty($conditions)) { + $this->logger->info('Table and conditions have to be set'); + return false; + } + + $commands = []; + + // Create a key for the loop prevention + $key = $table . ':' . json_encode($conditions); + + // We quit when this key already exists in the callstack. + if (isset($callstack[$key])) { + return $commands; + } + + $callstack[$key] = true; + + $table = $this->sanitizeIdentifier($table); + + $commands[$key] = ['table' => $table, 'conditions' => $conditions]; + + // Don't use "defaults" here, since it would set "false" to "true" + if (isset($options['cascade'])) { + $cascade = $options['cascade']; + } else { + $cascade = true; + } + + // To speed up the whole process we cache the table relations + if ($cascade && count($this->relation) == 0) { + $this->buildRelationData(); + } + + // Is there a relation entry for the table? + if ($cascade && isset($this->relation[$table])) { + // We only allow a simple "one field" relation. + $field = array_keys($this->relation[$table])[0]; + $rel_def = array_values($this->relation[$table])[0]; + + // Create a key for preventing double queries + $qkey = $field . '-' . $table . ':' . json_encode($conditions); + + // When the search field is the relation field, we don't need to fetch the rows + // This is useful when the leading record is already deleted in the frontend but the rest is done in the backend + if ((count($conditions) == 1) && ($field == array_keys($conditions)[0])) { + foreach ($rel_def AS $rel_table => $rel_fields) { + foreach ($rel_fields AS $rel_field) { + $this->delete($rel_table, [$rel_field => array_values($conditions)[0]], $options, $callstack); + } + } + // We quit when this key already exists in the callstack. + } elseif (!isset($callstack[$qkey])) { + $callstack[$qkey] = true; + + // Fetch all rows that are to be deleted + $data = $this->select($table, [$field], $conditions); + + while ($row = $this->fetch($data)) { + $this->delete($table, [$field => $row[$field]], $options, $callstack); + } + + $this->close($data); + + // Since we had split the delete command we don't need the original command anymore + unset($commands[$key]); + } + } + + // Now we finalize the process + $do_transaction = !$this->in_transaction; + + if ($do_transaction) { + $this->transaction(); + } + + $compacted = []; + $counter = []; + + foreach ($commands AS $command) { + $conditions = $command['conditions']; + reset($conditions); + $first_key = key($conditions); + + $condition_string = DBA::buildCondition($conditions); + + if ((count($command['conditions']) > 1) || is_int($first_key)) { + $sql = "DELETE FROM `" . $command['table'] . "`" . $condition_string; + $this->logger->debug($this->replaceParameters($sql, $conditions)); + + if (!$this->e($sql, $conditions)) { + if ($do_transaction) { + $this->rollback(); + } + return false; + } + } else { + $key_table = $command['table']; + $key_condition = array_keys($command['conditions'])[0]; + $value = array_values($command['conditions'])[0]; + + // Split the SQL queries in chunks of 100 values + // We do the $i stuff here to make the code better readable + $i = isset($counter[$key_table][$key_condition]) ? $counter[$key_table][$key_condition] : 0; + if (isset($compacted[$key_table][$key_condition][$i]) && count($compacted[$key_table][$key_condition][$i]) > 100) { + ++$i; + } + + $compacted[$key_table][$key_condition][$i][$value] = $value; + $counter[$key_table][$key_condition] = $i; + } + } + foreach ($compacted AS $table => $values) { + foreach ($values AS $field => $field_value_list) { + foreach ($field_value_list AS $field_values) { + $sql = "DELETE FROM `" . $table . "` WHERE `" . $field . "` IN (" . + substr(str_repeat("?, ", count($field_values)), 0, -2) . ");"; + + $this->logger->debug($this->replaceParameters($sql, $field_values)); + + if (!$this->e($sql, $field_values)) { + if ($do_transaction) { + $this->rollback(); + } + return false; + } + } + } + } + if ($do_transaction) { + $this->commit(); + } + return true; + } + + /** + * @brief Updates rows + * + * Updates rows in the database. When $old_fields is set to an array, + * the system will only do an update if the fields in that array changed. + * + * Attention: + * Only the values in $old_fields are compared. + * This is an intentional behaviour. + * + * Example: + * We include the timestamp field in $fields but not in $old_fields. + * Then the row will only get the new timestamp when the other fields had changed. + * + * When $old_fields is set to a boolean value the system will do this compare itself. + * When $old_fields is set to "true" the system will do an insert if the row doesn't exists. + * + * Attention: + * Only set $old_fields to a boolean value when you are sure that you will update a single row. + * When you set $old_fields to "true" then $fields must contain all relevant fields! + * + * @param string $table Table name + * @param array $fields contains the fields that are updated + * @param array $condition condition array with the key values + * @param array|boolean $old_fields array with the old field values that are about to be replaced (true = update on duplicate) + * + * @return boolean was the update successfull? + * @throws \Exception + */ + public function update($table, $fields, $condition, $old_fields = []) + { + + if (empty($table) || empty($fields) || empty($condition)) { + $this->logger->info('Table, fields and condition have to be set'); + return false; + } + + $condition_string = DBA::buildCondition($condition); + + if (is_bool($old_fields)) { + $do_insert = $old_fields; + + $old_fields = $this->selectFirst($table, [], $condition); + + if (is_bool($old_fields)) { + if ($do_insert) { + $values = array_merge($condition, $fields); + return $this->insert($table, $values, $do_insert); + } + $old_fields = []; + } + } + + $do_update = (count($old_fields) == 0); + + foreach ($old_fields AS $fieldname => $content) { + if (isset($fields[$fieldname])) { + if (($fields[$fieldname] == $content) && !is_null($content)) { + unset($fields[$fieldname]); + } else { + $do_update = true; + } + } + } + + if (!$do_update || (count($fields) == 0)) { + return true; + } + + $sql = "UPDATE " . $this->formatTableName($table) . " SET `" . + implode("` = ?, `", array_keys($fields)) . "` = ?" . $condition_string; + + $params1 = array_values($fields); + $params2 = array_values($condition); + $params = array_merge_recursive($params1, $params2); + + return $this->e($sql, $params); + } + + /** + * Retrieve a single record from a table and returns it in an associative array + * + * @brief Retrieve a single record from a table + * + * @param string $table + * @param array $fields + * @param array $condition + * @param array $params + * + * @return bool|array + * @throws \Exception + * @see $this->select + */ + public function selectFirst($table, array $fields = [], array $condition = [], $params = []) + { + $params['limit'] = 1; + $result = $this->select($table, $fields, $condition, $params); + + if (is_bool($result)) { + return $result; + } else { + $row = $this->fetch($result); + $this->close($result); + return $row; + } + } + + /** + * @brief Select rows from a table + * + * @param string $table Table name + * @param array $fields Array of selected fields, empty for all + * @param array $condition Array of fields for condition + * @param array $params Array of several parameters + * + * @return boolean|object + * + * Example: + * $table = "item"; + * $fields = array("id", "uri", "uid", "network"); + * + * $condition = array("uid" => 1, "network" => 'dspr'); + * or: + * $condition = array("`uid` = ? AND `network` IN (?, ?)", 1, 'dfrn', 'dspr'); + * + * $params = array("order" => array("id", "received" => true), "limit" => 10); + * + * $data = DBA::select($table, $fields, $condition, $params); + * @throws \Exception + */ + public function select($table, array $fields = [], array $condition = [], array $params = []) + { + if (empty($table)) { + return false; + } + + if (count($fields) > 0) { + $select_fields = "`" . implode("`, `", array_values($fields)) . "`"; + } else { + $select_fields = "*"; + } + + $condition_string = DBA::buildCondition($condition); + + $param_string = DBA::buildParameter($params); + + $sql = "SELECT " . $select_fields . " FROM " . $this->formatTableName($table) . $condition_string . $param_string; + + $result = $this->p($sql, $condition); + + return $result; + } + + /** + * @brief Counts the rows from a table satisfying the provided condition + * + * @param string $table Table name + * @param array $condition array of fields for condition + * + * @return int + * + * Example: + * $table = "item"; + * + * $condition = ["uid" => 1, "network" => 'dspr']; + * or: + * $condition = ["`uid` = ? AND `network` IN (?, ?)", 1, 'dfrn', 'dspr']; + * + * $count = DBA::count($table, $condition); + * @throws \Exception + */ + public function count($table, array $condition = []) + { + if (empty($table)) { + return false; + } + + $condition_string = DBA::buildCondition($condition); + + $sql = "SELECT COUNT(*) AS `count` FROM " . $this->formatTableName($table) . $condition_string; + + $row = $this->fetchFirst($sql, $condition); + + return $row['count']; + } + + /** + * @brief Fills an array with data from a query + * + * @param object $stmt statement object + * @param bool $do_close + * + * @return array Data array + */ + public function toArray($stmt, $do_close = true) + { + if (is_bool($stmt)) { + return $stmt; + } + + $data = []; + while ($row = $this->fetch($stmt)) { + $data[] = $row; + } + if ($do_close) { + $this->close($stmt); + } + return $data; + } + + /** + * @brief Returns the error number of the last query + * + * @return string Error number (0 if no error) + */ + public function errorNo() + { + return $this->errorno; + } + + /** + * @brief Returns the error message of the last query + * + * @return string Error message ('' if no error) + */ + public function errorMessage() + { + return $this->error; + } + + /** + * @brief Closes the current statement + * + * @param object $stmt statement object + * + * @return boolean was the close successful? + */ + public function close($stmt) + { + + $stamp1 = microtime(true); + + if (!is_object($stmt)) { + return false; + } + + switch ($this->driver) { + case 'pdo': + $ret = $stmt->closeCursor(); + break; + case 'mysqli': + // MySQLi offers both a mysqli_stmt and a mysqli_result class. + // We should be careful not to assume the object type of $stmt + // because DBA::p() has been able to return both types. + if ($stmt instanceof mysqli_stmt) { + $stmt->free_result(); + $ret = $stmt->close(); + } elseif ($stmt instanceof mysqli_result) { + $stmt->free(); + $ret = true; + } else { + $ret = false; + } + break; + } + + $this->profiler->saveTimestamp($stamp1, 'database', System::callstack()); + + return $ret; + } + + /** + * @brief Return a list of database processes + * + * @return array + * 'list' => List of processes, separated in their different states + * 'amount' => Number of concurrent database processes + * @throws \Exception + */ + public function processlist() + { + $ret = $this->p("SHOW PROCESSLIST"); + $data = $this->toArray($ret); + + $processes = 0; + $states = []; + foreach ($data as $process) { + $state = trim($process["State"]); + + // Filter out all non blocking processes + if (!in_array($state, ["", "init", "statistics", "updating"])) { + ++$states[$state]; + ++$processes; + } + } + + $statelist = ""; + foreach ($states as $state => $usage) { + if ($statelist != "") { + $statelist .= ", "; + } + $statelist .= $state . ": " . $usage; + } + return (["list" => $statelist, "amount" => $processes]); + } + + /** + * Checks if $array is a filled array with at least one entry. + * + * @param mixed $array A filled array with at least one entry + * + * @return boolean Whether $array is a filled array or an object with rows + */ + public function isResult($array) + { + // It could be a return value from an update statement + if (is_bool($array)) { + return $array; + } + + if (is_object($array)) { + return $this->numRows($array) > 0; + } + + return (is_array($array) && (count($array) > 0)); + } + + /** + * @brief Callback function for "esc_array" + * + * @param mixed $value Array value + * @param string $key Array key + * @param boolean $add_quotation add quotation marks for string values + * + * @return void + */ + private function escapeArrayCallback(&$value, $key, $add_quotation) + { + if (!$add_quotation) { + if (is_bool($value)) { + $value = ($value ? '1' : '0'); + } else { + $value = $this->escape($value); + } + return; + } + + if (is_bool($value)) { + $value = ($value ? 'true' : 'false'); + } elseif (is_float($value) || is_integer($value)) { + $value = (string)$value; + } else { + $value = "'" . $this->escape($value) . "'"; + } + } + + /** + * @brief Escapes a whole array + * + * @param mixed $arr Array with values to be escaped + * @param boolean $add_quotation add quotation marks for string values + * + * @return void + */ + public function escapeArray(&$arr, $add_quotation = false) + { + array_walk($arr, [$this, 'escapeArrayCallback'], $add_quotation); + } +} diff --git a/src/Factory/DBFactory.php b/src/Factory/DBFactory.php index 7caa63ec46..3a972c7128 100644 --- a/src/Factory/DBFactory.php +++ b/src/Factory/DBFactory.php @@ -17,14 +17,11 @@ class DBFactory * @param Profiler $profiler The profiler * @param array $server The $_SERVER variables * + * @return Database\Database * @throws \Exception if connection went bad */ public static function init(Cache\IConfigCache $configCache, Profiler $profiler, array $server) { - if (Database\DBA::connected()) { - return; - } - $db_host = $configCache->get('database', 'hostname'); $db_user = $configCache->get('database', 'username'); $db_pass = $configCache->get('database', 'password'); @@ -50,11 +47,15 @@ class DBFactory $db_data = $server['MYSQL_DATABASE']; } - if (Database\DBA::connect($configCache, $profiler, new VoidLogger(), $db_host, $db_user, $db_pass, $db_data, $charset)) { + $database = new Database\Database($configCache, $profiler, new VoidLogger(), $db_host, $db_user, $db_pass, $db_data, $charset); + + if ($database->connected()) { // Loads DB_UPDATE_VERSION constant Database\DBStructure::definition($configCache->get('system', 'basepath'), false); } unset($db_host, $db_user, $db_pass, $db_data, $charset); + + return $database; } } diff --git a/src/Factory/DependencyFactory.php b/src/Factory/DependencyFactory.php index aacd155060..5d92a50206 100644 --- a/src/Factory/DependencyFactory.php +++ b/src/Factory/DependencyFactory.php @@ -3,7 +3,6 @@ namespace Friendica\Factory; use Friendica\App; -use Friendica\Database\DBA; use Friendica\Factory; use Friendica\Util\BasePath; use Friendica\Util\BaseURL; @@ -30,15 +29,14 @@ class DependencyFactory $configLoader = new Config\ConfigFileLoader($basePath, $mode); $configCache = Factory\ConfigFactory::createCache($configLoader); $profiler = Factory\ProfilerFactory::create($configCache); - Factory\DBFactory::init($configCache, $profiler, $_SERVER); + $database = Factory\DBFactory::init($configCache, $profiler, $_SERVER); $config = Factory\ConfigFactory::createConfig($configCache); // needed to call PConfig::init() Factory\ConfigFactory::createPConfig($configCache); - $logger = Factory\LoggerFactory::create($channel, $config, $profiler); - DBA::setLogger($logger); + $logger = Factory\LoggerFactory::create($channel, $database, $config, $profiler); Factory\LoggerFactory::createDev($channel, $config, $profiler); $baseURL = new BaseURL($config, $_SERVER); - return new App($config, $mode, $router, $baseURL, $logger, $profiler, $isBackend); + return new App($database, $config, $mode, $router, $baseURL, $logger, $profiler, $isBackend); } } diff --git a/src/Factory/LoggerFactory.php b/src/Factory/LoggerFactory.php index bdd85cf3ae..67829546e9 100644 --- a/src/Factory/LoggerFactory.php +++ b/src/Factory/LoggerFactory.php @@ -4,6 +4,7 @@ namespace Friendica\Factory; use Friendica\Core\Config\Configuration; use Friendica\Core\Logger; +use Friendica\Database\Database; use Friendica\Network\HTTPException\InternalServerErrorException; use Friendica\Util\Introspection; use Friendica\Util\Logger\Monolog\DevelopHandler; @@ -47,10 +48,11 @@ class LoggerFactory * @throws \Exception * @throws InternalServerErrorException */ - public static function create($channel, Configuration $config, Profiler $profiler) + public static function create($channel, Database $database, Configuration $config, Profiler $profiler) { if (empty($config->get('system', 'debugging', false))) { $logger = new VoidLogger(); + $database->setLogger($logger); Logger::init($logger); return $logger; } @@ -101,6 +103,7 @@ class LoggerFactory $logger = new ProfilerLogger($logger, $profiler); } + $database->setLogger($logger); Logger::init($logger); return $logger; diff --git a/src/Model/Contact.php b/src/Model/Contact.php index a6026d6440..f38c69ed05 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -1328,7 +1328,7 @@ class Contact extends BaseObject // Update the contact in the background if needed but it is called by the frontend if ($update_contact && $no_update && in_array($contact['network'], Protocol::NATIVE_SUPPORT)) { - Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id); + Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id, ($uid == 0 ? 'force' : '')); } if (!$update_contact || $no_update) { @@ -1425,7 +1425,7 @@ class Contact extends BaseObject // Update in the background when we fetched the data solely from the database if ($background_update) { - Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id); + Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id, ($uid == 0 ? 'force' : '')); } // Update the newly created contact from data in the gcontact table @@ -1452,7 +1452,7 @@ class Contact extends BaseObject } } - if (!empty($data['photo'])) { + if (!empty($data['photo']) && ($data['network'] != Protocol::FEED)) { self::updateAvatar($data['photo'], $uid, $contact_id); } @@ -1755,6 +1755,51 @@ class Contact extends BaseObject return $data; } + /** + * @brief Helper function for "updateFromProbe". Updates personal and public contact + * + * @param array $contact The personal contact entry + * @param array $fields The fields that are updated + * @throws \Exception + */ + private static function updateContact($id, $uid, $url, array $fields) + { + DBA::update('contact', $fields, ['id' => $id]); + + if ($uid != 0) { + return; + } + + // Archive or unarchive the contact. We only need to do this for the public contact. + // The archive/unarchive function will update the personal contacts by themselves. + $contact = DBA::selectFirst('contact', [], ['id' => $id]); + if (!empty($fields['success_update'])) { + self::unmarkForArchival($contact); + } elseif (!empty($fields['failure_update'])) { + self::markForArchival($contact); + } + + $condition = ['self' => false, 'nurl' => Strings::normaliseLink($url), + 'network' => [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS]]; + + // These contacts are sharing with us, we don't poll them. + // This means that we don't set the update fields in "OnePoll.php". + $condition['rel'] = self::SHARING; + DBA::update('contact', $fields, $condition); + + unset($fields['last-update']); + unset($fields['success_update']); + unset($fields['failure_update']); + + if (empty($fields)) { + return; + } + + // We are polling these contacts, so we mustn't set the update fields here. + $condition['rel'] = [self::FOLLOWER, self::FRIEND]; + DBA::update('contact', $fields, $condition); + } + /** * @param integer $id contact id * @param string $network Optional network we are probing for @@ -1767,7 +1812,7 @@ class Contact extends BaseObject { /* Warning: Never ever fetch the public key via Probe::uri and write it into the contacts. - This will reliably kill your communication with Friendica contacts. + This will reliably kill your communication with old Friendica contacts. */ $fields = ['avatar', 'uid', 'name', 'nick', 'url', 'addr', 'batch', 'notify', @@ -1785,12 +1830,14 @@ class Contact extends BaseObject $ret = Probe::uri($contact['url'], $network, $uid, !$force); - // If Probe::uri fails the network code will be different (mostly "feed" or "unkn") - if (in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]) && ($ret['network'] != $contact['network'])) { - return false; - } + $updated = DateTimeFormat::utcNow(); - if (!in_array($ret['network'], Protocol::NATIVE_SUPPORT)) { + // If Probe::uri fails the network code will be different (mostly "feed" or "unkn") + if (!in_array($ret['network'], Protocol::NATIVE_SUPPORT) || + (in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]) && ($ret['network'] != $contact['network']))) { + if ($force && ($uid == 0)) { + self::updateContact($id, $uid, $ret['url'], ['last-update' => $updated, 'failure_update' => $updated]); + } return false; } @@ -1807,17 +1854,28 @@ class Contact extends BaseObject } } + if ($ret['network'] != Protocol::FEED) { + self::updateAvatar($ret['photo'], $uid, $id, $update || $force); + } + if (!$update) { + if ($force && ($uid == 0)) { + self::updateContact($id, $uid, $ret['url'], ['last-update' => $updated, 'success_update' => $updated]); + } return true; } $ret['nurl'] = Strings::normaliseLink($ret['url']); - $ret['updated'] = DateTimeFormat::utcNow(); + $ret['updated'] = $updated; - self::updateAvatar($ret['photo'], $uid, $id, true); + if ($force && ($uid == 0)) { + $ret['last-update'] = $updated; + $ret['success_update'] = $updated; + } unset($ret['photo']); - DBA::update('contact', $ret, ['id' => $id]); + + self::updateContact($id, $uid, $ret['url'], $ret); // Update the corresponding gcontact entry PortableContact::lastUpdated($ret["url"]); diff --git a/src/Model/Photo.php b/src/Model/Photo.php index 7df96fccdb..0e3661b0f3 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -130,18 +130,21 @@ class Photo extends BaseObject */ public static function getPhoto($resourceid, $scale = 0) { - $r = self::selectFirst(["uid"], ["resource-id" => $resourceid]); + $r = self::selectFirst(["uid", "allow_cid", "allow_gid", "deny_cid", "deny_gid"], ["resource-id" => $resourceid]); if ($r === false) { return false; } $uid = $r["uid"]; // This is the first place, when retrieving just a photo, that we know who owns the photo. - // Make sure that the requester's session is appropriately authenticated to that user + // Check if the photo is public (empty allow and deny means public), if so, skip auth attempt, if not + // make sure that the requester's session is appropriately authenticated to that user // otherwise permissions checks done by getPermissionsSQLByUserId() won't work correctly - $r = DBA::selectFirst("user", ["nickname"], ["uid" => $uid], []); - // this will either just return (if auth all ok) or will redirect and exit (starting over) - DFRN::autoRedir(self::getApp(), $r["nickname"]); + if (!empty($r["allow_cid"]) || !empty($r["allow_gid"]) || !empty($r["deny_cid"]) || !empty($r["deny_gid"])) { + $r = DBA::selectFirst("user", ["nickname"], ["uid" => $uid], []); + // this will either just return (if auth all ok) or will redirect and exit (starting over) + DFRN::autoRedir(self::getApp(), $r["nickname"]); + } $sql_acl = Security::getPermissionsSQLByUserId($uid); diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index f7377a6e71..0db70f5a8d 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -17,16 +17,10 @@ use Friendica\Util\DateTimeFormat; class Cron { - public static function execute($parameter = '', $generation = 0) + public static function execute() { $a = BaseObject::getApp(); - // Poll contacts with specific parameters - if (!empty($parameter)) { - self::pollContacts($parameter, $generation); - return; - } - $last = Config::get('system', 'last_cron'); $poll_interval = intval(Config::get('system', 'cron_interval')); @@ -115,7 +109,10 @@ class Cron } // Poll contacts - self::pollContacts($parameter, $generation); + self::pollContacts(); + + // Update contact information + self::updatePublicContacts(); Logger::log('cron: end'); @@ -125,98 +122,76 @@ class Cron } /** - * @brief Poll contacts for unreceived messages - * - * @todo Currently it seems as if the following parameter aren't used at all ... - * - * @param string $parameter Parameter (force, restart, ...) for the contact polling - * @param integer $generation + * @brief Update public contacts * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function pollContacts($parameter, $generation) { - $manual_id = 0; - $generation = 0; - $force = false; + private static function updatePublicContacts() { + $count = 0; + $last_updated = DateTimeFormat::utc('now - 1 week'); + $condition = ["`network` IN (?, ?, ?, ?) AND `uid` = ? AND NOT `self` AND `last-update` < ?", + Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, 0, $last_updated]; - if ($parameter == 'force') { - $force = true; - } - if ($parameter == 'restart') { - $generation = intval($generation); - if (!$generation) { - exit(); + $total = DBA::count('contact', $condition); + $oldest_date = ''; + $oldest_id = ''; + $contacts = DBA::select('contact', ['id', 'last-update'], $condition, ['limit' => 100, 'order' => ['last-update']]); + while ($contact = DBA::fetch($contacts)) { + if (empty($oldest_id)) { + $oldest_id = $contact['id']; + $oldest_date = $contact['last-update']; } + Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id'], 'force'); + ++$count; } + Logger::info('Initiated update for public contacts', ['interval' => $count, 'total' => $total, 'id' => $oldest_id, 'oldest' => $oldest_date]); + DBA::close($contacts); + } - if (intval($parameter)) { - $manual_id = intval($parameter); - $force = true; - } - + /** + * @brief Poll contacts for unreceived messages + * + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + private static function pollContacts() { $min_poll_interval = Config::get('system', 'min_poll_interval', 1); - $sql_extra = (($manual_id) ? " AND `id` = $manual_id " : ""); - Addon::reload(); - // Only poll from those with suitable relationships, - // and which have a polling address and ignore Diaspora since - // we are unable to match those posts with a Diaspora GUID and prevent duplicates. - - $abandon_days = intval(Config::get('system', 'account_abandon_days')); - if ($abandon_days < 1) { - $abandon_days = 0; - } - $abandon_sql = (($abandon_days) - ? sprintf(" AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL %d DAY ", intval($abandon_days)) - : '' - ); - - $contacts = q("SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`, `contact`.`archive`, + $sql = "SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`, `contact`.`archive`, `contact`.`last-update`, `contact`.`priority`, `contact`.`rel`, `contact`.`subhub` FROM `user` STRAIGHT_JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`poll` != '' - AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s', '%s') $sql_extra + AND `contact`.`network` IN (?, ?, ?, ?) AND NOT `contact`.`self` AND NOT `contact`.`blocked` - WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql", - DBA::escape(Protocol::ACTIVITYPUB), - DBA::escape(Protocol::DFRN), - DBA::escape(Protocol::OSTATUS), - DBA::escape(Protocol::DIASPORA), - DBA::escape(Protocol::FEED), - DBA::escape(Protocol::MAIL) - ); + AND `contact`.`rel` != ? + WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed`"; + + $parameters = [Protocol::DFRN, Protocol::OSTATUS, Protocol::FEED, Protocol::MAIL, Contact::FOLLOWER]; + + // Only poll from those with suitable relationships, + // and which have a polling address and ignore Diaspora since + // we are unable to match those posts with a Diaspora GUID and prevent duplicates. + $abandon_days = intval(Config::get('system', 'account_abandon_days')); + if ($abandon_days < 1) { + $abandon_days = 0; + } + + if (!empty($abandon_days)) { + $sql .= " AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL ? DAY"; + $parameters[] = $abandon_days; + } + + $contacts = DBA::p($sql, $parameters); if (!DBA::isResult($contacts)) { return; } - foreach ($contacts as $contact) { - - if ($manual_id) { - $contact['last-update'] = DBA::NULL_DATETIME; - } - + while ($contact = DBA::fetch($contacts)) { // Friendica and OStatus are checked once a day if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { - $contact['priority'] = 2; - } - - if ($contact['subhub'] && in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) { - /* - * We should be getting everything via a hub. But just to be sure, let's check once a day. - * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately) - * This also lets us update our subscription to the hub, and add or replace hubs in case it - * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'. - */ - $poll_interval = Config::get('system', 'pushpoll_frequency'); - $contact['priority'] = (!is_null($poll_interval) ? intval($poll_interval) : 3); - } - - // Check ActivityPub and Diaspora contacts or followers once a week - if (in_array($contact["network"], [Protocol::ACTIVITYPUB, Protocol::DIASPORA]) || ($contact["rel"] == Contact::FOLLOWER)) { - $contact['priority'] = 4; + $contact['priority'] = 3; } // Check archived contacts once a month @@ -224,7 +199,7 @@ class Cron $contact['priority'] = 5; } - if (($contact['priority'] >= 0) && !$force) { + if ($contact['priority'] >= 0) { $update = false; $t = $contact['last-update']; @@ -260,7 +235,7 @@ class Cron break; case 0: default: - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + ".$min_poll_interval." minute")) { + if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + " . $min_poll_interval . " minute")) { $update = true; } break; @@ -282,5 +257,6 @@ class Cron Worker::add(['priority' => $priority, 'dont_fork' => true, 'force_priority' => true], 'OnePoll', (int)$contact['id']); } + DBA::close($contacts); } } diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index a605ee92ed..3c14100ab2 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -11,9 +11,9 @@ use Friendica\Core\Logger; use Friendica\Core\PConfig; use Friendica\Core\Protocol; use Friendica\Database\DBA; -use Friendica\Model\APContact; use Friendica\Model\Contact; use Friendica\Model\Item; +use Friendica\Model\User; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Email; use Friendica\Protocol\PortableContact; @@ -30,7 +30,7 @@ class OnePoll Logger::log('Start for contact ' . $contact_id); - $force = false; + $force = false; if ($command == "force") { $force = true; @@ -41,6 +41,9 @@ class OnePoll return; } + if ($force) { + Contact::updateFromProbe($contact_id, true); + } $contact = DBA::selectFirst('contact', [], ['id' => $contact_id]); if (!DBA::isResult($contact)) { @@ -56,99 +59,29 @@ class OnePoll $importer_uid = $contact['uid']; + $updated = DateTimeFormat::utcNow(); + + if ($importer_uid == 0) { + Logger::log('Ignore public contacts'); + + // set the last-update so we don't keep polling + DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); + return; + } + // Possibly switch the remote contact to AP if ($protocol === Protocol::OSTATUS) { ActivityPub\Receiver::switchContact($contact['id'], $importer_uid, $contact['url']); $contact = DBA::selectFirst('contact', [], ['id' => $contact_id]); } - $updated = DateTimeFormat::utcNow(); - - // These three networks can be able to speak AP, so we are trying to fetch AP profile data here - if (in_array($protocol, [Protocol::ACTIVITYPUB, Protocol::DIASPORA, Protocol::DFRN])) { - $apcontact = APContact::getByURL($contact['url'], true); - - if (($protocol === Protocol::ACTIVITYPUB) && empty($apcontact)) { - self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]); - Contact::markForArchival($contact); - Logger::log('Contact archived'); - return; - } elseif (!empty($apcontact)) { - $fields = ['last-update' => $updated, 'success_update' => $updated]; - self::updateContact($contact, $fields); - Contact::unmarkForArchival($contact); - } - } - - // Diaspora users, archived users and followers are only checked if they still exist. - if (($protocol != Protocol::ACTIVITYPUB) && ($contact['archive'] || ($contact["network"] == Protocol::DIASPORA) || ($contact["rel"] == Contact::FOLLOWER))) { - $last_updated = PortableContact::lastUpdated($contact["url"], true); - - if ($last_updated) { - Logger::log('Contact '.$contact['id'].' had last update on '.$last_updated, Logger::DEBUG); - - // The last public item can be older than the last item we got - if ($last_updated < $contact['last-item']) { - $last_updated = $contact['last-item']; - } - - $fields = ['last-item' => DateTimeFormat::utc($last_updated), 'last-update' => $updated, 'success_update' => $updated]; - self::updateContact($contact, $fields); - Contact::unmarkForArchival($contact); - } else { - self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]); - Contact::markForArchival($contact); - Logger::log('Contact archived'); - return; - } - } - - // Update the contact entry - if (in_array($protocol, [Protocol::ACTIVITYPUB, Protocol::OSTATUS, Protocol::DIASPORA, Protocol::DFRN])) { - // Currently we can't check every AP implementation, so we don't do it at all - if (($protocol != Protocol::ACTIVITYPUB) && !PortableContact::reachable($contact['url'])) { - Logger::log("Skipping probably dead contact ".$contact['url']); - - // set the last-update so we don't keep polling - self::updateContact($contact, ['last-update' => $updated]); - return; - } - - if (!Contact::updateFromProbe($contact["id"])) { - // set the last-update so we don't keep polling - self::updateContact($contact, ['last-update' => $updated]); - Contact::markForArchival($contact); - Logger::log('Contact archived'); - return; - } else { - $fields = ['last-update' => $updated, 'success_update' => $updated]; - self::updateContact($contact, $fields); - Contact::unmarkForArchival($contact); - } - } - // load current friends if possible. if (!empty($contact['poco']) && ($contact['success_update'] > $contact['failure_update'])) { - $r = q("SELECT count(*) AS total FROM glink - WHERE `cid` = %d AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", - intval($contact['id']) - ); - if (DBA::isResult($r)) { - if (!$r[0]['total']) { - PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']); - } + if (!DBA::exists('glink', ["`cid` = ? AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", $contact['id']])) { + PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']); } } - // We don't poll our followers - if ($contact["rel"] == Contact::FOLLOWER) { - Logger::log("Don't poll follower"); - - // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); - return; - } - // Don't poll if polling is deactivated (But we poll feeds and mails anyway) if (!in_array($protocol, [Protocol::FEED, Protocol::MAIL]) && Config::get('system', 'disable_polling')) { Logger::log('Polling is disabled'); @@ -167,19 +100,9 @@ class OnePoll return; } - if ($importer_uid == 0) { - Logger::log('Ignore public contacts'); + $importer = User::getOwnerDataById($importer_uid); - // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); - return; - } - - $r = q("SELECT `contact`.*, `user`.`page-flags` FROM `contact` INNER JOIN `user` on `contact`.`uid` = `user`.`uid` WHERE `user`.`uid` = %d AND `contact`.`self` = 1 LIMIT 1", - intval($importer_uid) - ); - - if (!DBA::isResult($r)) { + if (empty($importer)) { Logger::log('No self contact for user '.$importer_uid); // set the last-update so we don't keep polling @@ -187,7 +110,6 @@ class OnePoll return; } - $importer = $r[0]; $url = ''; $xml = false; @@ -203,426 +125,21 @@ class OnePoll $hub_update = false; } - $last_update = (($contact['last-update'] <= DBA::NULL_DATETIME) - ? DateTimeFormat::utc('now - 7 days', DateTimeFormat::ATOM) - : DateTimeFormat::utc($contact['last-update'], DateTimeFormat::ATOM) - ); - Logger::log("poll: ({$protocol}-{$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}"); + $xml = ''; + if ($protocol === Protocol::DFRN) { - $idtosend = $orig_id = (($contact['dfrn-id']) ? $contact['dfrn-id'] : $contact['issued-id']); - if (intval($contact['duplex']) && $contact['dfrn-id']) { - $idtosend = '0:' . $orig_id; - } - if (intval($contact['duplex']) && $contact['issued-id']) { - $idtosend = '1:' . $orig_id; - } - - // they have permission to write to us. We already filtered this in the contact query. - $perm = 'rw'; - - // But this may be our first communication, so set the writable flag if it isn't set already. - - if (!intval($contact['writable'])) { - $fields = ['writable' => true]; - DBA::update('contact', $fields, ['id' => $contact['id']]); - } - - $url = $contact['poll'] . '?dfrn_id=' . $idtosend - . '&dfrn_version=' . DFRN_PROTOCOL_VERSION - . '&type=data&last_update=' . $last_update - . '&perm=' . $perm; - - $curlResult = Network::curl($url); - - if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) { - // set the last-update so we don't keep polling - self::updateContact($contact, ['last-update' => $updated]); - Contact::markForArchival($contact); - Logger::log('Contact archived'); - return; - } - - $handshake_xml = $curlResult->getBody(); - $html_code = $curlResult->getReturnCode(); - - Logger::log('handshake with url ' . $url . ' returns xml: ' . $handshake_xml, Logger::DATA); - - if (!strlen($handshake_xml) || ($html_code >= 400) || !$html_code) { - // dead connection - might be a transient event, or this might - // mean the software was uninstalled or the domain expired. - // Will keep trying for one month. - Logger::log("$url appears to be dead - marking for death "); - - // set the last-update so we don't keep polling - $fields = ['last-update' => $updated, 'failure_update' => $updated]; - self::updateContact($contact, $fields); - Contact::markForArchival($contact); - return; - } - - if (!strstr($handshake_xml, '<')) { - Logger::log('response from ' . $url . ' did not contain XML.'); - - $fields = ['last-update' => $updated, 'failure_update' => $updated]; - self::updateContact($contact, $fields); - Contact::markForArchival($contact); - return; - } - - - $res = XML::parseString($handshake_xml); - - if (intval($res->status) == 1) { - // we may not be friends anymore. Will keep trying for one month. - Logger::log("$url replied status 1 - marking for death "); - - // set the last-update so we don't keep polling - $fields = ['last-update' => $updated, 'failure_update' => $updated]; - self::updateContact($contact, $fields); - Contact::markForArchival($contact); - } elseif ($contact['term-date'] > DBA::NULL_DATETIME) { - Contact::unmarkForArchival($contact); - } - - if ((intval($res->status) != 0) || !strlen($res->challenge) || !strlen($res->dfrn_id)) { - // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); - Logger::log('Contact status is ' . $res->status); - return; - } - - if (((float)$res->dfrn_version > 2.21) && ($contact['poco'] == '')) { - $fields = ['poco' => str_replace('/profile/', '/poco/', $contact['url'])]; - DBA::update('contact', $fields, ['id' => $contact['id']]); - } - - $postvars = []; - - $sent_dfrn_id = hex2bin((string) $res->dfrn_id); - $challenge = hex2bin((string) $res->challenge); - - $final_dfrn_id = ''; - - if ($contact['duplex'] && strlen($contact['prvkey'])) { - openssl_private_decrypt($sent_dfrn_id, $final_dfrn_id, $contact['prvkey']); - openssl_private_decrypt($challenge, $postvars['challenge'], $contact['prvkey']); - } else { - openssl_public_decrypt($sent_dfrn_id, $final_dfrn_id, $contact['pubkey']); - openssl_public_decrypt($challenge, $postvars['challenge'], $contact['pubkey']); - } - - $final_dfrn_id = substr($final_dfrn_id, 0, strpos($final_dfrn_id, '.')); - - if (strpos($final_dfrn_id, ':') == 1) { - $final_dfrn_id = substr($final_dfrn_id, 2); - } - - // There are issues with the legacy DFRN transport layer. - // Since we mostly don't use it anyway, we won't dig into it deeper, but simply ignore it. - if (empty($final_dfrn_id) || empty($orig_id)) { - Logger::log('Contact has got no ID - quitting'); - return; - } - - if ($final_dfrn_id != $orig_id) { - // did not decode properly - cannot trust this site - Logger::log('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id); - - // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); - Contact::markForArchival($contact); - return; - } - - $postvars['dfrn_id'] = $idtosend; - $postvars['dfrn_version'] = DFRN_PROTOCOL_VERSION; - $postvars['perm'] = 'rw'; - - $xml = Network::post($contact['poll'], $postvars)->getBody(); - + $xml = self::pollDFRN($contact, $updated); } elseif (($protocol === Protocol::OSTATUS) || ($protocol === Protocol::DIASPORA) || ($protocol === Protocol::FEED)) { - - // Upgrading DB fields from an older Friendica version - // Will only do this once per notify-enabled OStatus contact - // or if relationship changes - - $stat_writeable = ((($contact['notify']) && ($contact['rel'] == Contact::FOLLOWER || $contact['rel'] == Contact::FRIEND)) ? 1 : 0); - - // Contacts from OStatus are always writable - if ($protocol === Protocol::OSTATUS) { - $stat_writeable = 1; - } - - if ($stat_writeable != $contact['writable']) { - $fields = ['writable' => $stat_writeable]; - DBA::update('contact', $fields, ['id' => $contact['id']]); - } - - // Are we allowed to import from this person? - if ($contact['rel'] == Contact::FOLLOWER || $contact['blocked']) { - // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); - Logger::log('Contact is blocked or only a follower'); - return; - } - - $cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-'); - $curlResult = Network::curl($contact['poll'], false, ['cookiejar' => $cookiejar]); - unlink($cookiejar); - - if ($curlResult->isTimeout()) { - // set the last-update so we don't keep polling - self::updateContact($contact, ['last-update' => $updated]); - Contact::markForArchival($contact); - Logger::log('Contact archived'); - return; - } - - $xml = $curlResult->getBody(); - + $xml = self::pollFeed($contact, $protocol, $updated); } elseif ($protocol === Protocol::MAIL) { - Logger::log("Mail: Fetching for ".$contact['addr'], Logger::DEBUG); - - $mail_disabled = ((function_exists('imap_open') && !Config::get('system', 'imap_disabled')) ? 0 : 1); - if ($mail_disabled) { - // set the last-update so we don't keep polling - self::updateContact($contact, ['last-update' => $updated]); - Contact::markForArchival($contact); - Logger::log('Contact archived'); - return; - } - - Logger::log("Mail: Enabled", Logger::DEBUG); - - $mbox = null; - $user = DBA::selectFirst('user', ['prvkey'], ['uid' => $importer_uid]); - - $condition = ["`server` != '' AND `uid` = ?", $importer_uid]; - $mailconf = DBA::selectFirst('mailacct', [], $condition); - if (DBA::isResult($user) && DBA::isResult($mailconf)) { - $mailbox = Email::constructMailboxName($mailconf); - $password = ''; - openssl_private_decrypt(hex2bin($mailconf['pass']), $password, $user['prvkey']); - $mbox = Email::connect($mailbox, $mailconf['user'], $password); - unset($password); - Logger::log("Mail: Connect to " . $mailconf['user']); - if ($mbox) { - $fields = ['last_check' => $updated]; - DBA::update('mailacct', $fields, ['id' => $mailconf['id']]); - Logger::log("Mail: Connected to " . $mailconf['user']); - } else { - Logger::log("Mail: Connection error ".$mailconf['user']." ".print_r(imap_errors(), true)); - } - } - - if ($mbox) { - $msgs = Email::poll($mbox, $contact['addr']); - - if (count($msgs)) { - Logger::log("Mail: Parsing ".count($msgs)." mails from ".$contact['addr']." for ".$mailconf['user'], Logger::DEBUG); - - $metas = Email::messageMeta($mbox, implode(',', $msgs)); - - if (count($metas) != count($msgs)) { - Logger::log("for " . $mailconf['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", Logger::DEBUG); - } else { - $msgs = array_combine($msgs, $metas); - - foreach ($msgs as $msg_uid => $meta) { - Logger::log("Mail: Parsing mail ".$msg_uid, Logger::DATA); - - $datarray = []; - $datarray['verb'] = ACTIVITY_POST; - $datarray['object-type'] = ACTIVITY_OBJ_NOTE; - $datarray['network'] = Protocol::MAIL; - // $meta = Email::messageMeta($mbox, $msg_uid); - - $datarray['uri'] = Email::msgid2iri(trim($meta->message_id, '<>')); - - // Have we seen it before? - $fields = ['deleted', 'id']; - $condition = ['uid' => $importer_uid, 'uri' => $datarray['uri']]; - $item = Item::selectFirst($fields, $condition); - if (DBA::isResult($item)) { - Logger::log("Mail: Seen before ".$msg_uid." for ".$mailconf['user']." UID: ".$importer_uid." URI: ".$datarray['uri'],Logger::DEBUG); - - // Only delete when mails aren't automatically moved or deleted - if (($mailconf['action'] != 1) && ($mailconf['action'] != 3)) - if ($meta->deleted && ! $item['deleted']) { - $fields = ['deleted' => true, 'changed' => $updated]; - Item::update($fields, ['id' => $item['id']]); - } - - switch ($mailconf['action']) { - case 0: - Logger::log("Mail: Seen before ".$msg_uid." for ".$mailconf['user'].". Doing nothing.", Logger::DEBUG); - break; - case 1: - Logger::log("Mail: Deleting ".$msg_uid." for ".$mailconf['user']); - imap_delete($mbox, $msg_uid, FT_UID); - break; - case 2: - Logger::log("Mail: Mark as seen ".$msg_uid." for ".$mailconf['user']); - imap_setflag_full($mbox, $msg_uid, "\\Seen", ST_UID); - break; - case 3: - Logger::log("Mail: Moving ".$msg_uid." to ".$mailconf['movetofolder']." for ".$mailconf['user']); - imap_setflag_full($mbox, $msg_uid, "\\Seen", ST_UID); - if ($mailconf['movetofolder'] != "") { - imap_mail_move($mbox, $msg_uid, $mailconf['movetofolder'], FT_UID); - } - break; - } - continue; - } - - - // look for a 'references' or an 'in-reply-to' header and try to match with a parent item we have locally. - $raw_refs = ((property_exists($meta, 'references')) ? str_replace("\t", '', $meta->references) : ''); - if (!trim($raw_refs)) { - $raw_refs = ((property_exists($meta, 'in_reply_to')) ? str_replace("\t", '', $meta->in_reply_to) : ''); - } - $raw_refs = trim($raw_refs); // Don't allow a blank reference in $refs_arr - - if ($raw_refs) { - $refs_arr = explode(' ', $raw_refs); - if (count($refs_arr)) { - for ($x = 0; $x < count($refs_arr); $x ++) { - $refs_arr[$x] = Email::msgid2iri(str_replace(['<', '>', ' '],['', '', ''], $refs_arr[$x])); - } - } - $condition = ['uri' => $refs_arr, 'uid' => $importer_uid]; - $parent = Item::selectFirst(['parent-uri'], $condition); - if (DBA::isResult($parent)) { - $datarray['parent-uri'] = $parent['parent-uri']; // Set the parent as the top-level item - } - } - - // Decoding the header - $subject = imap_mime_header_decode($meta->subject); - $datarray['title'] = ""; - foreach ($subject as $subpart) { - if ($subpart->charset != "default") { - $datarray['title'] .= iconv($subpart->charset, 'UTF-8//IGNORE', $subpart->text); - } else { - $datarray['title'] .= $subpart->text; - } - } - $datarray['title'] = Strings::escapeTags(trim($datarray['title'])); - - //$datarray['title'] = Strings::escapeTags(trim($meta->subject)); - $datarray['created'] = DateTimeFormat::utc($meta->date); - - // Is it a reply? - $reply = ((substr(strtolower($datarray['title']), 0, 3) == "re:") || - (substr(strtolower($datarray['title']), 0, 3) == "re-") || - ($raw_refs != "")); - - // Remove Reply-signs in the subject - $datarray['title'] = self::RemoveReply($datarray['title']); - - // If it seems to be a reply but a header couldn't be found take the last message with matching subject - if (empty($datarray['parent-uri']) && $reply) { - $condition = ['title' => $datarray['title'], 'uid' => $importer_uid, 'network' => Protocol::MAIL]; - $params = ['order' => ['created' => true]]; - $parent = Item::selectFirst(['parent-uri'], $condition, $params); - if (DBA::isResult($parent)) { - $datarray['parent-uri'] = $parent['parent-uri']; - } - } - - if (empty($datarray['parent-uri'])) { - $datarray['parent-uri'] = $datarray['uri']; - } - - $r = Email::getMessage($mbox, $msg_uid, $reply); - if (!$r) { - Logger::log("Mail: can't fetch msg ".$msg_uid." for ".$mailconf['user']); - continue; - } - $datarray['body'] = Strings::escapeHtml($r['body']); - $datarray['body'] = BBCode::limitBodySize($datarray['body']); - - Logger::log("Mail: Importing ".$msg_uid." for ".$mailconf['user']); - - /// @TODO Adding a gravatar for the original author would be cool - - $from = imap_mime_header_decode($meta->from); - $fromdecoded = ""; - foreach ($from as $frompart) { - if ($frompart->charset != "default") { - $fromdecoded .= iconv($frompart->charset, 'UTF-8//IGNORE', $frompart->text); - } else { - $fromdecoded .= $frompart->text; - } - } - - $fromarr = imap_rfc822_parse_adrlist($fromdecoded, $a->getHostName()); - - $frommail = $fromarr[0]->mailbox."@".$fromarr[0]->host; - - if (isset($fromarr[0]->personal)) { - $fromname = $fromarr[0]->personal; - } else { - $fromname = $frommail; - } - - $datarray['author-name'] = $fromname; - $datarray['author-link'] = "mailto:".$frommail; - $datarray['author-avatar'] = $contact['photo']; - - $datarray['owner-name'] = $contact['name']; - $datarray['owner-link'] = "mailto:".$contact['addr']; - $datarray['owner-avatar'] = $contact['photo']; - - $datarray['uid'] = $importer_uid; - $datarray['contact-id'] = $contact['id']; - if ($datarray['parent-uri'] === $datarray['uri']) { - $datarray['private'] = 1; - } - if (($protocol === Protocol::MAIL) && !PConfig::get($importer_uid, 'system', 'allow_public_email_replies')) { - $datarray['private'] = 1; - $datarray['allow_cid'] = '<' . $contact['id'] . '>'; - } - - Item::insert($datarray); - - switch ($mailconf['action']) { - case 0: - Logger::log("Mail: Seen before ".$msg_uid." for ".$mailconf['user'].". Doing nothing.", Logger::DEBUG); - break; - case 1: - Logger::log("Mail: Deleting ".$msg_uid." for ".$mailconf['user']); - imap_delete($mbox, $msg_uid, FT_UID); - break; - case 2: - Logger::log("Mail: Mark as seen ".$msg_uid." for ".$mailconf['user']); - imap_setflag_full($mbox, $msg_uid, "\\Seen", ST_UID); - break; - case 3: - Logger::log("Mail: Moving ".$msg_uid." to ".$mailconf['movetofolder']." for ".$mailconf['user']); - imap_setflag_full($mbox, $msg_uid, "\\Seen", ST_UID); - if ($mailconf['movetofolder'] != "") { - imap_mail_move($mbox, $msg_uid, $mailconf['movetofolder'], FT_UID); - } - break; - } - } - } - } else { - Logger::log("Mail: no mails for ".$mailconf['user']); - } - - Logger::log("Mail: closing connection for ".$mailconf['user']); - imap_close($mbox); - } + self::pollMail($contact, $importer_uid, $updated); } - if ($xml) { + if (!empty($xml)) { Logger::log('received xml : ' . $xml, Logger::DATA); if (!strstr($xml, '<')) { Logger::log('post_handshake: response from ' . $url . ' did not contain XML.'); @@ -711,4 +228,447 @@ class OnePoll DBA::update('contact', $fields, ['id' => $contact['id']]); DBA::update('contact', $fields, ['uid' => 0, 'nurl' => $contact['nurl']]); } + + /** + * @brief Poll DFRN contacts + * + * @param array $contact The personal contact entry + * @param string $updated The updated date + * @return string polled XML + * @throws \Exception + */ + private static function pollDFRN(array $contact, $updated) + { + $idtosend = $orig_id = (($contact['dfrn-id']) ? $contact['dfrn-id'] : $contact['issued-id']); + if (intval($contact['duplex']) && $contact['dfrn-id']) { + $idtosend = '0:' . $orig_id; + } + if (intval($contact['duplex']) && $contact['issued-id']) { + $idtosend = '1:' . $orig_id; + } + + // they have permission to write to us. We already filtered this in the contact query. + $perm = 'rw'; + + // But this may be our first communication, so set the writable flag if it isn't set already. + if (!intval($contact['writable'])) { + $fields = ['writable' => true]; + DBA::update('contact', $fields, ['id' => $contact['id']]); + } + + $last_update = (($contact['last-update'] <= DBA::NULL_DATETIME) + ? DateTimeFormat::utc('now - 7 days', DateTimeFormat::ATOM) + : DateTimeFormat::utc($contact['last-update'], DateTimeFormat::ATOM) + ); + + $url = $contact['poll'] . '?dfrn_id=' . $idtosend + . '&dfrn_version=' . DFRN_PROTOCOL_VERSION + . '&type=data&last_update=' . $last_update + . '&perm=' . $perm; + + $curlResult = Network::curl($url); + + if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) { + // set the last-update so we don't keep polling + self::updateContact($contact, ['last-update' => $updated]); + Contact::markForArchival($contact); + Logger::log('Contact archived'); + return false; + } + + $handshake_xml = $curlResult->getBody(); + $html_code = $curlResult->getReturnCode(); + + Logger::log('handshake with url ' . $url . ' returns xml: ' . $handshake_xml, Logger::DATA); + + if (!strlen($handshake_xml) || ($html_code >= 400) || !$html_code) { + // dead connection - might be a transient event, or this might + // mean the software was uninstalled or the domain expired. + // Will keep trying for one month. + Logger::log("$url appears to be dead - marking for death "); + + // set the last-update so we don't keep polling + $fields = ['last-update' => $updated, 'failure_update' => $updated]; + self::updateContact($contact, $fields); + Contact::markForArchival($contact); + return false; + } + + if (!strstr($handshake_xml, '<')) { + Logger::log('response from ' . $url . ' did not contain XML.'); + + $fields = ['last-update' => $updated, 'failure_update' => $updated]; + self::updateContact($contact, $fields); + Contact::markForArchival($contact); + return false; + } + + $res = XML::parseString($handshake_xml); + + if (intval($res->status) == 1) { + // we may not be friends anymore. Will keep trying for one month. + Logger::log("$url replied status 1 - marking for death "); + + // set the last-update so we don't keep polling + $fields = ['last-update' => $updated, 'failure_update' => $updated]; + self::updateContact($contact, $fields); + Contact::markForArchival($contact); + } elseif ($contact['term-date'] > DBA::NULL_DATETIME) { + Contact::unmarkForArchival($contact); + } + + if ((intval($res->status) != 0) || !strlen($res->challenge) || !strlen($res->dfrn_id)) { + // set the last-update so we don't keep polling + DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); + Logger::log('Contact status is ' . $res->status); + return false; + } + + if (((float)$res->dfrn_version > 2.21) && ($contact['poco'] == '')) { + $fields = ['poco' => str_replace('/profile/', '/poco/', $contact['url'])]; + DBA::update('contact', $fields, ['id' => $contact['id']]); + } + + $postvars = []; + + $sent_dfrn_id = hex2bin((string) $res->dfrn_id); + $challenge = hex2bin((string) $res->challenge); + + $final_dfrn_id = ''; + + if ($contact['duplex'] && strlen($contact['prvkey'])) { + openssl_private_decrypt($sent_dfrn_id, $final_dfrn_id, $contact['prvkey']); + openssl_private_decrypt($challenge, $postvars['challenge'], $contact['prvkey']); + } else { + openssl_public_decrypt($sent_dfrn_id, $final_dfrn_id, $contact['pubkey']); + openssl_public_decrypt($challenge, $postvars['challenge'], $contact['pubkey']); + } + + $final_dfrn_id = substr($final_dfrn_id, 0, strpos($final_dfrn_id, '.')); + + if (strpos($final_dfrn_id, ':') == 1) { + $final_dfrn_id = substr($final_dfrn_id, 2); + } + + // There are issues with the legacy DFRN transport layer. + // Since we mostly don't use it anyway, we won't dig into it deeper, but simply ignore it. + if (empty($final_dfrn_id) || empty($orig_id)) { + Logger::log('Contact has got no ID - quitting'); + return false; + } + + if ($final_dfrn_id != $orig_id) { + // did not decode properly - cannot trust this site + Logger::log('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id); + + // set the last-update so we don't keep polling + DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); + Contact::markForArchival($contact); + return false; + } + + $postvars['dfrn_id'] = $idtosend; + $postvars['dfrn_version'] = DFRN_PROTOCOL_VERSION; + $postvars['perm'] = 'rw'; + + return Network::post($contact['poll'], $postvars)->getBody(); + } + + /** + * @brief Poll Feed/OStatus contacts + * + * @param array $contact The personal contact entry + * @param string $protocol The used protocol of the contact + * @param string $updated The updated date + * @return string polled XML + * @throws \Exception + */ + private static function pollFeed(array $contact, $protocol, $updated) + { + // Upgrading DB fields from an older Friendica version + // Will only do this once per notify-enabled OStatus contact + // or if relationship changes + + $stat_writeable = ((($contact['notify']) && ($contact['rel'] == Contact::FOLLOWER || $contact['rel'] == Contact::FRIEND)) ? 1 : 0); + + // Contacts from OStatus are always writable + if ($protocol === Protocol::OSTATUS) { + $stat_writeable = 1; + } + + if ($stat_writeable != $contact['writable']) { + $fields = ['writable' => $stat_writeable]; + DBA::update('contact', $fields, ['id' => $contact['id']]); + } + + // Are we allowed to import from this person? + if ($contact['rel'] == Contact::FOLLOWER || $contact['blocked']) { + // set the last-update so we don't keep polling + DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); + Logger::log('Contact is blocked or only a follower'); + return false; + } + + $cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-'); + $curlResult = Network::curl($contact['poll'], false, ['cookiejar' => $cookiejar]); + unlink($cookiejar); + + if ($curlResult->isTimeout()) { + // set the last-update so we don't keep polling + self::updateContact($contact, ['last-update' => $updated]); + Contact::markForArchival($contact); + Logger::log('Contact archived'); + return false; + } + + return $curlResult->getBody(); + } + + /** + * @brief Poll Mail contacts + * + * @param array $contact The personal contact entry + * @param integer $importer_uid The UID of the importer + * @param string $updated The updated date + * @throws \Exception + */ + private static function pollMail(array $contact, $importer_uid, $updated) + { + Logger::log("Mail: Fetching for ".$contact['addr'], Logger::DEBUG); + + $mail_disabled = ((function_exists('imap_open') && !Config::get('system', 'imap_disabled')) ? 0 : 1); + if ($mail_disabled) { + // set the last-update so we don't keep polling + self::updateContact($contact, ['last-update' => $updated]); + Contact::markForArchival($contact); + Logger::log('Contact archived'); + return; + } + + Logger::log("Mail: Enabled", Logger::DEBUG); + + $mbox = null; + $user = DBA::selectFirst('user', ['prvkey'], ['uid' => $importer_uid]); + + $condition = ["`server` != '' AND `uid` = ?", $importer_uid]; + $mailconf = DBA::selectFirst('mailacct', [], $condition); + if (DBA::isResult($user) && DBA::isResult($mailconf)) { + $mailbox = Email::constructMailboxName($mailconf); + $password = ''; + openssl_private_decrypt(hex2bin($mailconf['pass']), $password, $user['prvkey']); + $mbox = Email::connect($mailbox, $mailconf['user'], $password); + unset($password); + Logger::log("Mail: Connect to " . $mailconf['user']); + if ($mbox) { + $fields = ['last_check' => $updated]; + DBA::update('mailacct', $fields, ['id' => $mailconf['id']]); + Logger::log("Mail: Connected to " . $mailconf['user']); + } else { + Logger::log("Mail: Connection error ".$mailconf['user']." ".print_r(imap_errors(), true)); + } + } + + if (!$mbox) { + return; + } + + $msgs = Email::poll($mbox, $contact['addr']); + + if (count($msgs)) { + Logger::log("Mail: Parsing ".count($msgs)." mails from ".$contact['addr']." for ".$mailconf['user'], Logger::DEBUG); + + $metas = Email::messageMeta($mbox, implode(',', $msgs)); + + if (count($metas) != count($msgs)) { + Logger::log("for " . $mailconf['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", Logger::DEBUG); + } else { + $msgs = array_combine($msgs, $metas); + + foreach ($msgs as $msg_uid => $meta) { + Logger::log("Mail: Parsing mail ".$msg_uid, Logger::DATA); + + $datarray = []; + $datarray['verb'] = ACTIVITY_POST; + $datarray['object-type'] = ACTIVITY_OBJ_NOTE; + $datarray['network'] = Protocol::MAIL; + // $meta = Email::messageMeta($mbox, $msg_uid); + + $datarray['uri'] = Email::msgid2iri(trim($meta->message_id, '<>')); + + // Have we seen it before? + $fields = ['deleted', 'id']; + $condition = ['uid' => $importer_uid, 'uri' => $datarray['uri']]; + $item = Item::selectFirst($fields, $condition); + if (DBA::isResult($item)) { + Logger::log("Mail: Seen before ".$msg_uid." for ".$mailconf['user']." UID: ".$importer_uid." URI: ".$datarray['uri'],Logger::DEBUG); + + // Only delete when mails aren't automatically moved or deleted + if (($mailconf['action'] != 1) && ($mailconf['action'] != 3)) + if ($meta->deleted && ! $item['deleted']) { + $fields = ['deleted' => true, 'changed' => $updated]; + Item::update($fields, ['id' => $item['id']]); + } + + switch ($mailconf['action']) { + case 0: + Logger::log("Mail: Seen before ".$msg_uid." for ".$mailconf['user'].". Doing nothing.", Logger::DEBUG); + break; + case 1: + Logger::log("Mail: Deleting ".$msg_uid." for ".$mailconf['user']); + imap_delete($mbox, $msg_uid, FT_UID); + break; + case 2: + Logger::log("Mail: Mark as seen ".$msg_uid." for ".$mailconf['user']); + imap_setflag_full($mbox, $msg_uid, "\\Seen", ST_UID); + break; + case 3: + Logger::log("Mail: Moving ".$msg_uid." to ".$mailconf['movetofolder']." for ".$mailconf['user']); + imap_setflag_full($mbox, $msg_uid, "\\Seen", ST_UID); + if ($mailconf['movetofolder'] != "") { + imap_mail_move($mbox, $msg_uid, $mailconf['movetofolder'], FT_UID); + } + break; + } + continue; + } + + // look for a 'references' or an 'in-reply-to' header and try to match with a parent item we have locally. + $raw_refs = ((property_exists($meta, 'references')) ? str_replace("\t", '', $meta->references) : ''); + if (!trim($raw_refs)) { + $raw_refs = ((property_exists($meta, 'in_reply_to')) ? str_replace("\t", '', $meta->in_reply_to) : ''); + } + $raw_refs = trim($raw_refs); // Don't allow a blank reference in $refs_arr + + if ($raw_refs) { + $refs_arr = explode(' ', $raw_refs); + if (count($refs_arr)) { + for ($x = 0; $x < count($refs_arr); $x ++) { + $refs_arr[$x] = Email::msgid2iri(str_replace(['<', '>', ' '],['', '', ''], $refs_arr[$x])); + } + } + $condition = ['uri' => $refs_arr, 'uid' => $importer_uid]; + $parent = Item::selectFirst(['parent-uri'], $condition); + if (DBA::isResult($parent)) { + $datarray['parent-uri'] = $parent['parent-uri']; // Set the parent as the top-level item + } + } + + // Decoding the header + $subject = imap_mime_header_decode($meta->subject); + $datarray['title'] = ""; + foreach ($subject as $subpart) { + if ($subpart->charset != "default") { + $datarray['title'] .= iconv($subpart->charset, 'UTF-8//IGNORE', $subpart->text); + } else { + $datarray['title'] .= $subpart->text; + } + } + $datarray['title'] = Strings::escapeTags(trim($datarray['title'])); + + //$datarray['title'] = Strings::escapeTags(trim($meta->subject)); + $datarray['created'] = DateTimeFormat::utc($meta->date); + + // Is it a reply? + $reply = ((substr(strtolower($datarray['title']), 0, 3) == "re:") || + (substr(strtolower($datarray['title']), 0, 3) == "re-") || + ($raw_refs != "")); + + // Remove Reply-signs in the subject + $datarray['title'] = self::RemoveReply($datarray['title']); + + // If it seems to be a reply but a header couldn't be found take the last message with matching subject + if (empty($datarray['parent-uri']) && $reply) { + $condition = ['title' => $datarray['title'], 'uid' => $importer_uid, 'network' => Protocol::MAIL]; + $params = ['order' => ['created' => true]]; + $parent = Item::selectFirst(['parent-uri'], $condition, $params); + if (DBA::isResult($parent)) { + $datarray['parent-uri'] = $parent['parent-uri']; + } + } + + if (empty($datarray['parent-uri'])) { + $datarray['parent-uri'] = $datarray['uri']; + } + + $r = Email::getMessage($mbox, $msg_uid, $reply); + if (!$r) { + Logger::log("Mail: can't fetch msg ".$msg_uid." for ".$mailconf['user']); + continue; + } + $datarray['body'] = Strings::escapeHtml($r['body']); + $datarray['body'] = BBCode::limitBodySize($datarray['body']); + + Logger::log("Mail: Importing ".$msg_uid." for ".$mailconf['user']); + + /// @TODO Adding a gravatar for the original author would be cool + + $from = imap_mime_header_decode($meta->from); + $fromdecoded = ""; + foreach ($from as $frompart) { + if ($frompart->charset != "default") { + $fromdecoded .= iconv($frompart->charset, 'UTF-8//IGNORE', $frompart->text); + } else { + $fromdecoded .= $frompart->text; + } + } + + $fromarr = imap_rfc822_parse_adrlist($fromdecoded, $a->getHostName()); + + $frommail = $fromarr[0]->mailbox."@".$fromarr[0]->host; + + if (isset($fromarr[0]->personal)) { + $fromname = $fromarr[0]->personal; + } else { + $fromname = $frommail; + } + + $datarray['author-name'] = $fromname; + $datarray['author-link'] = "mailto:".$frommail; + $datarray['author-avatar'] = $contact['photo']; + + $datarray['owner-name'] = $contact['name']; + $datarray['owner-link'] = "mailto:".$contact['addr']; + $datarray['owner-avatar'] = $contact['photo']; + + $datarray['uid'] = $importer_uid; + $datarray['contact-id'] = $contact['id']; + if ($datarray['parent-uri'] === $datarray['uri']) { + $datarray['private'] = 1; + } + if (!PConfig::get($importer_uid, 'system', 'allow_public_email_replies')) { + $datarray['private'] = 1; + $datarray['allow_cid'] = '<' . $contact['id'] . '>'; + } + + Item::insert($datarray); + + switch ($mailconf['action']) { + case 0: + Logger::log("Mail: Seen before ".$msg_uid." for ".$mailconf['user'].". Doing nothing.", Logger::DEBUG); + break; + case 1: + Logger::log("Mail: Deleting ".$msg_uid." for ".$mailconf['user']); + imap_delete($mbox, $msg_uid, FT_UID); + break; + case 2: + Logger::log("Mail: Mark as seen ".$msg_uid." for ".$mailconf['user']); + imap_setflag_full($mbox, $msg_uid, "\\Seen", ST_UID); + break; + case 3: + Logger::log("Mail: Moving ".$msg_uid." to ".$mailconf['movetofolder']." for ".$mailconf['user']); + imap_setflag_full($mbox, $msg_uid, "\\Seen", ST_UID); + if ($mailconf['movetofolder'] != "") { + imap_mail_move($mbox, $msg_uid, $mailconf['movetofolder'], FT_UID); + } + break; + } + } + } + } else { + Logger::log("Mail: no mails for ".$mailconf['user']); + } + + Logger::log("Mail: closing connection for ".$mailconf['user']); + imap_close($mbox); + } } diff --git a/src/Worker/UpdateContact.php b/src/Worker/UpdateContact.php index ae3b06b506..f23c5c0a07 100644 --- a/src/Worker/UpdateContact.php +++ b/src/Worker/UpdateContact.php @@ -13,16 +13,12 @@ use Friendica\Database\DBA; class UpdateContact { - public static function execute($contact_id) + public static function execute($contact_id, $command = '') { - $success = Contact::updateFromProbe($contact_id); - // Update the "updated" field if the contact could be probed. - // We don't do this in the function above, since we don't want to - // update the contact whenever that function is called from anywhere. - if ($success) { - DBA::update('contact', ['updated' => DateTimeFormat::utcNow()], ['id' => $contact_id]); - } + $force = ($command == "force"); - Logger::info('Updated from probe', ['id' => $contact_id, 'success' => $success]); + $success = Contact::updateFromProbe($contact_id, '', $force); + + Logger::info('Updated from probe', ['id' => $contact_id, 'force' => $force, 'success' => $success]); } } diff --git a/tests/include/ApiTest.php b/tests/include/ApiTest.php index 6f1172666a..a0c1e47f4c 100644 --- a/tests/include/ApiTest.php +++ b/tests/include/ApiTest.php @@ -55,12 +55,12 @@ class ApiTest extends DatabaseTest $configLoader = new ConfigFileLoader($basePath, $mode); $configCache = Factory\ConfigFactory::createCache($configLoader); $profiler = Factory\ProfilerFactory::create($configCache); - Factory\DBFactory::init($configCache, $profiler, $_SERVER); + $database = Factory\DBFactory::init($configCache, $profiler, $_SERVER); $config = Factory\ConfigFactory::createConfig($configCache); Factory\ConfigFactory::createPConfig($configCache); - $logger = Factory\LoggerFactory::create('test', $config, $profiler); + $logger = Factory\LoggerFactory::create('test', $database, $config, $profiler); $baseUrl = new BaseURL($config, $_SERVER); - $this->app = new App($config, $mode, $router, $baseUrl, $logger, $profiler, false); + $this->app = new App($database, $config, $mode, $router, $baseUrl, $logger, $profiler, false); parent::setUp(); diff --git a/tests/src/Content/Text/BBCodeTest.php b/tests/src/Content/Text/BBCodeTest.php index 3efb408937..6ae28e8c29 100644 --- a/tests/src/Content/Text/BBCodeTest.php +++ b/tests/src/Content/Text/BBCodeTest.php @@ -40,6 +40,9 @@ class BBCodeTest extends MockedTest $this->configMock->shouldReceive('get') ->with('system', 'url') ->andReturn('friendica.local'); + $this->configMock->shouldReceive('get') + ->with('system', 'no_smilies') + ->andReturn(false); $this->mockL10nT(); } diff --git a/tests/src/Database/DBATest.php b/tests/src/Database/DBATest.php index 36bba1e65b..1443d99200 100644 --- a/tests/src/Database/DBATest.php +++ b/tests/src/Database/DBATest.php @@ -20,12 +20,12 @@ class DBATest extends DatabaseTest $configLoader = new ConfigFileLoader($basePath, $mode); $configCache = Factory\ConfigFactory::createCache($configLoader); $profiler = Factory\ProfilerFactory::create($configCache); - Factory\DBFactory::init($configCache, $profiler, $_SERVER); + $database = Factory\DBFactory::init($configCache, $profiler, $_SERVER); $config = Factory\ConfigFactory::createConfig($configCache); Factory\ConfigFactory::createPConfig($configCache); - $logger = Factory\LoggerFactory::create('test', $config, $profiler); + $logger = Factory\LoggerFactory::create('test', $database, $config, $profiler); $baseUrl = new BaseURL($config, $_SERVER); - $this->app = new App($config, $mode, $router, $baseUrl, $logger, $profiler, false); + $this->app = new App($database, $config, $mode, $router, $baseUrl, $logger, $profiler, false); parent::setUp(); diff --git a/tests/src/Database/DBStructureTest.php b/tests/src/Database/DBStructureTest.php index 6050b7073a..ada73476a9 100644 --- a/tests/src/Database/DBStructureTest.php +++ b/tests/src/Database/DBStructureTest.php @@ -20,12 +20,12 @@ class DBStructureTest extends DatabaseTest $configLoader = new ConfigFileLoader($basePath, $mode); $configCache = Factory\ConfigFactory::createCache($configLoader); $profiler = Factory\ProfilerFactory::create($configCache); - Factory\DBFactory::init($configCache, $profiler, $_SERVER); + $database = Factory\DBFactory::init($configCache, $profiler, $_SERVER); $config = Factory\ConfigFactory::createConfig($configCache); Factory\ConfigFactory::createPConfig($configCache); - $logger = Factory\LoggerFactory::create('test', $config, $profiler); + $logger = Factory\LoggerFactory::create('test', $database, $config, $profiler); $baseUrl = new BaseURL($config, $_SERVER); - $this->app = new App($config, $mode, $router, $baseUrl, $logger, $profiler, false); + $this->app = new App($database, $config, $mode, $router, $baseUrl, $logger, $profiler, false); parent::setUp(); }