Merge pull request #2836 from Hypolite/Issue-#2816-2

Fix Issue #2816 - Unable to save config items present in .htconfig.php but not in DB
This commit is contained in:
Michael Vogel 2016-10-07 15:28:19 +02:00 committed by GitHub
commit 43df08f860
8 changed files with 137 additions and 89 deletions

View File

@ -38,7 +38,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica');
define ( 'FRIENDICA_CODENAME', 'Asparagus'); define ( 'FRIENDICA_CODENAME', 'Asparagus');
define ( 'FRIENDICA_VERSION', '3.5.1-dev' ); define ( 'FRIENDICA_VERSION', '3.5.1-dev' );
define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' );
define ( 'DB_UPDATE_VERSION', 1204 ); define ( 'DB_UPDATE_VERSION', 1205 );
/** /**
* @brief Constant with a HTML line break. * @brief Constant with a HTML line break.

View File

@ -1,6 +1,6 @@
-- ------------------------------------------ -- ------------------------------------------
-- Friendica 3.5.1-dev (Asparagus) -- Friendica 3.5.1-dev (Asparagus)
-- DB_UPDATE_VERSION 1204 -- DB_UPDATE_VERSION 1205
-- ------------------------------------------ -- ------------------------------------------
@ -97,7 +97,7 @@ CREATE TABLE IF NOT EXISTS `config` (
`k` varchar(255) NOT NULL DEFAULT '', `k` varchar(255) NOT NULL DEFAULT '',
`v` text, `v` text,
PRIMARY KEY(`id`), PRIMARY KEY(`id`),
INDEX `cat_k` (`cat`(30),`k`(30)) UNIQUE INDEX `cat_k` (`cat`(30),`k`(30))
) DEFAULT CHARSET=utf8mb4; ) DEFAULT CHARSET=utf8mb4;
-- --
@ -707,7 +707,7 @@ CREATE TABLE IF NOT EXISTS `pconfig` (
`k` varchar(255) NOT NULL DEFAULT '', `k` varchar(255) NOT NULL DEFAULT '',
`v` mediumtext, `v` mediumtext,
PRIMARY KEY(`id`), PRIMARY KEY(`id`),
INDEX `uid_cat_k` (`uid`,`cat`(30),`k`(30)) UNIQUE INDEX `uid_cat_k` (`uid`,`cat`(30),`k`(30))
) DEFAULT CHARSET=utf8mb4; ) DEFAULT CHARSET=utf8mb4;
-- --

34
doc/upgrade.md Normal file
View File

@ -0,0 +1,34 @@
# Considerations before upgrading Friendica
* [Home](help)
## MySQL >= 5.7.4
Starting from MySQL version 5.7.4, the IGNORE keyword in ALTER TABLE statements is ignored.
This prevents automatic table deduplication if a UNIQUE index is added to a Friendica table's structure.
If a DB update fails for you while creating a UNIQUE index, make sure to manually deduplicate the table before trying the update again.
### Manual deduplication
There are two main ways of doing it, either by manually removing the duplicates or by recreating the table.
Manually removing the duplicates is usually faster if they're not too numerous.
To manually remove the duplicates, you need to know the UNIQUE index columns available in `database.sql`.
```SQL
SELECT GROUP_CONCAT(id), <index columns>, count(*) as count FROM users
GROUP BY <index columns> HAVING count >= 2;
/* delete or merge duplicate from above query */;
```
If there are too many rows to handle manually, you can create a new table with the same structure as the table with duplicates and insert the existing content with INSERT IGNORE.
To recreate the table you need to know the table structure available in `database.sql`.
```SQL
CREATE TABLE <table_name>_new <rest of the CREATE TABLE>;
INSERT IGNORE INTO <table_name>_new SELECT * FROM <table_name>;
DROP TABLE <table_name>;
RENAME TABLE <table_name>_new TO <table_name>;
```
This method is slower overall, but it is better suited for large numbers of duplicates.

View File

@ -126,22 +126,17 @@ class Config {
public static function set($family,$key,$value) { public static function set($family,$key,$value) {
global $a; global $a;
// If $a->config[$family] has been previously set to '!<unset>!', then $a->config[$family][$key] = $value;
// $a->config[$family][$key] will evaluate to $a->config[$family][0], and
// $a->config[$family][$key] = $value will be equivalent to
// $a->config[$family][0] = $value[0] (this causes infuriating bugs),
// so unset the family before assigning a value to a family's key
if($a->config[$family] === '!<unset>!')
unset($a->config[$family]);
// manage array value // manage array value
$dbvalue = (is_array($value)?serialize($value):$value); $dbvalue = (is_array($value)?serialize($value):$value);
$dbvalue = (is_bool($dbvalue) ? intval($dbvalue) : $dbvalue); $dbvalue = (is_bool($dbvalue) ? intval($dbvalue) : $dbvalue);
if(is_null(self::get($family,$key,null,true))) {
$a->config[$family][$key] = $value; $ret = q("INSERT INTO `config` ( `cat`, `k`, `v` ) VALUES ( '%s', '%s', '%s' )
$ret = q("INSERT INTO `config` ( `cat`, `k`, `v` ) VALUES ( '%s', '%s', '%s' ) ", ON DUPLICATE KEY UPDATE `v` = '%s'",
dbesc($family), dbesc($family),
dbesc($key), dbesc($key),
dbesc($dbvalue),
dbesc($dbvalue) dbesc($dbvalue)
); );
if($ret) if($ret)
@ -149,19 +144,6 @@ class Config {
return $ret; return $ret;
} }
$ret = q("UPDATE `config` SET `v` = '%s' WHERE `cat` = '%s' AND `k` = '%s'",
dbesc($dbvalue),
dbesc($family),
dbesc($key)
);
$a->config[$family][$key] = $value;
if($ret)
return $value;
return $ret;
}
/** /**
* @brief Deletes the given key from the system configuration. * @brief Deletes the given key from the system configuration.
* *

View File

@ -126,31 +126,20 @@ class PConfig {
// manage array value // manage array value
$dbvalue = (is_array($value)?serialize($value):$value); $dbvalue = (is_array($value)?serialize($value):$value);
if(is_null(self::get($uid,$family,$key,null, true))) {
$a->config[$uid][$family][$key] = $value; $a->config[$uid][$family][$key] = $value;
$ret = q("INSERT INTO `pconfig` ( `uid`, `cat`, `k`, `v` ) VALUES ( %d, '%s', '%s', '%s' ) ",
$ret = q("INSERT INTO `pconfig` ( `uid`, `cat`, `k`, `v` ) VALUES ( %d, '%s', '%s', '%s' )
ON DUPLICATE KEY UPDATE `v` = '%s'",
intval($uid), intval($uid),
dbesc($family), dbesc($family),
dbesc($key), dbesc($key),
dbesc($dbvalue),
dbesc($dbvalue) dbesc($dbvalue)
); );
if($ret) if($ret)
return $value; return $value;
return $ret; return $ret;
} }
$ret = q("UPDATE `pconfig` SET `v` = '%s' WHERE `uid` = %d AND `cat` = '%s' AND `k` = '%s'",
dbesc($dbvalue),
intval($uid),
dbesc($family),
dbesc($key)
);
$a->config[$uid][$family][$key] = $value;
if($ret)
return $value;
return $ret;
}
/** /**
* @brief Deletes the given key from the users's configuration. * @brief Deletes the given key from the users's configuration.

View File

@ -91,6 +91,23 @@ class dba {
return $this->db; return $this->db;
} }
/**
* @brief Returns the MySQL server version string
*
* This function discriminate between the deprecated mysql API and the current
* object-oriented mysqli API. Example of returned string: 5.5.46-0+deb8u1
*
* @return string
*/
public function server_info() {
if ($this->mysqli) {
$return = $this->db->server_info;
} else {
$return = mysql_get_server_info($this->db);
}
return $return;
}
public function q($sql, $onlyquery = false) { public function q($sql, $onlyquery = false) {
global $a; global $a;

View File

@ -78,6 +78,10 @@ function table_structure($table) {
if ($index["Index_type"] == "FULLTEXT") if ($index["Index_type"] == "FULLTEXT")
continue; continue;
if ($index['Key_name'] != 'PRIMARY' && $index['Non_unique'] == '0' && !isset($indexdata[$index["Key_name"]])) {
$indexdata[$index["Key_name"]] = array('UNIQUE');
}
$column = $index["Column_name"]; $column = $index["Column_name"];
// On utf8mb4 a varchar index can only have a length of 191 // On utf8mb4 a varchar index can only have a length of 191
// To avoid the need to add this to every index definition we just ignore it here. // To avoid the need to add this to every index definition we just ignore it here.
@ -154,6 +158,19 @@ function update_structure($verbose, $action, $tables=null, $definition=null) {
if (is_null($definition)) if (is_null($definition))
$definition = db_definition($charset); $definition = db_definition($charset);
// Ensure index conversion to unique removes duplicates
$sql_config = "SET session old_alter_table=1;";
if ($verbose)
echo $sql_config."\n";
if ($action)
@$db->q($sql_config);
// MySQL >= 5.7.4 doesn't support the IGNORE keyword in ALTER TABLE statements
if (version_compare($db->server_info(), '5.7.4') >= 0) {
$ignore = '';
}else {
$ignore = ' IGNORE';
}
// Compare it // Compare it
foreach ($definition AS $name => $structure) { foreach ($definition AS $name => $structure) {
@ -223,7 +240,7 @@ function update_structure($verbose, $action, $tables=null, $definition=null) {
$sql2=db_create_index($indexname, $fieldnames); $sql2=db_create_index($indexname, $fieldnames);
if ($sql2 != "") { if ($sql2 != "") {
if ($sql3 == "") if ($sql3 == "")
$sql3 = "ALTER TABLE `".$name."` ".$sql2; $sql3 = "ALTER" . $ignore . " TABLE `".$name."` ".$sql2;
else else
$sql3 .= ", ".$sql2; $sql3 .= ", ".$sql2;
} }
@ -330,6 +347,11 @@ function db_create_index($indexname, $fieldnames, $method="ADD") {
killme(); killme();
} }
if ($fieldnames[0] == "UNIQUE") {
array_shift($fieldnames);
$method .= ' UNIQUE';
}
$names = ""; $names = "";
foreach ($fieldnames AS $fieldname) { foreach ($fieldnames AS $fieldname) {
if ($names != "") if ($names != "")
@ -457,7 +479,7 @@ function db_definition($charset) {
), ),
"indexes" => array( "indexes" => array(
"PRIMARY" => array("id"), "PRIMARY" => array("id"),
"cat_k" => array("cat(30)","k(30)"), "cat_k" => array("UNIQUE", "cat(30)","k(30)"),
) )
); );
$database["contact"] = array( $database["contact"] = array(
@ -1067,7 +1089,7 @@ function db_definition($charset) {
), ),
"indexes" => array( "indexes" => array(
"PRIMARY" => array("id"), "PRIMARY" => array("id"),
"uid_cat_k" => array("uid","cat(30)","k(30)"), "uid_cat_k" => array("UNIQUE", "uid","cat(30)","k(30)"),
) )
); );
$database["photo"] = array( $database["photo"] = array(
@ -1491,6 +1513,9 @@ function dbstructure_run(&$argv, &$argc) {
if ($argc==2) { if ($argc==2) {
switch ($argv[1]) { switch ($argv[1]) {
case "dryrun":
update_structure(true, false);
return;
case "update": case "update":
update_structure(true, true); update_structure(true, true);
@ -1523,7 +1548,8 @@ function dbstructure_run(&$argv, &$argc) {
// print help // print help
echo $argv[0]." <command>\n"; echo $argv[0]." <command>\n";
echo "\n"; echo "\n";
echo "commands:\n"; echo "Commands:\n";
echo "dryrun show database update schema queries without running them\n";
echo "update update database schema\n"; echo "update update database schema\n";
echo "dumpsql dump database schema\n"; echo "dumpsql dump database schema\n";
return; return;

View File

@ -1,6 +1,6 @@
<?php <?php
define('UPDATE_VERSION' , 1204); define('UPDATE_VERSION' , 1205);
/** /**
* *