Tupambae_Org/friendica_2021-01
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #6910 from MrPetovan/task/6778-storage-move-loop

Browse source 
6778 Alternative 2: Wrap storage move in a loop
This commit is contained in:
Michael Vogel 2019-03-20 15:31:38 +01:00 committed by GitHub
parent 3a0a6cd659 385e3a09f2
commit 591d7fcbff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 108 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
mod/admin.php
View file

@ -1200,7 +1200,9 @@ function admin_page_site_post(App $a)
* @var $storagebackend \Friendica\Model\Storage\IStorage * @var $storagebackend \Friendica\Model\Storage\IStorage
*/ */
$storagebackend = Strings::escapeTags(trim(defaults($_POST, 'storagebackend', ''))); $storagebackend = Strings::escapeTags(trim(defaults($_POST, 'storagebackend', '')));
StorageManager::setBackend($storagebackend); if (!StorageManager::setBackend($storagebackend)) {
info(L10n::t('Invalid storage backend setting value.'));
}
// save storage backend form // save storage backend form
if (!is_null($storagebackend) && $storagebackend != "") { if (!is_null($storagebackend) && $storagebackend != "") {

53
src/Core/Console/Storage.php
View file

@ -28,9 +28,10 @@ Synopsis
Set current storage backend Set current storage backend
name storage backend to use. see "list". name storage backend to use. see "list".
bin/console storage move [table] bin/console storage move [table] [-n 5000]
Move stored data to current storage backend. Move stored data to current storage backend.
table one of "photo" or "attach". default to both table one of "photo" or "attach". default to both
-n limit of processed entry batch size
HELP; HELP;
return $help; return $help;
} }
@ -49,36 +50,36 @@ HELP;
return -1; return -1;
} }
switch($this->args[0]) { switch ($this->args[0]) {
case 'list': case 'list':
return $this->do_list(); return $this->doList();
break; break;
case 'set': case 'set':
return $this->do_set(); return $this->doSet();
break; break;
case 'move': case 'move':
return $this->do_move(); return $this->doMove();
break; break;
} }
$this->out(sprintf('Invalid action "%s"', $this->args[0])); $this->out(sprintf('Invalid action "%s"', $this->args[0]));
return -1; return -1;
} }
protected function do_list() protected function doList()
{ {
$rowfmt = ' %-3s | %-20s'; $rowfmt = ' %-3s | %-20s';
$current = StorageManager::getBackend(); $current = StorageManager::getBackend();
$this->out(sprintf($rowfmt, 'Sel', 'Name')); $this->out(sprintf($rowfmt, 'Sel', 'Name'));
$this->out('-----------------------'); $this->out('-----------------------');
$isregisterd = false; $isregisterd = false;
foreach(StorageManager::listBackends() as $name => $class) { foreach (StorageManager::listBackends() as $name => $class) {
$issel = ' '; $issel = ' ';
if ($current === $class) { if ($current === $class) {
$issel = '*'; $issel = '*';
$isregisterd = true; $isregisterd = true;
}; };
$this->out(sprintf($rowfmt, $issel , $name )); $this->out(sprintf($rowfmt, $issel, $name));
} }
if ($current === '') { if ($current === '') {
@ -92,7 +93,7 @@ HELP;
return 0; return 0;
} }
protected function do_set() protected function doSet()
{ {
if (count($this->args) !== 2) { if (count($this->args) !== 2) {
throw new CommandArgsException('Invalid arguments'); throw new CommandArgsException('Invalid arguments');
@ -106,11 +107,15 @@ HELP;
return -1; return -1;
} }
StorageManager::setBackend($class); if (!StorageManager::setBackend($class)) {
$this->out($class . ' is not a valid backend storage class.');
return -1;
}
return 0; return 0;
} }
protected function do_move() protected function doMove()
{ {
$tables = null; $tables = null;
if (count($this->args) < 1 || count($this->args) > 2) { if (count($this->args) < 1 || count($this->args) > 2) {
@ -126,7 +131,17 @@ HELP;
} }
$current = StorageManager::getBackend(); $current = StorageManager::getBackend();
$r = StorageManager::move($current, $tables); $total = 0;
$this->out(sprintf('Moved %d files', $r));
do {
$moved = StorageManager::move($current, $tables, $this->getOption('n', 5000));
if ($moved) {
$this->out(date('[Y-m-d H:i:s] ') . sprintf('Moved %d files', $moved));
}
$total += $moved;
} while ($moved);
$this->out(sprintf(date('[Y-m-d H:i:s] ') . 'Moved %d files total', $total));
} }
} }

79
src/Core/StorageManager.php
View file

@ -23,7 +23,7 @@ class StorageManager
private static function setup() private static function setup()
{ {
if (count(self::$backends)==0) { if (count(self::$backends) == 0) {
self::$backends = Config::get('storage', 'backends', self::$default_backends); self::$backends = Config::get('storage', 'backends', self::$default_backends);
} }
} }
@ -54,12 +54,18 @@ class StorageManager
* @brief Set current storage backend class * @brief Set current storage backend class
* *
* @param string $class Backend class name * @param string $class Backend class name
* @return bool
* @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/ */
public static function setBackend($class) public static function setBackend($class)
{ {
/// @todo Check that $class implements IStorage if (!in_array('Friendica\Model\Storage\IStorage', class_implements($class))) {
return false;
}
Config::set('storage', 'class', $class); Config::set('storage', 'class', $class);
return true;
} }
/** /**
@ -105,20 +111,20 @@ class StorageManager
/** /**
* @brief Move resources to storage $dest * @brief Move up to 5000 resources to storage $dest
* *
* Copy existing data to destination storage and delete from source. * Copy existing data to destination storage and delete from source.
* This method cannot move to legacy in-table `data` field. * This method cannot move to legacy in-table `data` field.
* *
* @param string $dest Destination storage class name * @param string $destination Storage class name
* @param array|null $tables Tables to look in for resources. Optional, defaults to ['photo', 'attach'] * @param array|null $tables Tables to look in for resources. Optional, defaults to ['photo', 'attach']
* * @param int $limit Limit of the process batch size, defaults to 5000
* @throws \Exception
* @return int Number of moved resources * @return int Number of moved resources
* @throws \Exception
*/ */
public static function move($dest, $tables = null) public static function move($destination, $tables = null, $limit = 5000)
{ {
if (is_null($dest) || empty($dest)) { if (empty($destination)) {
throw new \Exception('Can\'t move to NULL storage backend'); throw new \Exception('Can\'t move to NULL storage backend');
} }
@ -129,43 +135,42 @@ class StorageManager
$moved = 0; $moved = 0;
foreach ($tables as $table) { foreach ($tables as $table) {
// Get the rows where backend class is not the destination backend class // Get the rows where backend class is not the destination backend class
$rr = DBA::select( $resources = DBA::select(
$table, $table,
['id', 'data', 'backend-class', 'backend-ref'], ['id', 'data', 'backend-class', 'backend-ref'],
['`backend-class` IS NULL or `backend-class` != ?' , $dest ] ['`backend-class` IS NULL or `backend-class` != ?', $destination],
['limit' => $limit]
); );
if (DBA::isResult($rr)) { while ($resource = DBA::fetch($resources)) {
while($r = DBA::fetch($rr)) { $id = $resource['id'];
$id = $r['id']; $data = $resource['data'];
$data = $r['data']; /** @var IStorage $backendClass */
/** @var IStorage $backendClass */ $backendClass = $resource['backend-class'];
$backendClass = $r['backend-class']; $backendRef = $resource['backend-ref'];
$backendRef = $r['backend-ref']; if (!empty($backendClass)) {
if (!is_null($backendClass) && $backendClass !== '') { Logger::log("get data from old backend " . $backendClass . " : " . $backendRef);
Logger::log("get data from old backend " . $backendClass . " : " . $backendRef); $data = $backendClass::get($backendRef);
$data = $backendClass::get($backendRef); }
}
Logger::log("save data to new backend " . $dest);
/** @var IStorage $dest */
$ref = $dest::put($data);
Logger::log("saved data as " . $ref);
if ($ref !== '') { Logger::log("save data to new backend " . $destination);
Logger::log("update row"); /** @var IStorage $destination */
$ru = DBA::update($table, ['backend-class' => $dest, 'backend-ref' => $ref, 'data' => ''], ['id' => $id]); $ref = $destination::put($data);
Logger::log("saved data as " . $ref);
if ($ru) {
if (!is_null($backendClass) && $backendClass !== '') { if ($ref !== '') {
Logger::log("delete data from old backend " . $backendClass . " : " . $backendRef); Logger::log("update row");
$backendClass::delete($backendRef); if (DBA::update($table, ['backend-class' => $destination, 'backend-ref' => $ref, 'data' => ''], ['id' => $id])) {
} if (!empty($backendClass)) {
$moved++; Logger::log("delete data from old backend " . $backendClass . " : " . $backendRef);
$backendClass::delete($backendRef);
} }
$moved++;
} }
} }
} }
DBA::close($resources);
} }
return $moved; return $moved;

2
src/Database/DBA.php
View file

@ -1484,7 +1484,7 @@ class DBA
} }
$limit_string = ''; $limit_string = '';
if (isset($params['limit']) && is_int($params['limit'])) { if (isset($params['limit']) && is_numeric($params['limit'])) {
$limit_string = " LIMIT " . intval($params['limit']); $limit_string = " LIMIT " . intval($params['limit']);
} }

68
src/Worker/CronJobs.php
View file

@ -35,54 +35,42 @@ class CronJobs
Logger::log("Starting cronjob " . $command, Logger::DEBUG); Logger::log("Starting cronjob " . $command, Logger::DEBUG);
// Call possible post update functions switch($command) {
// see src/Database/PostUpdate.php for more details case 'post_update':
if ($command == 'post_update') { PostUpdate::update();
PostUpdate::update(); break;
return;
}
// update nodeinfo data case 'nodeinfo':
if ($command == 'nodeinfo') { nodeinfo_cron();
nodeinfo_cron(); break;
return;
}
// Expire and remove user entries case 'expire_and_remove_users':
if ($command == 'expire_and_remove_users') { self::expireAndRemoveUsers();
self::expireAndRemoveUsers(); break;
return;
}
if ($command == 'update_contact_birthdays') { case 'update_contact_birthdays':
Contact::updateBirthdays(); Contact::updateBirthdays();
return; break;
}
if ($command == 'update_photo_albums') { case 'update_photo_albums':
self::updatePhotoAlbums(); self::updatePhotoAlbums();
return; break;
}
// Clear cache entries case 'clear_cache':
if ($command == 'clear_cache') { self::clearCache($a);
self::clearCache($a); break;
return;
}
// Repair missing Diaspora values in contacts case 'repair_diaspora':
if ($command == 'repair_diaspora') { self::repairDiaspora($a);
self::repairDiaspora($a); break;
return;
}
// Repair entries in the database case 'repair_database':
if ($command == 'repair_database') { self::repairDatabase();
self::repairDatabase(); break;
return;
}
Logger::log("Xronjob " . $command . " is unknown.", Logger::DEBUG); default:
Logger::log("Xronjob " . $command . " is unknown.", Logger::DEBUG);
}
return; return;
} }