Tupambae_Org/friendica_2021-01
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Add limit parameter to storage move query

Browse source 
- Fixes out-of-memory errors with large tables
- Add database statement closing
- Add meaningful variable names
- Remove useless DBA::isResult check
This commit is contained in:
Hypolite Petovan 2019-03-20 00:41:57 -04:00
parent 8a48ff1f95
commit 8ddbeb087f
1 changed files with 34 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
src/Core/StorageManager.php
View file

@ -111,20 +111,20 @@ class StorageManager
/** /**
* @brief Move resources to storage $dest * @brief Move up to 5000 resources to storage $dest
* *
* Copy existing data to destination storage and delete from source. * Copy existing data to destination storage and delete from source.
* This method cannot move to legacy in-table `data` field. * This method cannot move to legacy in-table `data` field.
* *
* @param string $dest Destination storage class name * @param string $destination Storage class name
* @param array|null $tables Tables to look in for resources. Optional, defaults to ['photo', 'attach'] * @param array|null $tables Tables to look in for resources. Optional, defaults to ['photo', 'attach']
* * @param int $limit Limit of the process batch size, defaults to 5000
* @throws \Exception
* @return int Number of moved resources * @return int Number of moved resources
* @throws \Exception
*/ */
public static function move($dest, $tables = null) public static function move($destination, $tables = null, $limit = 5000)
{ {
if (is_null($dest) || empty($dest)) { if (empty($destination)) {
throw new \Exception('Can\'t move to NULL storage backend'); throw new \Exception('Can\'t move to NULL storage backend');
} }
@ -135,43 +135,42 @@ class StorageManager
$moved = 0; $moved = 0;
foreach ($tables as $table) { foreach ($tables as $table) {
// Get the rows where backend class is not the destination backend class // Get the rows where backend class is not the destination backend class
$rr = DBA::select( $resources = DBA::select(
$table, $table,
['id', 'data', 'backend-class', 'backend-ref'], ['id', 'data', 'backend-class', 'backend-ref'],
['`backend-class` IS NULL or `backend-class` != ?' , $dest ] ['`backend-class` IS NULL or `backend-class` != ?', $destination],
['limit' => $limit]
); );
if (DBA::isResult($rr)) { while ($resource = DBA::fetch($resources)) {
while($r = DBA::fetch($rr)) { $id = $resource['id'];
$id = $r['id']; $data = $resource['data'];
$data = $r['data']; /** @var IStorage $backendClass */
/** @var IStorage $backendClass */ $backendClass = $resource['backend-class'];
$backendClass = $r['backend-class']; $backendRef = $resource['backend-ref'];
$backendRef = $r['backend-ref']; if (!empty($backendClass)) {
if (!is_null($backendClass) && $backendClass !== '') { Logger::log("get data from old backend " . $backendClass . " : " . $backendRef);
Logger::log("get data from old backend " . $backendClass . " : " . $backendRef); $data = $backendClass::get($backendRef);
$data = $backendClass::get($backendRef); }
}
Logger::log("save data to new backend " . $dest);
/** @var IStorage $dest */
$ref = $dest::put($data);
Logger::log("saved data as " . $ref);
if ($ref !== '') { Logger::log("save data to new backend " . $destination);
Logger::log("update row"); /** @var IStorage $destination */
$ru = DBA::update($table, ['backend-class' => $dest, 'backend-ref' => $ref, 'data' => ''], ['id' => $id]); $ref = $destination::put($data);
Logger::log("saved data as " . $ref);
if ($ru) {
if (!is_null($backendClass) && $backendClass !== '') { if ($ref !== '') {
Logger::log("delete data from old backend " . $backendClass . " : " . $backendRef); Logger::log("update row");
$backendClass::delete($backendRef); if (DBA::update($table, ['backend-class' => $destination, 'backend-ref' => $ref, 'data' => ''], ['id' => $id])) {
} if (!empty($backendClass)) {
$moved++; Logger::log("delete data from old backend " . $backendClass . " : " . $backendRef);
$backendClass::delete($backendRef);
} }
$moved++;
} }
} }
} }
DBA::close($resources);
} }
return $moved; return $moved;