Lock abstraction (like the Cache)

- adding interface
- adding seperate drivers
- moving Lock to the Core package
This commit is contained in:
Philipp Holzer 2018-06-26 22:31:04 +02:00
parent 258683b49a
commit 0218d16335
No known key found for this signature in database
GPG key ID: 58160D7D6AF942B6
8 changed files with 396 additions and 223 deletions

97
src/Core/Lock.php Normal file
View file

@ -0,0 +1,97 @@
<?php
/**
* @file src/Core/Lock.php
*/
namespace Friendica\Util;
/**
* @file src/Core/Lock.php
* @brief Functions for preventing parallel execution of functions
*/
use Friendica\Core\Config;
use Friendica\Core\Lock;
require_once 'include/dba.php';
/**
* @brief This class contain Functions for preventing parallel execution of functions
*/
class Lock
{
/**
* @var Lock\ILockDriver;
*/
static $driver = null;
public static function init()
{
switch(Config::get('system', 'lock_driver', 'default')) {
case 'memcache':
self::$driver = new Lock\MemcacheLockDriver();
break;
case 'database':
self::$driver = new Lock\DatabaseLockDriver();
break;
case 'semaphore':
self::$driver = new Lock\SemaphoreLockDriver();
break;
default:
// Determine what's the best/fastest locking driver (default behavior in Friendica)
if (function_exists('sem_get') && version_compare(PHP_VERSION, '5.6.1', '>=')) {
self::$driver = new Lock\SemaphoreLockDriver();
} elseif (Config::get('system', 'cache_driver', 'database') == 'memcache') {
self::$driver = new Lock\MemcacheLockDriver();
} else {
self::$driver = new Lock\DatabaseLockDriver();
}
}
}
/**
* Returns the current cache driver
*
* @return Lock\ILockDriver;
*/
private static function getDriver()
{
if (self::$driver === null) {
self::init();
}
return self::$driver;
}
/**
* @brief Acquires a lock for a given name
*
* @param string $key Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public static function acquireLock($key, $timeout = 120)
{
return self::getDriver()->acquireLock($key, $timeout);
}
/**
* @brief Releases a lock if it was set by us
*
* @param string $key Name of the lock
* @return mixed
*/
public static function releaseLock($key)
{
return self::getDriver()->releaseLock($key);
}
/**
* @brief Releases all lock that were set by us
* @return void
*/
public static function releaseAll()
{
self::getDriver()->releaseAll();
}
}

View file

@ -0,0 +1,83 @@
<?php
namespace Friendica\Core\Lock;
use dba;
use Friendica\Database\DBM;
/**
* Locking driver that stores the locks in the database
*/
class DatabaseLockDriver implements ILockDriver
{
/**
* @brief Sets a lock for a given name
*
* @param string $key The Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public function acquireLock($key, $timeout = 120)
{
$got_lock = false;
$start = time();
do {
dba::lock('locks');
$lock = dba::selectFirst('locks', ['locked', 'pid'], ['name' => $key]);
if (DBM::is_result($lock)) {
if ($lock['locked']) {
// When the process id isn't used anymore, we can safely claim the lock for us.
if (!posix_kill($lock['pid'], 0)) {
$lock['locked'] = false;
}
// We want to lock something that was already locked by us? So we got the lock.
if ($lock['pid'] == getmypid()) {
$got_lock = true;
}
}
if (!$lock['locked']) {
dba::update('locks', ['locked' => true, 'pid' => getmypid()], ['name' => $key]);
$got_lock = true;
}
} elseif (!DBM::is_result($lock)) {
dba::insert('locks', ['name' => $key, 'locked' => true, 'pid' => getmypid()]);
$got_lock = true;
}
dba::unlock();
if (!$got_lock && ($timeout > 0)) {
usleep(rand(100000, 2000000));
}
} while (!$got_lock && ((time() - $start) < $timeout));
return $got_lock;
}
/**
* @brief Removes a lock if it was set by us
*
* @param string $key Name of the lock
*
* @return mixed
*/
public function releaseLock($key)
{
dba::update('locks', ['locked' => false, 'pid' => 0], ['name' => $key, 'pid' => getmypid()]);
return;
}
/**
* @brief Removes all lock that were set by us
*
* @return void
*/
public function releaseAll()
{
dba::update('locks', ['locked' => false, 'pid' => 0], ['pid' => getmypid()]);
}
}

View file

@ -0,0 +1,38 @@
<?php
namespace Friendica\Core\Lock;
/**
* Lock Driver Interface
*
* @author Philipp Holzer <admin@philipp.info>
*/
interface ILockDriver
{
/**
*
* @brief Acquires a lock for a given name
*
* @param string $key The Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public function acquireLock($key, $timeout = 120);
/**
* @brief Releases a lock if it was set by us
*
* @param string $key Name of the lock
*
* @return mixed
*/
public function releaseLock($key);
/**
* @brief Releases all lock that were set by us
*
* @return void
*/
public function releaseAll();
}

View file

@ -0,0 +1,86 @@
<?php
namespace Friendica\Core\Lock;
use Friendica\Core\Cache;
use dba;
class MemcacheLockDriver implements ILockDriver
{
/**
*
* @brief Sets a lock for a given name
*
* @param string $key The Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public function acquireLock($key, $timeout = 120)
{
$got_lock = false;
$start = time();
$cachekey = get_app()->get_hostname() . ";lock:" . $key;
do {
// We only lock to be sure that nothing happens at exactly the same time
dba::lock('locks');
$lock = Cache::get($cachekey);
if (!is_bool($lock)) {
$pid = (int)$lock;
// When the process id isn't used anymore, we can safely claim the lock for us.
// Or we do want to lock something that was already locked by us.
if (!posix_kill($pid, 0) || ($pid == getmypid())) {
$lock = false;
}
}
if (is_bool($lock)) {
Cache::set($cachekey, getmypid(), 300);
$got_lock = true;
}
dba::unlock();
if (!$got_lock && ($timeout > 0)) {
usleep(rand(10000, 200000));
}
} while (!$got_lock && ((time() - $start) < $timeout));
return $got_lock;
}
/**
* @brief Removes a lock if it was set by us
*
* @param string $key Name of the lock
*
* @return mixed
*/
public function releaseLock($key)
{
$cachekey = get_app()->get_hostname() . ";lock:" . $key;
$lock = Cache::get($cachekey);
if (!is_bool($lock)) {
if ((int)$lock == getmypid()) {
Cache::delete($cachekey);
}
}
return;
}
/**
* @brief Removes all lock that were set by us
*
* @return void
*/
public function releaseAll()
{
// We cannot delete all cache entries, but this doesn't matter with memcache
return;
}
}

View file

@ -0,0 +1,82 @@
<?php
namespace Friendica\Core\Lock;
class SemaphoreLockDriver implements ILockDriver
{
/**
* @var array stored semaphores
*/
private static $semaphore = [];
/**
* @brief Creates a semaphore key
*
* @param string $key Name of the lock
*
* @return integer the semaphore key
*/
private static function semaphoreKey($key)
{
$temp = get_temppath();
$file = $temp.'/'.$key.'.sem';
if (!file_exists($file)) {
file_put_contents($file, $key);
}
return ftok($file, 'f');
}
/**
*
* @brief Sets a lock for a given name
*
* @param string $key The Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public function acquireLock($key, $timeout = 120)
{
// The second parameter for "sem_acquire" doesn't exist before 5.6.1
if (function_exists('sem_get') && version_compare(PHP_VERSION, '5.6.1', '>=')) {
self::$semaphore[$key] = sem_get(self::semaphoreKey($key));
if (self::$semaphore[$key]) {
return sem_acquire(self::$semaphore[$key], ($timeout == 0));
}
}
}
/**
* @brief Removes a lock if it was set by us
*
* @param string $key Name of the lock
*
* @return mixed
*/
public function releaseLock($key)
{
if (function_exists('sem_get') && version_compare(PHP_VERSION, '5.6.1', '>=')) {
if (empty(self::$semaphore[$key])) {
return false;
} else {
$success = @sem_release(self::$semaphore[$key]);
unset(self::$semaphore[$key]);
return $success;
}
}
}
/**
* @brief Removes all lock that were set by us
*
* @return void
*/
public function releaseAll()
{
// not needed/supported
return;
}
}

View file

@ -10,7 +10,6 @@ use Friendica\Core\System;
use Friendica\Database\DBM; use Friendica\Database\DBM;
use Friendica\Model\Process; use Friendica\Model\Process;
use Friendica\Util\DateTimeFormat; use Friendica\Util\DateTimeFormat;
use Friendica\Util\Lock;
use Friendica\Util\Network; use Friendica\Util\Network;
use dba; use dba;
@ -108,16 +107,16 @@ class Worker
} }
// If possible we will fetch new jobs for this worker // If possible we will fetch new jobs for this worker
if (!$refetched && Lock::set('worker_process', 0)) { if (!$refetched && Lock::acquireLock('worker_process', 0)) {
$stamp = (float)microtime(true); $stamp = (float)microtime(true);
$refetched = self::findWorkerProcesses($passing_slow); $refetched = self::findWorkerProcesses($passing_slow);
self::$db_duration += (microtime(true) - $stamp); self::$db_duration += (microtime(true) - $stamp);
Lock::remove('worker_process'); Lock::releaseLock('worker_process');
} }
} }
// To avoid the quitting of multiple workers only one worker at a time will execute the check // To avoid the quitting of multiple workers only one worker at a time will execute the check
if (Lock::set('worker', 0)) { if (Lock::acquireLock('worker', 0)) {
$stamp = (float)microtime(true); $stamp = (float)microtime(true);
// Count active workers and compare them with a maximum value that depends on the load // Count active workers and compare them with a maximum value that depends on the load
if (self::tooMuchWorkers()) { if (self::tooMuchWorkers()) {
@ -130,7 +129,7 @@ class Worker
logger('Memory limit reached, quitting.', LOGGER_DEBUG); logger('Memory limit reached, quitting.', LOGGER_DEBUG);
return; return;
} }
Lock::remove('worker'); Lock::releaseLock('worker');
self::$db_duration += (microtime(true) - $stamp); self::$db_duration += (microtime(true) - $stamp);
} }
@ -883,7 +882,7 @@ class Worker
dba::close($r); dba::close($r);
$stamp = (float)microtime(true); $stamp = (float)microtime(true);
if (!Lock::set('worker_process')) { if (!Lock::acquireLock('worker_process')) {
return false; return false;
} }
self::$lock_duration = (microtime(true) - $stamp); self::$lock_duration = (microtime(true) - $stamp);
@ -892,7 +891,7 @@ class Worker
$found = self::findWorkerProcesses($passing_slow); $found = self::findWorkerProcesses($passing_slow);
self::$db_duration += (microtime(true) - $stamp); self::$db_duration += (microtime(true) - $stamp);
Lock::remove('worker_process'); Lock::releaseLock('worker_process');
if ($found) { if ($found) {
$r = dba::select('workerqueue', [], ['pid' => getmypid(), 'done' => false]); $r = dba::select('workerqueue', [], ['pid' => getmypid(), 'done' => false]);
@ -1097,13 +1096,13 @@ class Worker
} }
// If there is a lock then we don't have to check for too much worker // If there is a lock then we don't have to check for too much worker
if (!Lock::set('worker', 0)) { if (!Lock::acquireLock('worker', 0)) {
return true; return true;
} }
// If there are already enough workers running, don't fork another one // If there are already enough workers running, don't fork another one
$quit = self::tooMuchWorkers(); $quit = self::tooMuchWorkers();
Lock::remove('worker'); Lock::releaseLock('worker');
if ($quit) { if ($quit) {
return true; return true;

View file

@ -19,7 +19,6 @@ use Friendica\Model\User;
use Friendica\Network\Probe; use Friendica\Network\Probe;
use Friendica\Object\Image; use Friendica\Object\Image;
use Friendica\Util\DateTimeFormat; use Friendica\Util\DateTimeFormat;
use Friendica\Util\Lock;
use Friendica\Util\Network; use Friendica\Util\Network;
use Friendica\Util\XML; use Friendica\Util\XML;
use dba; use dba;
@ -513,9 +512,9 @@ class OStatus
logger("Item with uri ".$item["uri"]." is from a blocked contact.", LOGGER_DEBUG); logger("Item with uri ".$item["uri"]." is from a blocked contact.", LOGGER_DEBUG);
} else { } else {
// We are having duplicated entries. Hopefully this solves it. // We are having duplicated entries. Hopefully this solves it.
if (Lock::set('ostatus_process_item_insert')) { if (Lock::acquireLock('ostatus_process_item_insert')) {
$ret = Item::insert($item); $ret = Item::insert($item);
Lock::remove('ostatus_process_item_insert'); Lock::releaseLock('ostatus_process_item_insert');
logger("Item with uri ".$item["uri"]." for user ".$importer["uid"].' stored. Return value: '.$ret); logger("Item with uri ".$item["uri"]." for user ".$importer["uid"].' stored. Return value: '.$ret);
} else { } else {
$ret = Item::insert($item); $ret = Item::insert($item);

View file

@ -1,211 +0,0 @@
<?php
/**
* @file src/Util/Lock.php
*/
namespace Friendica\Util;
/**
* @file src/Util/Lock.php
* @brief Functions for preventing parallel execution of functions
*/
use Friendica\Core\Config;
use Friendica\Database\DBM;
use Memcache;
use dba;
require_once 'include/dba.php';
/**
* @brief This class contain Functions for preventing parallel execution of functions
*/
class Lock
{
private static $semaphore = [];
/**
* @brief Check for memcache and open a connection if configured
*
* @return object|boolean The memcache object - or "false" if not successful
*/
private static function connectMemcache()
{
if (!function_exists('memcache_connect')) {
return false;
}
if (!Config::get('system', 'memcache')) {
return false;
}
$memcache_host = Config::get('system', 'memcache_host', '127.0.0.1');
$memcache_port = Config::get('system', 'memcache_port', 11211);
$memcache = new Memcache;
if (!$memcache->connect($memcache_host, $memcache_port)) {
return false;
}
return $memcache;
}
/**
* @brief Creates a semaphore key
*
* @param string $fn_name Name of the lock
*
* @return ressource the semaphore key
*/
private static function semaphoreKey($fn_name)
{
$temp = get_temppath();
$file = $temp.'/'.$fn_name.'.sem';
if (!file_exists($file)) {
file_put_contents($file, $fn_name);
}
return ftok($file, 'f');
}
/**
* @brief Sets a lock for a given name
*
* @param string $fn_name Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public static function set($fn_name, $timeout = 120)
{
$got_lock = false;
$start = time();
// The second parameter for "sem_acquire" doesn't exist before 5.6.1
if (function_exists('sem_get') && version_compare(PHP_VERSION, '5.6.1', '>=')) {
self::$semaphore[$fn_name] = sem_get(self::semaphoreKey($fn_name));
if (self::$semaphore[$fn_name]) {
return sem_acquire(self::$semaphore[$fn_name], ($timeout == 0));
}
}
$memcache = self::connectMemcache();
if (is_object($memcache)) {
$cachekey = get_app()->get_hostname().";lock:".$fn_name;
do {
// We only lock to be sure that nothing happens at exactly the same time
dba::lock('locks');
$lock = $memcache->get($cachekey);
if (!is_bool($lock)) {
$pid = (int)$lock;
// When the process id isn't used anymore, we can safely claim the lock for us.
// Or we do want to lock something that was already locked by us.
if (!posix_kill($pid, 0) || ($pid == getmypid())) {
$lock = false;
}
}
if (is_bool($lock)) {
$memcache->set($cachekey, getmypid(), MEMCACHE_COMPRESSED, 300);
$got_lock = true;
}
dba::unlock();
if (!$got_lock && ($timeout > 0)) {
usleep(rand(10000, 200000));
}
} while (!$got_lock && ((time() - $start) < $timeout));
return $got_lock;
}
do {
dba::lock('locks');
$lock = dba::selectFirst('locks', ['locked', 'pid'], ['name' => $fn_name]);
if (DBM::is_result($lock)) {
if ($lock['locked']) {
// When the process id isn't used anymore, we can safely claim the lock for us.
if (!posix_kill($lock['pid'], 0)) {
$lock['locked'] = false;
}
// We want to lock something that was already locked by us? So we got the lock.
if ($lock['pid'] == getmypid()) {
$got_lock = true;
}
}
if (!$lock['locked']) {
dba::update('locks', ['locked' => true, 'pid' => getmypid()], ['name' => $fn_name]);
$got_lock = true;
}
} elseif (!DBM::is_result($lock)) {
dba::insert('locks', ['name' => $fn_name, 'locked' => true, 'pid' => getmypid()]);
$got_lock = true;
}
dba::unlock();
if (!$got_lock && ($timeout > 0)) {
usleep(rand(100000, 2000000));
}
} while (!$got_lock && ((time() - $start) < $timeout));
return $got_lock;
}
/**
* @brief Removes a lock if it was set by us
*
* @param string $fn_name Name of the lock
* @return mixed
*/
public static function remove($fn_name)
{
if (function_exists('sem_get') && version_compare(PHP_VERSION, '5.6.1', '>=')) {
if (empty(self::$semaphore[$fn_name])) {
return false;
} else {
$success = @sem_release(self::$semaphore[$fn_name]);
unset(self::$semaphore[$fn_name]);
return $success;
}
}
$memcache = self::connectMemcache();
if (is_object($memcache)) {
$cachekey = get_app()->get_hostname().";lock:".$fn_name;
$lock = $memcache->get($cachekey);
if (!is_bool($lock)) {
if ((int)$lock == getmypid()) {
$memcache->delete($cachekey);
}
}
return;
}
dba::update('locks', ['locked' => false, 'pid' => 0], ['name' => $fn_name, 'pid' => getmypid()]);
return;
}
/**
* @brief Removes all lock that were set by us
* @return void
*/
public static function removeAll()
{
$memcache = self::connectMemcache();
if (is_object($memcache)) {
// We cannot delete all cache entries, but this doesn't matter with memcache
return;
}
dba::update('locks', ['locked' => false, 'pid' => 0], ['pid' => getmypid()]);
return;
}
}