Merge pull request #5291 from nupplaphil/lock_abstraction

Lock abstraction
This commit is contained in:
Hypolite Petovan 2018-07-05 15:05:18 -04:00 committed by GitHub
commit 3be013361e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 1186 additions and 344 deletions

View file

@ -4,8 +4,7 @@
*/
namespace Friendica\Core;
use Friendica\Core\Cache;
use Friendica\Core\Config;
use Friendica\Core\Cache\CacheDriverFactory;
/**
* @brief Class for storing data for a short time
@ -24,31 +23,13 @@ class Cache extends \Friendica\BaseObject
/**
* @var Cache\ICacheDriver
*/
static $driver = null;
private static $driver = null;
public static function init()
{
switch(Config::get('system', 'cache_driver', 'database')) {
case 'memcache':
$memcache_host = Config::get('system', 'memcache_host', '127.0.0.1');
$memcache_port = Config::get('system', 'memcache_port', 11211);
$driver_name = Config::get('system', 'cache_driver', 'database');
self::$driver = new Cache\MemcacheCacheDriver($memcache_host, $memcache_port);
break;
case 'memcached':
$memcached_hosts = Config::get('system', 'memcached_hosts', [['127.0.0.1', 11211]]);
self::$driver = new Cache\MemcachedCacheDriver($memcached_hosts);
break;
case 'redis':
$redis_host = Config::get('system', 'redis_host', '127.0.0.1');
$redis_port = Config::get('system', 'redis_port', 6379);
self::$driver = new Cache\RedisCacheDriver($redis_host, $redis_port);
break;
default:
self::$driver = new Cache\DatabaseCacheDriver();
}
self::$driver = CacheDriverFactory::create($driver_name);
}
/**

View file

@ -0,0 +1,83 @@
<?php
namespace Friendica\Core\Cache;
use Friendica\Core\Cache;
/**
* Implementation of the IMemoryCacheDriver mainly for testing purpose
*
* Class ArrayCache
*
* @package Friendica\Core\Cache
*/
class ArrayCache implements IMemoryCacheDriver
{
use TraitCompareDelete;
/** @var array Array with the cached data */
protected $cachedData = array();
/**
* (@inheritdoc)
*/
public function get($key)
{
if (isset($this->cachedData[$key])) {
return $this->cachedData[$key];
}
return null;
}
/**
* (@inheritdoc)
*/
public function set($key, $value, $ttl = Cache::FIVE_MINUTES)
{
$this->cachedData[$key] = $value;
return true;
}
/**
* (@inheritdoc)
*/
public function delete($key)
{
unset($this->cachedData[$key]);
return true;
}
/**
* (@inheritdoc)
*/
public function clear()
{
$this->cachedData = [];
return true;
}
/**
* (@inheritdoc)
*/
public function add($key, $value, $ttl = Cache::FIVE_MINUTES)
{
if (isset($this->cachedData[$key])) {
return false;
} else {
return $this->set($key, $value, $ttl);
}
}
/**
* (@inheritdoc)
*/
public function compareSet($key, $oldValue, $newValue, $ttl = Cache::FIVE_MINUTES)
{
if ($this->get($key) === $oldValue) {
return $this->set($key, $newValue);
} else {
return false;
}
}
}

View file

@ -0,0 +1,48 @@
<?php
namespace Friendica\Core\Cache;
use Friendica\Core\Config;
/**
* Class CacheDriverFactory
*
* @package Friendica\Core\Cache
*
* A basic class to generate a CacheDriver
*/
class CacheDriverFactory
{
/**
* This method creates a CacheDriver for the given cache driver name
*
* @param string $driver The name of the cache driver
* @return ICacheDriver The instance of the CacheDriver
* @throws \Exception The exception if something went wrong during the CacheDriver creation
*/
public static function create($driver) {
switch ($driver) {
case 'memcache':
$memcache_host = Config::get('system', 'memcache_host', '127.0.0.1');
$memcache_port = Config::get('system', 'memcache_port', 11211);
return new MemcacheCacheDriver($memcache_host, $memcache_port);
break;
case 'memcached':
$memcached_hosts = Config::get('system', 'memcached_hosts', [['127.0.0.1', 11211]]);
return new MemcachedCacheDriver($memcached_hosts);
break;
case 'redis':
$redis_host = Config::get('system', 'redis_host', '127.0.0.1');
$redis_port = Config::get('system', 'redis_port', 6379);
return new RedisCacheDriver($redis_host, $redis_port);
break;
default:
return new DatabaseCacheDriver();
}
}
}

View file

@ -33,11 +33,11 @@ class DatabaseCacheDriver implements ICacheDriver
return null;
}
public function set($key, $value, $duration = Cache::MONTH)
public function set($key, $value, $ttl = Cache::FIVE_MINUTES)
{
$fields = [
'v' => serialize($value),
'expires' => DateTimeFormat::utc('now + ' . $duration . ' seconds'),
'expires' => DateTimeFormat::utc('now + ' . $ttl . ' seconds'),
'updated' => DateTimeFormat::utcNow()
];

View file

@ -25,17 +25,16 @@ interface ICacheDriver
*
* @param string $key The cache key
* @param mixed $value The value to store
* @param integer $duration The cache lifespan, must be one of the Cache constants
* @param integer $ttl The cache lifespan, must be one of the Cache constants
*
* @return bool
*/
public function set($key, $value, $duration = Cache::MONTH);
public function set($key, $value, $ttl = Cache::FIVE_MINUTES);
/**
* Delete a key from the cache
*
* @param string $key
* @param string $key The cache key
*
* @return bool
*/

View file

@ -0,0 +1,45 @@
<?php
namespace Friendica\Core\Cache;
use Friendica\Core\Cache;
/**
* This interface defines methods for Memory-Caches only
*
* Interface IMemoryCacheDriver
*
* @package Friendica\Core\Cache
*/
interface IMemoryCacheDriver extends ICacheDriver
{
/**
* Sets a value if it's not already stored
*
* @param string $key The cache key
* @param mixed $value The old value we know from the cache
* @param int $ttl The cache lifespan, must be one of the Cache constants
* @return bool
*/
public function add($key, $value, $ttl = Cache::FIVE_MINUTES);
/**
* Compares if the old value is set and sets the new value
*
* @param string $key The cache key
* @param mixed $oldValue The old value we know from the cache
* @param mixed $newValue The new value we want to set
* @param int $ttl The cache lifespan, must be one of the Cache constants
*
* @return bool
*/
public function compareSet($key, $oldValue, $newValue, $ttl = Cache::FIVE_MINUTES);
/**
* Compares if the old value is set and removes it
*
* @param string $key The cache key
* @param mixed $value The old value we know and want to delete
* @return bool
*/
public function compareDelete($key, $value);
}

View file

@ -10,10 +10,13 @@ use Friendica\Core\Cache;
*
* @author Hypolite Petovan <mrpetovan@gmail.com>
*/
class MemcacheCacheDriver extends BaseObject implements ICacheDriver
class MemcacheCacheDriver extends BaseObject implements IMemoryCacheDriver
{
use TraitCompareSet;
use TraitCompareDelete;
/**
* @var Memcache
* @var \Memcache
*/
private $memcache;
@ -30,6 +33,9 @@ class MemcacheCacheDriver extends BaseObject implements ICacheDriver
}
}
/**
* (@inheritdoc)
*/
public function get($key)
{
$return = null;
@ -54,24 +60,49 @@ class MemcacheCacheDriver extends BaseObject implements ICacheDriver
return $return;
}
public function set($key, $value, $duration = Cache::MONTH)
/**
* (@inheritdoc)
*/
public function set($key, $value, $ttl = Cache::FIVE_MINUTES)
{
// We store with the hostname as key to avoid problems with other applications
return $this->memcache->set(
self::getApp()->get_hostname() . ":" . $key,
serialize($value),
MEMCACHE_COMPRESSED,
time() + $duration
);
if ($ttl > 0) {
return $this->memcache->set(
self::getApp()->get_hostname() . ":" . $key,
serialize($value),
MEMCACHE_COMPRESSED,
time() + $ttl
);
} else {
return $this->memcache->set(
self::getApp()->get_hostname() . ":" . $key,
serialize($value),
MEMCACHE_COMPRESSED
);
}
}
/**
* (@inheritdoc)
*/
public function delete($key)
{
return $this->memcache->delete($key);
}
/**
* (@inheritdoc)
*/
public function clear()
{
return true;
return $this->memcache->flush();
}
/**
* (@inheritdoc)
*/
public function add($key, $value, $ttl = Cache::FIVE_MINUTES)
{
return $this->memcache->add(self::getApp()->get_hostname() . ":" . $key, $value, $ttl);
}
}

View file

@ -10,10 +10,13 @@ use Friendica\Core\Cache;
*
* @author Hypolite Petovan <mrpetovan@gmail.com>
*/
class MemcachedCacheDriver extends BaseObject implements ICacheDriver
class MemcachedCacheDriver extends BaseObject implements IMemoryCacheDriver
{
use TraitCompareSet;
use TraitCompareDelete;
/**
* @var Memcached
* @var \Memcached
*/
private $memcached;
@ -46,14 +49,22 @@ class MemcachedCacheDriver extends BaseObject implements ICacheDriver
return $return;
}
public function set($key, $value, $duration = Cache::MONTH)
public function set($key, $value, $ttl = Cache::FIVE_MINUTES)
{
// We store with the hostname as key to avoid problems with other applications
return $this->memcached->set(
self::getApp()->get_hostname() . ':' . $key,
$value,
time() + $duration
);
if ($ttl > 0) {
return $this->memcached->set(
self::getApp()->get_hostname() . ':' . $key,
$value,
time() + $ttl
);
} else {
return $this->memcached->set(
self::getApp()->get_hostname() . ':' . $key,
$value
);
}
}
public function delete($key)
@ -67,4 +78,17 @@ class MemcachedCacheDriver extends BaseObject implements ICacheDriver
{
return true;
}
/**
* @brief Sets a value if it's not already stored
*
* @param string $key The cache key
* @param mixed $value The old value we know from the cache
* @param int $ttl The cache lifespan, must be one of the Cache constants
* @return bool
*/
public function add($key, $value, $ttl = Cache::FIVE_MINUTES)
{
return $this->memcached->add(self::getApp()->get_hostname() . ":" . $key, $value, $ttl);
}
}

View file

@ -11,10 +11,10 @@ use Friendica\Core\Cache;
* @author Hypolite Petovan <mrpetovan@gmail.com>
* @author Roland Haeder <roland@mxchange.org>
*/
class RedisCacheDriver extends BaseObject implements ICacheDriver
class RedisCacheDriver extends BaseObject implements IMemoryCacheDriver
{
/**
* @var Redis
* @var \Redis
*/
private $redis;
@ -55,14 +55,21 @@ class RedisCacheDriver extends BaseObject implements ICacheDriver
return $return;
}
public function set($key, $value, $duration = Cache::MONTH)
public function set($key, $value, $ttl = Cache::FIVE_MINUTES)
{
// We store with the hostname as key to avoid problems with other applications
return $this->redis->set(
self::getApp()->get_hostname() . ":" . $key,
serialize($value),
time() + $duration
);
if ($ttl > 0) {
return $this->redis->setex(
self::getApp()->get_hostname() . ":" . $key,
time() + $ttl,
serialize($value)
);
} else {
return $this->redis->set(
self::getApp()->get_hostname() . ":" . $key,
serialize($value)
);
}
}
public function delete($key)
@ -74,4 +81,60 @@ class RedisCacheDriver extends BaseObject implements ICacheDriver
{
return true;
}
/**
* (@inheritdoc)
*/
public function add($key, $value, $ttl = Cache::FIVE_MINUTES)
{
if (!is_int($value)) {
$value = serialize($value);
}
return $this->redis->setnx(self::getApp()->get_hostname() . ":" . $key, $value);
}
/**
* (@inheritdoc)
*/
public function compareSet($key, $oldValue, $newValue, $ttl = Cache::FIVE_MINUTES)
{
if (!is_int($newValue)) {
$newValue = serialize($newValue);
}
$this->redis->watch(self::getApp()->get_hostname() . ":" . $key);
// If the old value isn't what we expected, somebody else changed the key meanwhile
if ($this->get($key) === $oldValue) {
if ($ttl > 0) {
$result = $this->redis->multi()
->setex(self::getApp()->get_hostname() . ":" . $ttl, $key, $newValue)
->exec();
} else {
$result = $this->redis->multi()
->set(self::getApp()->get_hostname() . ":" . $key, $newValue)
->exec();
}
return $result !== false;
}
$this->redis->unwatch();
return false;
}
/**
* (@inheritdoc)
*/
public function compareDelete($key, $value)
{
$this->redis->watch(self::getApp()->get_hostname() . ":" . $key);
// If the old value isn't what we expected, somebody else changed the key meanwhile
if ($this->get($key) === $value) {
$result = $this->redis->multi()
->del(self::getApp()->get_hostname() . ":" . $key)
->exec();
return $result !== false;
}
$this->redis->unwatch();
return false;
}
}

View file

@ -0,0 +1,45 @@
<?php
namespace Friendica\Core\Cache;
use Friendica\Core\Cache;
/**
* Trait TraitCompareSetDelete
*
* This Trait is to compensate non native "exclusive" sets/deletes in caches
*
* @package Friendica\Core\Cache
*/
trait TraitCompareDelete
{
abstract public function get($key);
abstract public function set($key, $value, $ttl = Cache::FIVE_MINUTES);
abstract public function delete($key);
abstract public function add($key, $value, $ttl = Cache::FIVE_MINUTES);
/**
* NonNative - Compares if the old value is set and removes it
*
* @param string $key The cache key
* @param mixed $value The old value we know and want to delete
* @return bool
*/
public function compareDelete($key, $value) {
if ($this->add($key . "_lock", true)) {
if ($this->get($key) === $value) {
$this->delete($key);
$this->delete($key . "_lock");
return true;
} else {
$this->delete($key . "_lock");
return false;
}
} else {
return false;
}
}
}

View file

@ -0,0 +1,48 @@
<?php
namespace Friendica\Core\Cache;
use Friendica\Core\Cache;
/**
* Trait TraitCompareSetDelete
*
* This Trait is to compensate non native "exclusive" sets/deletes in caches
*
* @package Friendica\Core\Cache
*/
trait TraitCompareSet
{
abstract public function get($key);
abstract public function set($key, $value, $ttl = Cache::FIVE_MINUTES);
abstract public function delete($key);
abstract public function add($key, $value, $ttl = Cache::FIVE_MINUTES);
/**
* NonNative - Compares if the old value is set and sets the new value
*
* @param string $key The cache key
* @param mixed $oldValue The old value we know from the cache
* @param mixed $newValue The new value we want to set
* @param int $ttl The cache lifespan, must be one of the Cache constants
*
* @return bool
*/
public function compareSet($key, $oldValue, $newValue, $ttl = Cache::FIVE_MINUTES) {
if ($this->add($key . "_lock", true)) {
if ($this->get($key) === $oldValue) {
$this->set($key, $newValue, $ttl);
$this->delete($key . "_lock");
return true;
} else {
$this->delete($key . "_lock");
return false;
}
} else {
return false;
}
}
}

140
src/Core/Lock.php Normal file
View file

@ -0,0 +1,140 @@
<?php
namespace Friendica\Core;
/**
* @file src/Core/Lock.php
* @brief Functions for preventing parallel execution of functions
*/
use Friendica\Core\Cache\CacheDriverFactory;
use Friendica\Core\Cache\IMemoryCacheDriver;
/**
* @brief This class contain Functions for preventing parallel execution of functions
*/
class Lock
{
/**
* @var Lock\ILockDriver;
*/
static $driver = null;
public static function init()
{
$lock_driver = Config::get('system', 'lock_driver', 'default');
try {
switch ($lock_driver) {
case 'memcache':
case 'memcached':
case 'redis':
$cache_driver = CacheDriverFactory::create($lock_driver);
if ($cache_driver instanceof IMemoryCacheDriver) {
self::$driver = new Lock\CacheLockDriver($cache_driver);
}
break;
case 'database':
self::$driver = new Lock\DatabaseLockDriver();
break;
case 'semaphore':
self::$driver = new Lock\SemaphoreLockDriver();
break;
default:
self::useAutoDriver();
}
} catch (\Exception $exception) {
logger ('Driver \'' . $lock_driver . '\' failed - Fallback to \'useAutoDriver()\'');
self::useAutoDriver();
}
}
/**
* @brief This method tries to find the best - local - locking method for Friendica
*
* The following sequence will be tried:
* 1. Semaphore Locking
* 2. Cache Locking
* 3. Database Locking
*
*/
private static function useAutoDriver() {
// 1. Try to use Semaphores for - local - locking
if (function_exists('sem_get')) {
try {
self::$driver = new Lock\SemaphoreLockDriver();
return;
} catch (\Exception $exception) {
logger ('Using Semaphore driver for locking failed: ' . $exception->getMessage());
}
}
// 2. Try to use Cache Locking (don't use the DB-Cache Locking because it works different!)
$cache_driver = Config::get('system', 'cache_driver', 'database');
if ($cache_driver != 'database') {
try {
$lock_driver = CacheDriverFactory::create($cache_driver);
if ($lock_driver instanceof IMemoryCacheDriver) {
self::$driver = new Lock\CacheLockDriver($lock_driver);
}
return;
} catch (\Exception $exception) {
logger('Using Cache driver for locking failed: ' . $exception->getMessage());
}
}
// 3. Use Database Locking as a Fallback
self::$driver = new Lock\DatabaseLockDriver();
}
/**
* Returns the current cache driver
*
* @return Lock\ILockDriver;
*/
private static function getDriver()
{
if (self::$driver === null) {
self::init();
}
return self::$driver;
}
/**
* @brief Acquires a lock for a given name
*
* @param string $key Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public static function acquire($key, $timeout = 120)
{
return self::getDriver()->acquireLock($key, $timeout);
}
/**
* @brief Releases a lock if it was set by us
*
* @param string $key Name of the lock
* @return void
*/
public static function release($key)
{
self::getDriver()->releaseLock($key);
}
/**
* @brief Releases all lock that were set by us
* @return void
*/
public static function releaseAll()
{
self::getDriver()->releaseAll();
}
}

View file

@ -0,0 +1,58 @@
<?php
namespace Friendica\Core\Lock;
use Friendica\BaseObject;
/**
* Class AbstractLockDriver
*
* @package Friendica\Core\Lock
*
* Basic class for Locking with common functions (local acquired locks, releaseAll, ..)
*/
abstract class AbstractLockDriver extends BaseObject implements ILockDriver
{
/**
* @var array The local acquired locks
*/
protected $acquiredLocks = [];
/**
* Check if we've locally acquired a lock
*
* @param string key The Name of the lock
* @return bool Returns true if the lock is set
*/
protected function hasAcquiredLock($key) {
return isset($this->acquireLock[$key]) && $this->acquiredLocks[$key] === true;
}
/**
* Mark a locally acquired lock
*
* @param string $key The Name of the lock
*/
protected function markAcquire($key) {
$this->acquiredLocks[$key] = true;
}
/**
* Mark a release of a locally acquired lock
*
* @param string $key The Name of the lock
*/
protected function markRelease($key) {
unset($this->acquiredLocks[$key]);
}
/**
* Releases all lock that were set by us
*
* @return void
*/
public function releaseAll() {
foreach ($this->acquiredLocks as $acquiredLock => $hasLock) {
$this->releaseLock($acquiredLock);
}
}
}

View file

@ -0,0 +1,88 @@
<?php
namespace Friendica\Core\Lock;
use Friendica\Core\Cache\IMemoryCacheDriver;
class CacheLockDriver extends AbstractLockDriver
{
/**
* @var \Friendica\Core\Cache\ICacheDriver;
*/
private $cache;
/**
* CacheLockDriver constructor.
*
* @param IMemoryCacheDriver $cache The CacheDriver for this type of lock
*/
public function __construct(IMemoryCacheDriver $cache)
{
$this->cache = $cache;
}
/**
* (@inheritdoc)
*/
public function acquireLock($key, $timeout = 120)
{
$got_lock = false;
$start = time();
$cachekey = self::getCacheKey($key);
do {
$lock = $this->cache->get($cachekey);
// When we do want to lock something that was already locked by us.
if ((int)$lock == getmypid()) {
$got_lock = true;
}
// When we do want to lock something new
if (is_null($lock)) {
// At first initialize it with "0"
$this->cache->add($cachekey, 0);
// Now the value has to be "0" because otherwise the key was used by another process meanwhile
if ($this->cache->compareSet($cachekey, 0, getmypid(), 300)) {
$got_lock = true;
$this->markAcquire($key);
}
}
if (!$got_lock && ($timeout > 0)) {
usleep(rand(10000, 200000));
}
} while (!$got_lock && ((time() - $start) < $timeout));
return $got_lock;
}
/**
* (@inheritdoc)
*/
public function releaseLock($key)
{
$cachekey = self::getCacheKey($key);
$this->cache->compareDelete($cachekey, getmypid());
$this->markRelease($key);
}
/**
* (@inheritdoc)
*/
public function isLocked($key)
{
$cachekey = self::getCacheKey($key);
$lock = $this->cache->get($cachekey);
return isset($lock) && ($lock !== false);
}
/**
* @param string $key The original key
* @return string The cache key used for the cache
*/
private static function getCacheKey($key) {
return self::getApp()->get_hostname() . ";lock:" . $key;
}
}

View file

@ -0,0 +1,90 @@
<?php
namespace Friendica\Core\Lock;
use dba;
use Friendica\Database\DBM;
use Friendica\Util\DateTimeFormat;
/**
* Locking driver that stores the locks in the database
*/
class DatabaseLockDriver extends AbstractLockDriver
{
/**
* (@inheritdoc)
*/
public function acquireLock($key, $timeout = 120)
{
$got_lock = false;
$start = time();
do {
dba::lock('locks');
$lock = dba::selectFirst('locks', ['locked', 'pid'], ['`name` = ? AND `expires` >= ?', $key, DateTimeFormat::utcNow()]);
if (DBM::is_result($lock)) {
if ($lock['locked']) {
// We want to lock something that was already locked by us? So we got the lock.
if ($lock['pid'] == getmypid()) {
$got_lock = true;
$this->markAcquire($key);
}
}
if (!$lock['locked']) {
dba::update('locks', ['locked' => true, 'pid' => getmypid(), 'expires' => DateTimeFormat::utc('now + 300seconds')], ['name' => $key]);
$got_lock = true;
$this->markAcquire($key);
}
} else {
dba::insert('locks', ['name' => $key, 'locked' => true, 'pid' => getmypid(), 'expires' => DateTimeFormat::utc('now + 300seconds')]);
$got_lock = true;
$this->markAcquire($key);
}
dba::unlock();
if (!$got_lock && ($timeout > 0)) {
usleep(rand(100000, 2000000));
}
} while (!$got_lock && ((time() - $start) < $timeout));
return $got_lock;
}
/**
* (@inheritdoc)
*/
public function releaseLock($key)
{
dba::delete('locks', ['name' => $key, 'pid' => getmypid()]);
$this->markRelease($key);
return;
}
/**
* (@inheritdoc)
*/
public function releaseAll()
{
dba::delete('locks', ['pid' => getmypid()]);
$this->acquiredLocks = [];
}
/**
* (@inheritdoc)
*/
public function isLocked($key)
{
$lock = dba::selectFirst('locks', ['locked'], ['`name` = ? AND `expires` >= ?', $key, DateTimeFormat::utcNow()]);
if (DBM::is_result($lock)) {
return $lock['locked'] !== false;
} else {
return false;
}
}
}

View file

@ -0,0 +1,46 @@
<?php
namespace Friendica\Core\Lock;
/**
* Lock Driver Interface
*
* @author Philipp Holzer <admin@philipp.info>
*/
interface ILockDriver
{
/**
* Checks, if a key is currently locked to a or my process
*
* @param string $key The name of the lock
* @return bool
*/
public function isLocked($key);
/**
*
* Acquires a lock for a given name
*
* @param string $key The Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public function acquireLock($key, $timeout = 120);
/**
* Releases a lock if it was set by us
*
* @param string $key The Name of the lock
*
* @return void
*/
public function releaseLock($key);
/**
* Releases all lock that were set by us
*
* @return void
*/
public function releaseAll();
}

View file

@ -0,0 +1,71 @@
<?php
namespace Friendica\Core\Lock;
class SemaphoreLockDriver extends AbstractLockDriver
{
private static $semaphore = [];
public function __construct()
{
if (!function_exists('sem_get')) {
throw new \Exception('Semaphore lock not supported');
}
}
/**
* (@inheritdoc)
*/
private static function semaphoreKey($key)
{
$temp = get_temppath();
$file = $temp . '/' . $key . '.sem';
if (!file_exists($file)) {
file_put_contents($file, $key);
}
return ftok($file, 'f');
}
/**
*
* (@inheritdoc)
*/
public function acquireLock($key, $timeout = 120)
{
self::$semaphore[$key] = sem_get(self::semaphoreKey($key));
if (self::$semaphore[$key]) {
if (sem_acquire(self::$semaphore[$key], ($timeout == 0))) {
$this->markAcquire($key);
return true;
}
}
return false;
}
/**
* (@inheritdoc)
*/
public function releaseLock($key)
{
if (empty(self::$semaphore[$key])) {
return false;
} else {
$success = @sem_release(self::$semaphore[$key]);
unset(self::$semaphore[$key]);
$this->markRelease($key);
return $success;
}
}
/**
* (@inheritdoc)
*/
public function isLocked($key)
{
return isset(self::$semaphore[$key]);
}
}

View file

@ -4,15 +4,11 @@
*/
namespace Friendica\Core;
use Friendica\Core\Addon;
use Friendica\Core\Config;
use Friendica\Core\System;
use dba;
use Friendica\Database\DBM;
use Friendica\Model\Process;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Lock;
use Friendica\Util\Network;
use dba;
require_once 'include/dba.php';
@ -108,16 +104,16 @@ class Worker
}
// If possible we will fetch new jobs for this worker
if (!$refetched && Lock::set('worker_process', 0)) {
if (!$refetched && Lock::acquire('worker_process', 0)) {
$stamp = (float)microtime(true);
$refetched = self::findWorkerProcesses($passing_slow);
self::$db_duration += (microtime(true) - $stamp);
Lock::remove('worker_process');
Lock::release('worker_process');
}
}
// To avoid the quitting of multiple workers only one worker at a time will execute the check
if (Lock::set('worker', 0)) {
if (Lock::acquire('worker', 0)) {
$stamp = (float)microtime(true);
// Count active workers and compare them with a maximum value that depends on the load
if (self::tooMuchWorkers()) {
@ -130,7 +126,7 @@ class Worker
logger('Memory limit reached, quitting.', LOGGER_DEBUG);
return;
}
Lock::remove('worker');
Lock::release('worker');
self::$db_duration += (microtime(true) - $stamp);
}
@ -883,7 +879,7 @@ class Worker
dba::close($r);
$stamp = (float)microtime(true);
if (!Lock::set('worker_process')) {
if (!Lock::acquire('worker_process')) {
return false;
}
self::$lock_duration = (microtime(true) - $stamp);
@ -892,7 +888,7 @@ class Worker
$found = self::findWorkerProcesses($passing_slow);
self::$db_duration += (microtime(true) - $stamp);
Lock::remove('worker_process');
Lock::release('worker_process');
if ($found) {
$r = dba::select('workerqueue', [], ['pid' => getmypid(), 'done' => false]);
@ -1097,13 +1093,13 @@ class Worker
}
// If there is a lock then we don't have to check for too much worker
if (!Lock::set('worker', 0)) {
if (!Lock::acquire('worker', 0)) {
return true;
}
// If there are already enough workers running, don't fork another one
$quit = self::tooMuchWorkers();
Lock::remove('worker');
Lock::release('worker');
if ($quit) {
return true;

View file

@ -4,10 +4,9 @@
*/
namespace Friendica\Database;
use dba;
use Friendica\Core\Config;
use Friendica\Core\L10n;
use Friendica\Database\DBM;
use dba;
require_once 'boot.php';
require_once 'include/dba.php';
@ -1283,9 +1282,11 @@ class DBStructure
"name" => ["type" => "varchar(128)", "not null" => "1", "default" => "", "comment" => ""],
"locked" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"pid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Process ID"],
],
"expires" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "datetime of cache expiration"],
],
"indexes" => [
"PRIMARY" => ["id"],
"name_expires" => ["name", "expires"]
]
];
$database["mail"] = [

View file

@ -6,26 +6,22 @@
namespace Friendica\Model;
use dba;
use Friendica\BaseObject;
use Friendica\Content\Text;
use Friendica\Core\Addon;
use Friendica\Core\Config;
use Friendica\Core\L10n;
use Friendica\Core\Lock;
use Friendica\Core\PConfig;
use Friendica\Core\System;
use Friendica\Core\Worker;
use Friendica\Database\DBM;
use Friendica\Model\Contact;
use Friendica\Model\Conversation;
use Friendica\Model\Group;
use Friendica\Model\Term;
use Friendica\Object\Image;
use Friendica\Protocol\Diaspora;
use Friendica\Protocol\OStatus;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\XML;
use Friendica\Util\Lock;
use dba;
use Text_LanguageDetect;
require_once 'boot.php';
@ -1595,7 +1591,7 @@ class Item extends BaseObject
}
// To avoid timing problems, we are using locks.
$locked = Lock::set('item_insert_content');
$locked = Lock::acquire('item_insert_content');
if (!$locked) {
logger("Couldn't acquire lock for URI " . $item['uri'] . " - proceeding anyway.");
}
@ -1615,7 +1611,7 @@ class Item extends BaseObject
logger('Could not insert content for URI ' . $item['uri'] . ' - trying asynchronously');
}
if ($locked) {
Lock::remove('item_insert_content');
Lock::release('item_insert_content');
}
}

View file

@ -4,11 +4,15 @@
*/
namespace Friendica\Protocol;
use dba;
use DOMDocument;
use DOMXPath;
use Friendica\Content\Text\BBCode;
use Friendica\Content\Text\HTML;
use Friendica\Core\Cache;
use Friendica\Core\Config;
use Friendica\Core\L10n;
use Friendica\Core\Lock;
use Friendica\Core\System;
use Friendica\Database\DBM;
use Friendica\Model\Contact;
@ -19,12 +23,8 @@ use Friendica\Model\User;
use Friendica\Network\Probe;
use Friendica\Object\Image;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Lock;
use Friendica\Util\Network;
use Friendica\Util\XML;
use dba;
use DOMDocument;
use DOMXPath;
require_once 'include/dba.php';
require_once 'include/items.php';
@ -513,9 +513,9 @@ class OStatus
logger("Item with uri ".$item["uri"]." is from a blocked contact.", LOGGER_DEBUG);
} else {
// We are having duplicated entries. Hopefully this solves it.
if (Lock::set('ostatus_process_item_insert')) {
if (Lock::acquire('ostatus_process_item_insert')) {
$ret = Item::insert($item);
Lock::remove('ostatus_process_item_insert');
Lock::release('ostatus_process_item_insert');
logger("Item with uri ".$item["uri"]." for user ".$importer["uid"].' stored. Return value: '.$ret);
} else {
$ret = Item::insert($item);

View file

@ -1,211 +0,0 @@
<?php
/**
* @file src/Util/Lock.php
*/
namespace Friendica\Util;
/**
* @file src/Util/Lock.php
* @brief Functions for preventing parallel execution of functions
*/
use Friendica\Core\Config;
use Friendica\Database\DBM;
use Memcache;
use dba;
require_once 'include/dba.php';
/**
* @brief This class contain Functions for preventing parallel execution of functions
*/
class Lock
{
private static $semaphore = [];
/**
* @brief Check for memcache and open a connection if configured
*
* @return object|boolean The memcache object - or "false" if not successful
*/
private static function connectMemcache()
{
if (!function_exists('memcache_connect')) {
return false;
}
if (!Config::get('system', 'memcache')) {
return false;
}
$memcache_host = Config::get('system', 'memcache_host', '127.0.0.1');
$memcache_port = Config::get('system', 'memcache_port', 11211);
$memcache = new Memcache;
if (!$memcache->connect($memcache_host, $memcache_port)) {
return false;
}
return $memcache;
}
/**
* @brief Creates a semaphore key
*
* @param string $fn_name Name of the lock
*
* @return ressource the semaphore key
*/
private static function semaphoreKey($fn_name)
{
$temp = get_temppath();
$file = $temp.'/'.$fn_name.'.sem';
if (!file_exists($file)) {
file_put_contents($file, $fn_name);
}
return ftok($file, 'f');
}
/**
* @brief Sets a lock for a given name
*
* @param string $fn_name Name of the lock
* @param integer $timeout Seconds until we give up
*
* @return boolean Was the lock successful?
*/
public static function set($fn_name, $timeout = 120)
{
$got_lock = false;
$start = time();
// The second parameter for "sem_acquire" doesn't exist before 5.6.1
if (function_exists('sem_get') && version_compare(PHP_VERSION, '5.6.1', '>=')) {
self::$semaphore[$fn_name] = sem_get(self::semaphoreKey($fn_name));
if (self::$semaphore[$fn_name]) {
return sem_acquire(self::$semaphore[$fn_name], ($timeout == 0));
}
}
$memcache = self::connectMemcache();
if (is_object($memcache)) {
$cachekey = get_app()->get_hostname().";lock:".$fn_name;
do {
// We only lock to be sure that nothing happens at exactly the same time
dba::lock('locks');
$lock = $memcache->get($cachekey);
if (!is_bool($lock)) {
$pid = (int)$lock;
// When the process id isn't used anymore, we can safely claim the lock for us.
// Or we do want to lock something that was already locked by us.
if (!posix_kill($pid, 0) || ($pid == getmypid())) {
$lock = false;
}
}
if (is_bool($lock)) {
$memcache->set($cachekey, getmypid(), MEMCACHE_COMPRESSED, 300);
$got_lock = true;
}
dba::unlock();
if (!$got_lock && ($timeout > 0)) {
usleep(rand(10000, 200000));
}
} while (!$got_lock && ((time() - $start) < $timeout));
return $got_lock;
}
do {
dba::lock('locks');
$lock = dba::selectFirst('locks', ['locked', 'pid'], ['name' => $fn_name]);
if (DBM::is_result($lock)) {
if ($lock['locked']) {
// When the process id isn't used anymore, we can safely claim the lock for us.
if (!posix_kill($lock['pid'], 0)) {
$lock['locked'] = false;
}
// We want to lock something that was already locked by us? So we got the lock.
if ($lock['pid'] == getmypid()) {
$got_lock = true;
}
}
if (!$lock['locked']) {
dba::update('locks', ['locked' => true, 'pid' => getmypid()], ['name' => $fn_name]);
$got_lock = true;
}
} elseif (!DBM::is_result($lock)) {
dba::insert('locks', ['name' => $fn_name, 'locked' => true, 'pid' => getmypid()]);
$got_lock = true;
}
dba::unlock();
if (!$got_lock && ($timeout > 0)) {
usleep(rand(100000, 2000000));
}
} while (!$got_lock && ((time() - $start) < $timeout));
return $got_lock;
}
/**
* @brief Removes a lock if it was set by us
*
* @param string $fn_name Name of the lock
* @return mixed
*/
public static function remove($fn_name)
{
if (function_exists('sem_get') && version_compare(PHP_VERSION, '5.6.1', '>=')) {
if (empty(self::$semaphore[$fn_name])) {
return false;
} else {
$success = @sem_release(self::$semaphore[$fn_name]);
unset(self::$semaphore[$fn_name]);
return $success;
}
}
$memcache = self::connectMemcache();
if (is_object($memcache)) {
$cachekey = get_app()->get_hostname().";lock:".$fn_name;
$lock = $memcache->get($cachekey);
if (!is_bool($lock)) {
if ((int)$lock == getmypid()) {
$memcache->delete($cachekey);
}
}
return;
}
dba::update('locks', ['locked' => false, 'pid' => 0], ['name' => $fn_name, 'pid' => getmypid()]);
return;
}
/**
* @brief Removes all lock that were set by us
* @return void
*/
public static function removeAll()
{
$memcache = self::connectMemcache();
if (is_object($memcache)) {
// We cannot delete all cache entries, but this doesn't matter with memcache
return;
}
dba::update('locks', ['locked' => false, 'pid' => 0], ['pid' => getmypid()]);
return;
}
}

View file

@ -14,7 +14,7 @@ user:
uid: 42
username: Test user
nickname: selfcontact
verified: true
verified: 1
password: $2y$10$DLRNTRmJgKe1cSrFJ5Jb0edCqvXlA9sh/RHdSnfxjbR.04yZRm4Qm
theme: frio
@ -24,12 +24,12 @@ contact:
uid: 42
name: Self contact
nick: selfcontact
self: true
self: 1
nurl: http://localhost/profile/selfcontact
url: http://localhost/profile/selfcontact
about: User used in tests
pending: false
blocked: false
pending: 0
blocked: 0
rel: 1
network: dfrn
-
@ -39,11 +39,11 @@ contact:
# the fallback to api_get_nick() in api_get_user()
name: othercontact
nick: othercontact
self: false
self: 0
nurl: http://localhost/profile/othercontact
url: http://localhost/profile/othercontact
pending: false
blocked: false
pending: 0
blocked: 0
rel: 0
network: dfrn
-
@ -51,150 +51,150 @@ contact:
uid: 0
name: Friend contact
nick: friendcontact
self: false
self: 0
nurl: http://localhost/profile/friendcontact
url: http://localhost/profile/friendcontact
pending: false
blocked: false
pending: 0
blocked: 0
rel: 2
network: dfrn
item:
-
id: 1
visible: true
visible: 1
contact-id: 42
author-id: 42
owner-id: 45
uid: 42
verb: http://activitystrea.ms/schema/1.0/post
unseen: true
unseen: 1
body: Parent status
parent: 1
author-link: http://localhost/profile/selfcontact
wall: true
starred: true
origin: true
wall: 1
starred: 1
origin: 1
allow_cid: ''
allow_gid: ''
deny_cid: ''
deny_gid: ''
-
id: 2
visible: true
visible: 1
contact-id: 42
author-id: 42
owner-id: 45
uid: 42
verb: http://activitystrea.ms/schema/1.0/post
unseen: false
unseen: 0
body: Reply
parent: 1
author-link: http://localhost/profile/selfcontact
wall: true
starred: false
origin: true
wall: 1
starred: 0
origin: 1
-
id: 3
visible: true
visible: 1
contact-id: 43
author-id: 43
owner-id: 42
uid: 42
verb: http://activitystrea.ms/schema/1.0/post
unseen: false
unseen: 0
body: Other user status
parent: 3
author-link: http://localhost/profile/othercontact
wall: true
starred: false
origin: true
wall: 1
starred: 0
origin: 1
-
id: 4
visible: true
visible: 1
contact-id: 44
author-id: 44
owner-id: 42
uid: 42
verb: http://activitystrea.ms/schema/1.0/post
unseen: false
unseen: 0
body: Friend user reply
parent: 1
author-link: http://localhost/profile/othercontact
wall: true
starred: false
origin: true
wall: 1
starred: 0
origin: 1
-
id: 5
visible: true
visible: 1
contact-id: 42
author-id: 42
owner-id: 42
uid: 42
verb: http://activitystrea.ms/schema/1.0/post
unseen: false
unseen: 0
body: '[share]Shared status[/share]'
parent: 1
author-link: http://localhost/profile/othercontact
wall: true
starred: false
origin: true
wall: 1
starred: 0
origin: 1
allow_cid: ''
allow_gid: ''
deny_cid: ''
deny_gid: ''
-
id: 6
visible: true
visible: 1
contact-id: 44
author-id: 44
owner-id: 42
uid: 42
verb: http://activitystrea.ms/schema/1.0/post
unseen: false
unseen: 0
body: Friend user status
parent: 6
author-link: http://localhost/profile/othercontact
wall: true
starred: false
origin: true
wall: 1
starred: 0
origin: 1
thread:
-
iid: 1
visible: true
visible: 1
contact-id: 42
author-id: 42
owner-id: 42
uid: 42
wall: true
wall: 1
-
iid: 3
visible: true
visible: 1
contact-id: 43
author-id: 43
owner-id: 43
uid: 0
wall: true
wall: 1
-
iid: 6
visible: true
visible: 1
contact-id: 44
author-id: 44
owner-id: 44
uid: 0
wall: true
wall: 1
group:
-
id: 1
uid: 42
visible: true
visible: 1
name: Visible list
-
id: 2
uid: 42
visible: false
visible: 0
name: Private list
search:

View file

@ -0,0 +1,27 @@
<?php
namespace Friendica\Test\src\Core\Lock;
use Friendica\Core\Cache\ArrayCache;
use Friendica\Core\Lock\CacheLockDriver;
class CacheLockDriverTest extends LockTest
{
/**
* @var \Friendica\Core\Cache\IMemoryCacheDriver
*/
private $cache;
protected function getInstance()
{
$this->cache = new ArrayCache();
return new CacheLockDriver($this->cache);
}
public function tearDown()
{
$this->cache->clear();
parent::tearDown();
}
}

View file

@ -0,0 +1,67 @@
<?php
namespace Friendica\Test\src\Core\Lock;
use dba;
use Friendica\Core\Lock\DatabaseLockDriver;
use Friendica\Database\DBStructure;
use PHPUnit\DbUnit\DataSet\YamlDataSet;
use PHPUnit\DbUnit\TestCaseTrait;
use PHPUnit_Extensions_Database_DB_IDatabaseConnection;
class DatabaseLockDriverTest extends LockTest
{
use TestCaseTrait;
/**
* Get database connection.
*
* This function is executed before each test in order to get a database connection that can be used by tests.
* If no prior connection is available, it tries to create one using the USER, PASS and DB environment variables.
*
* If it could not connect to the database, the test is skipped.
*
* @return PHPUnit_Extensions_Database_DB_IDatabaseConnection
* @see https://phpunit.de/manual/5.7/en/database.html
*/
protected function getConnection()
{
if (!dba::$connected) {
dba::connect('localhost', getenv('USER'), getenv('PASS'), getenv('DB'));
if (dba::$connected) {
$app = get_app();
// We need to do this in order to disable logging
$app->module = 'install';
// Create database structure
DBStructure::update(false, true, true);
} else {
$this->markTestSkipped('Could not connect to the database.');
}
}
return $this->createDefaultDBConnection(dba::get_db(), getenv('DB'));
}
/**
* Get dataset to populate the database with.
* @return YamlDataSet
* @see https://phpunit.de/manual/5.7/en/database.html
*/
protected function getDataSet()
{
return new YamlDataSet(__DIR__ . '/../../../datasets/api.yml');
}
protected function getInstance()
{
return new DatabaseLockDriver();
}
public function tearDown()
{
dba::delete('locks', [ 'id > 0']);
parent::tearDown();
}
}

View file

@ -0,0 +1,80 @@
<?php
namespace Friendica\Test\src\Core\Lock;
use Friendica\App;
use Friendica\Core\Config;
use PHPUnit\Framework\TestCase;
abstract class LockTest extends TestCase
{
/**
* @var \Friendica\Core\Lock\ILockDriver
*/
protected $instance;
abstract protected function getInstance();
protected function setUp()
{
global $a;
parent::setUp();
$this->instance = $this->getInstance();
// Reusable App object
$this->app = new App(__DIR__.'/../');
$a = $this->app;
// Default config
Config::set('config', 'hostname', 'localhost');
Config::set('system', 'throttle_limit_day', 100);
Config::set('system', 'throttle_limit_week', 100);
Config::set('system', 'throttle_limit_month', 100);
Config::set('system', 'theme', 'system_theme');
}
public function testLock() {
$this->instance->acquireLock('foo', 1);
$this->assertTrue($this->instance->isLocked('foo'));
$this->assertFalse($this->instance->isLocked('bar'));
}
public function testDoubleLock() {
$this->instance->acquireLock('foo', 1);
$this->assertTrue($this->instance->isLocked('foo'));
// We already locked it
$this->assertTrue($this->instance->acquireLock('foo', 1));
}
public function testReleaseLock() {
$this->instance->acquireLock('foo', 1);
$this->assertTrue($this->instance->isLocked('foo'));
$this->instance->releaseLock('foo');
$this->assertFalse($this->instance->isLocked('foo'));
}
public function testReleaseAll() {
$this->instance->acquireLock('foo', 1);
$this->instance->acquireLock('bar', 1);
$this->instance->acquireLock('nice', 1);
$this->instance->releaseAll();
$this->assertFalse($this->instance->isLocked('foo'));
$this->assertFalse($this->instance->isLocked('bar'));
$this->assertFalse($this->instance->isLocked('nice'));
}
public function testReleaseAfterUnlock() {
$this->instance->acquireLock('foo', 1);
$this->instance->acquireLock('bar', 1);
$this->instance->acquireLock('nice', 1);
$this->instance->releaseLock('foo');
$this->instance->releaseAll();
$this->assertFalse($this->instance->isLocked('bar'));
$this->assertFalse($this->instance->isLocked('#/$%§'));
}
}

View file

@ -0,0 +1,26 @@
<?php
namespace Friendica\Test\src\Core\Lock;
use Friendica\Core\Lock\SemaphoreLockDriver;
class SemaphoreLockDriverTest extends LockTest
{
/**
* @var \Friendica\Core\Lock\SemaphoreLockDriver
*/
private $semaphoreLockDriver;
protected function getInstance()
{
$this->semaphoreLockDriver = new SemaphoreLockDriver();
return $this->semaphoreLockDriver;
}
public function tearDown()
{
$this->semaphoreLockDriver->releaseAll();
parent::tearDown();
}
}