Merge pull request #1162 from nupplaphil/feat/webdav_storage

WebDav Storage backend

webdav_storage/composer.json

@@ -0,0 +1,22 @@
+{
+  "name": "friendica-addons/webdav_storage",
+  "description": "Adds the possibility to use WebDAV as a selectable storage backend",
+  "type": "friendica-addon",
+  "authors": [
+	{
+	  "name": "Philipp Holzer",
+	  "email": "admin@philipp.info",
+	  "homepage": "https://blog.philipp.info",
+	  "role": "Developer"
+	}
+  ],
+  "require": {
+	"php": ">=7.0"
+  },
+  "license": "3-clause BSD license",
+  "config": {
+	"optimize-autoloader": true,
+	"autoloader-suffix": "WebDavStorageAddon",
+	"preferred-install": "dist"
+  }
+}

webdav_storage/lang/C/messages.po

@@ -0,0 +1,58 @@
+# ADDON webdav_storage
+# Copyright (C) 
+# This file is distributed under the same license as the Friendica webdav_storage addon package.
+# 
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2021-10-04 11:59+0200\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: src/WebDavConfig.php:82
+msgid "URL"
+msgstr ""
+
+#: src/WebDavConfig.php:84
+msgid "URL to the Webdav endpoint, where files can be saved"
+msgstr ""
+
+#: src/WebDavConfig.php:89
+msgid "Username"
+msgstr ""
+
+#: src/WebDavConfig.php:91
+msgid "Username to authenticate to the Webdav endpoint"
+msgstr ""
+
+#: src/WebDavConfig.php:95
+msgid "Password"
+msgstr ""
+
+#: src/WebDavConfig.php:97
+msgid "Password to authenticate to the Webdav endpoint"
+msgstr ""
+
+#: src/WebDavConfig.php:101
+msgid "Authentication type"
+msgstr ""
+
+#: src/WebDavConfig.php:103
+msgid "authentication type to the Webdav endpoint"
+msgstr ""
+
+#: src/WebDavConfig.php:127
+msgid "Authentication type is invalid."
+msgstr ""
+
+#: src/WebDavConfig.php:143
+msgid "url is either invalid or not reachable"
+msgstr ""

webdav_storage/src/WebDav.php

@@ -0,0 +1,268 @@
+<?php
+
+namespace Friendica\Addon\webdav_storage\src;
+
+use Exception;
+use Friendica\Model\Storage\IWritableStorage;
+use Friendica\Model\Storage\ReferenceStorageException;
+use Friendica\Model\Storage\StorageException;
+use Friendica\Network\HTTPClientOptions;
+use Friendica\Network\IHTTPClient;
+use Friendica\Util\Strings;
+use Psr\Log\LoggerInterface;
+
+/**
+ * A WebDav Backend Storage class
+ */
+class WebDav implements IWritableStorage
+{
+	const NAME = 'WebDav';
+
+	/** @var string */
+	private $url;
+
+	/** @var IHTTPClient */
+	private $client;
+
+	/** @var LoggerInterface */
+	private $logger;
+
+	/** @var array */
+	private $authOptions;
+
+	/**
+	 * WebDav constructor
+	 *
+	 * @param string          $url         The full URL to the webdav endpoint (including the subdirectories)
+	 * @param array           $authOptions The authentication options for the http calls ( ['username', 'password', 'auth_type'] )
+	 * @param IHTTPClient     $client      The http client for communicating with the WebDav endpoint
+	 * @param LoggerInterface $logger      The standard logging class
+	 */
+	public function __construct(string $url, array $authOptions, IHTTPClient $client, LoggerInterface $logger)
+	{
+		$this->client = $client;
+		$this->logger = $logger;
+
+		$this->authOptions = $authOptions;
+		$this->url         = $url;
+	}
+
+	/**
+	 * Split data ref and return file path
+	 *
+	 * @param string $reference Data reference
+	 *
+	 * @return string[]
+	 */
+	private function pathForRef(string $reference): array
+	{
+		$fold1 = substr($reference, 0, 2);
+		$fold2 = substr($reference, 2, 2);
+		$file  = substr($reference, 4);
+
+		return [$this->encodePath(implode('/', [$fold1, $fold2, $file])), implode('/', [$fold1, $fold2]), $file];
+	}
+
+	/**
+	 * URL encodes the given path but keeps the slashes
+	 *
+	 * @param string $path to encode
+	 *
+	 * @return string encoded path
+	 */
+	protected function encodePath(string $path): string
+	{
+		// slashes need to stay
+		return str_replace('%2F', '/', rawurlencode($path));
+	}
+
+	/**
+	 * Checks if the URL exists
+	 *
+	 * @param string $uri the URL to check
+	 *
+	 * @return bool true in case the file/folder exists
+	 */
+	protected function exists(string $uri): bool
+	{
+		return $this->client->head($uri, [HTTPClientOptions::AUTH => $this->authOptions])->getReturnCode() == 200;
+	}
+
+	/**
+	 * Checks if a folder has items left
+	 *
+	 * @param string $uri the URL to check
+	 *
+	 * @return bool true in case there are items left in the folder
+	 */
+	protected function hasItems(string $uri): bool
+	{
+		$dom               = new \DOMDocument('1.0', 'UTF-8');
+		$dom->formatOutput = true;
+		$root              = $dom->createElementNS('DAV:', 'd:propfind');
+		$prop              = $dom->createElement('d:allprop');
+
+		$dom->appendChild($root)->appendChild($prop);
+
+		$opts = [
+			HTTPClientOptions::AUTH    => $this->authOptions,
+			HTTPClientOptions::HEADERS => ['Depth' => 1, 'Prefer' => 'return-minimal', 'Content-Type' => 'application/xml'],
+			HTTPClientOptions::BODY    => $dom->saveXML(),
+		];
+
+		$response = $this->client->request('propfind', $uri, $opts);
+
+		$responseDoc = new \DOMDocument();
+		$responseDoc->loadXML($response->getBody());
+		$responseDoc->formatOutput = true;
+
+		$xpath = new \DOMXPath($responseDoc);
+		$xpath->registerNamespace('d', 'DAV');
+		$result = $xpath->query('//d:multistatus/d:response');
+
+		// returns at least its own directory, so >1
+		return $result !== false && count($result) > 1;
+	}
+
+	/**
+	 * Creates a DAV-collection (= folder) for the given uri
+	 *
+	 * @param string $uri The uri for creating a DAV-collection
+	 *
+	 * @return bool true in case the creation was successful (not immutable!)
+	 */
+	protected function mkcol(string $uri): bool
+	{
+		return $this->client->request('mkcol', $uri, [HTTPClientOptions::AUTH => $this->authOptions])
+							->getReturnCode() == 200;
+	}
+
+	/**
+	 * Checks if the given path exists and if not creates it
+	 *
+	 * @param string $fullPath the full path (the folder structure after the hostname)
+	 */
+	protected function checkAndCreatePath(string $fullPath): void
+	{
+		$finalUrl = $this->url . '/' . trim($fullPath, '/');
+
+		if ($this->exists($finalUrl)) {
+			return;
+		}
+
+		$pathParts = explode('/', trim($fullPath, '/'));
+		$path      = '';
+
+		foreach ($pathParts as $part) {
+			$path .= '/' . $part;
+			$partUrl = $this->url . $path;
+			if (!$this->exists($partUrl)) {
+				$this->mkcol($partUrl);
+			}
+		}
+	}
+
+	/**
+	 * Checks recursively, if paths are empty and deletes them
+	 *
+	 * @param string $fullPath the full path (the folder structure after the hostname)
+	 *
+	 * @throws StorageException In case a directory cannot get deleted
+	 */
+	protected function checkAndDeletePath(string $fullPath): void
+	{
+		$pathParts = explode('/', trim($fullPath, '/'));
+		$partURL   = '/' . implode('/', $pathParts);
+
+		foreach ($pathParts as $pathPart) {
+			$checkUrl = $this->url . $partURL;
+			if (!empty($partURL) && !$this->hasItems($checkUrl)) {
+				$response = $this->client->request('delete', $checkUrl, [HTTPClientOptions::AUTH => $this->authOptions]);
+
+				if (!$response->isSuccess()) {
+					if ($response->getReturnCode() == "404") {
+						$this->logger->warning('Directory already deleted.', ['uri' => $checkUrl]);
+					} else {
+						throw new StorageException(sprintf('Unpredicted error for %s: %s', $checkUrl, $response->getError()), $response->getReturnCode());
+					}
+				}
+			}
+
+			$partURL = substr($partURL, 0, -strlen('/' . $pathPart));
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public function get(string $reference): string
+	{
+		$file = $this->pathForRef($reference);
+
+		$response = $this->client->request('get', $this->url . '/' . $file[0], [HTTPClientOptions::AUTH => $this->authOptions]);
+
+		if (!$response->isSuccess()) {
+			throw new ReferenceStorageException(sprintf('Invalid reference %s', $reference));
+		}
+
+		return $response->getBody();
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public function put(string $data, string $reference = ""): string
+	{
+		if ($reference === '') {
+			try {
+				$reference = Strings::getRandomHex();
+			} catch (Exception $exception) {
+				throw new StorageException('Webdav storage failed to generate a random hex', $exception->getCode(), $exception);
+			}
+		}
+		$file = $this->pathForRef($reference);
+
+		$this->checkAndCreatePath($file[1]);
+
+		$opts = [
+			HTTPClientOptions::BODY => $data,
+			HTTPClientOptions::AUTH => $this->authOptions,
+		];
+
+		$this->client->request('put', $this->url . '/' . $file[0], $opts);
+
+		return $reference;
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public function delete(string $reference)
+	{
+		$file = $this->pathForRef($reference);
+
+		$response = $this->client->request('delete', $this->url . '/' . $file[0], [HTTPClientOptions::AUTH => $this->authOptions]);
+
+		if (!$response->isSuccess()) {
+			throw new ReferenceStorageException(sprintf('Invalid reference %s', $reference));
+		}
+
+		$this->checkAndDeletePath($file[1]);
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public function __toString()
+	{
+		return self::getName();
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	public static function getName(): string
+	{
+		return self::NAME;
+	}
+}

webdav_storage/src/WebDavConfig.php

@@ -0,0 +1,156 @@
+<?php
+
+namespace Friendica\Addon\webdav_storage\src;
+
+use Friendica\Core\Config\IConfig;
+use Friendica\Core\L10n;
+use Friendica\Model\Storage\IStorageConfiguration;
+use Friendica\Network\HTTPClientOptions;
+use Friendica\Network\IHTTPClient;
+
+/**
+ * The WebDav Backend Storage configuration class
+ */
+class WebDavConfig implements IStorageConfiguration
+{
+	const NAME = 'WebDav';
+
+	/** @var L10n */
+	private $l10n;
+
+	/** @var IConfig */
+	private $config;
+
+	/** @var string */
+	private $url;
+
+	/** @var IHTTPClient */
+	private $client;
+
+	/** @var array */
+	private $authOptions;
+
+	/**
+	 * @return string
+	 */
+	public function getUrl(): string
+	{
+		return $this->url;
+	}
+
+	/**
+	 * @return array
+	 */
+	public function getAuthOptions(): array
+	{
+		return $this->authOptions;
+	}
+
+	public function __construct(L10n $l10n, IConfig $config, IHTTPClient $client)
+	{
+		$this->l10n   = $l10n;
+		$this->config = $config;
+		$this->client = $client;
+
+		$this->authOptions = null;
+
+		if (!empty($this->config->get('webdav', 'username'))) {
+			$this->authOptions = [
+				$this->config->get('webdav', 'username'),
+				(string)$this->config->get('webdav', 'password', ''),
+				$this->config->get('webdav', 'auth_type', 'basic')
+			];
+		}
+
+		$this->url = $this->config->get('webdav', 'url');
+	}
+
+	/**
+	 * @inheritDoc
+	 */
+	public function getOptions(): array
+	{
+		$auths = [
+			''       => 'None',
+			'basic'  => 'Basic',
+			'digest' => 'Digest',
+		];
+
+		return [
+			'url' => [
+				'input',
+				$this->l10n->t('URL'),
+				$this->url,
+				$this->l10n->t('URL to the Webdav endpoint, where files can be saved'),
+				true
+			],
+			'username' => [
+				'input',
+				$this->l10n->t('Username'),
+				$this->config->get('webdav', 'username', ''),
+				$this->l10n->t('Username to authenticate to the Webdav endpoint')
+			],
+			'password' => [
+				'password',
+				$this->l10n->t('Password'),
+				$this->config->get('webdav', 'username', ''),
+				$this->l10n->t('Password to authenticate to the Webdav endpoint')
+			],
+			'auth_type' => [
+				'select',
+				$this->l10n->t('Authentication type'),
+				$this->config->get('webdav', 'auth_type', ''),
+				$this->l10n->t('authentication type to the Webdav endpoint'),
+				$auths,
+			]
+		];
+	}
+
+	/**
+	 * @inheritDoc
+	 */
+	public function saveOptions(array $data): array
+	{
+		$url      = $data['url']      ?? '';
+		$username = $data['username'] ?? '';
+		$password = $data['password'] ?? '';
+
+		$auths = [
+			''       => 'None',
+			'basic'  => 'Basic',
+			'digest' => 'Digest',
+		];
+
+		$authType = $data['auth_type'] ?? '';
+		if (!key_exists($authType, $auths)) {
+			return [
+				'auth_type' => $this->l10n->t('Authentication type is invalid.'),
+			];
+		}
+
+		$options = null;
+
+		if (!empty($username)) {
+			$options = [
+				$username,
+				$password,
+				$authType
+			];
+		}
+
+		if (!$this->client->head($url, [HTTPClientOptions::AUTH => $options])->isSuccess()) {
+			return [
+				'url' => $this->l10n->t('url is either invalid or not reachable'),
+			];
+		}
+
+		$this->config->set('webdav', 'url', $url);
+		$this->config->set('webdav', 'username', $username);
+		$this->config->set('webdav', 'password', $password);
+		$this->config->set('webdav', 'auth_type', $authType);
+
+		$this->url = $url;
+
+		return [];
+	}
+}

webdav_storage/tests/WebDavTest.php.bak

@@ -0,0 +1,121 @@
+<?php
+
+namespace Friendica\Addon\webdav_storage\tests;
+
+use Friendica\Addon\webdav_storage\src\WebDav;
+use Friendica\DI;
+use Friendica\Factory\HTTPClientFactory;
+use Friendica\Test\src\Model\Storage\StorageTest;
+use Friendica\Util\Logger\VoidLogger;
+
+/// @todo remove when constant is moved to a class constant
+/// Necessary for DB_UPDATE_VERSION constant in case of direct calls, where dbstructure isn't included during the calling process
+require_once __DIR__ . '/../../../static/dbstructure.config.php';
+
+class WebDavTest extends StorageTest
+{
+	public function dataMultiStatus()
+	{
+		return [
+			'nextcloud' => [
+				'xml' => <<<EOF
+<?xml version="1.0"?>
+<d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns"
+			   xmlns:nc="http://nextcloud.org/ns">
+	<d:response>
+		<d:href>/remote.php/dav/files/admin/Friendica_test/97/18/</d:href>
+		<d:propstat>
+			<d:prop>
+				<d:getlastmodified>Mon, 30 Aug 2021 12:58:54 GMT</d:getlastmodified>
+				<d:resourcetype>
+					<d:collection/>
+				</d:resourcetype>
+				<d:quota-used-bytes>45017</d:quota-used-bytes>
+				<d:quota-available-bytes>59180834349</d:quota-available-bytes>
+				<d:getetag>&quot;612cd60ec9fd5&quot;</d:getetag>
+			</d:prop>
+			<d:status>HTTP/1.1 200 OK</d:status>
+		</d:propstat>
+	</d:response>
+	<d:response>
+		<d:href>
+			/remote.php/dav/files/admin/Friendica_test/97/18/4d9d36f614dc005756bdfb9abbf1d8d24aa9ae842e5d6b5e7eb1dafbe767
+															  </d:href>
+		<d:propstat>
+			<d:prop>
+				<d:getlastmodified>Mon, 30 Aug 2021 12:58:54 GMT</d:getlastmodified>
+				<d:getcontentlength>45017</d:getcontentlength>
+				<d:resourcetype/>
+				<d:getetag>&quot;4f7a144092532141d0e6b925e50a896e&quot;</d:getetag>
+				<d:getcontenttype>application/octet-stream
+				</d:getcontenttype>
+			</d:prop>
+			<d:status>HTTP/1.1 200 OK</d:status>
+		</d:propstat>
+		<d:propstat>
+			<d:prop>
+				<d:quota-used-bytes/>
+				<d:quota-available-bytes/>
+			</d:prop>
+			<d:status>HTTP/1.1 404 Not Found
+									   </d:status>
+		</d:propstat>
+	</d:response>
+</d:multistatus>
+EOF,
+				'assertionCount' => 2,
+			],
+			'onlyDir' => [
+				'xml' => <<<EOF
+<d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
+  <d:response>
+    <d:href>/remote.php/dav/files/admin/Friendica_test/34/cf/</d:href>
+    <d:propstat>
+      <d:prop>
+        <d:getlastmodified>Sun, 05 Sep 2021 17:56:05 GMT</d:getlastmodified>
+        <d:resourcetype>
+          <d:collection/>
+        </d:resourcetype>
+        <d:quota-used-bytes>0</d:quota-used-bytes>
+        <d:quota-available-bytes>59182800697</d:quota-available-bytes>
+        <d:getetag>"613504b55db4f"</d:getetag>
+      </d:prop>
+      <d:status>HTTP/1.1 200 OK</d:status>
+    </d:propstat>
+  </d:response>
+</d:multistatus>
+EOF,
+				'assertionCount' => 1,
+			],
+		];
+	}
+
+	/**
+	 * @dataProvider dataMultiStatus
+	 */
+	public function testMultistatus(string $xml, int $assertionCount)
+	{
+		$responseDoc = new \DOMDocument();
+		$responseDoc->loadXML($xml);
+
+		$xpath = new \DOMXPath($responseDoc);
+		$xpath->registerNamespace('d', 'DAV');
+
+		self::assertCount($assertionCount, $xpath->query('//d:multistatus/d:response'));
+	}
+
+	/**
+	 * @inheritDoc
+	 */
+	protected function getInstance()
+	{
+		/** @var HTTPClientFactory $factory */
+		$factory = DI::getDice()->create(HTTPClientFactory::class);
+
+		return new WebDav(getenv('WEBDAV_URL'), [
+			getenv('WEBDAV_USERNAME'),
+			getenv('WEBDAV_PASSWORD'),
+			'basic',
+		], $factory->createClient(), new VoidLogger());
+	}
+}

webdav_storage/webdav_storage.php

@@ -0,0 +1,36 @@
+<?php
+/*
+ * Name: WebDAV Storage
+ * Description: Adds the possibility to use WebDAV as a selectable storage backend
+ * Version: 1.0
+ * Author: Philipp Holzer
+ */
+
+use Friendica\Addon\webdav_storage\src\WebDav;
+use Friendica\Addon\webdav_storage\src\WebDavConfig;
+use Friendica\App;
+use Friendica\Core\Hook;
+use Friendica\DI;
+
+function webdav_storage_install($a)
+{
+	Hook::register('storage_instance' , __FILE__, 'webdav_storage_instance');
+	Hook::register('storage_config' , __FILE__, 'webdav_storage_config');
+	DI::storageManager()->register(WebDav::class);
+}
+
+function webdav_storage_uninstall()
+{
+	DI::storageManager()->unregister(WebDav::class);
+}
+
+function webdav_storage_instance(App $a, array &$data)
+{
+	$config          = new WebDavConfig(DI::l10n(), DI::config(), DI::httpClient());
+	$data['storage'] = new WebDav($config->getUrl(), $config->getAuthOptions(), DI::httpClient(), DI::logger());
+}
+
+function webdav_storage_config(App $a, array &$data)
+{
+	$data['storage_config'] = new WebDavConfig(DI::l10n(), DI::config(), DI::httpClient());
+}