From 71eb147c1346748e6e985562e8a5c8bee910bb89 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 18 Apr 2023 05:56:32 +0000 Subject: [PATCH 1/4] Tumblr: Import the timeline --- tumblr/library/tumblroauth.php | 109 +++- tumblr/templates/connector_settings.tpl | 1 + tumblr/tumblr.php | 683 +++++++++++++++++------- 3 files changed, 601 insertions(+), 192 deletions(-) diff --git a/tumblr/library/tumblroauth.php b/tumblr/library/tumblroauth.php index 2c95759ab..162750f77 100644 --- a/tumblr/library/tumblroauth.php +++ b/tumblr/library/tumblroauth.php @@ -6,35 +6,62 @@ * The first PHP Library to support OAuth for Tumblr's REST API. (Originally for Twitter, modified for Tumblr by Lucas) */ +use Friendica\Core\Logger; use Friendica\DI; use Friendica\Security\OAuth1\OAuthConsumer; use Friendica\Security\OAuth1\OAuthRequest; use Friendica\Security\OAuth1\Signature\OAuthSignatureMethod_HMAC_SHA1; use Friendica\Security\OAuth1\OAuthToken; use Friendica\Security\OAuth1\OAuthUtil; +use GuzzleHttp\Client; +use GuzzleHttp\Exception\RequestException; +use GuzzleHttp\HandlerStack; +use GuzzleHttp\Subscriber\Oauth\Oauth1; +use Psr\Http\Message\ResponseInterface; /** * Tumblr OAuth class */ class TumblrOAuth { - /* Contains the last HTTP status code returned. */ - public $http_code; + private $consumer_key; + private $consumer_secret; + private $oauth_token; + private $oauth_token_secret; - /** @var OAuthConsumer */ - private $consumer; - /** @var \Friendica\Security\OAuth1\Signature\OAuthSignatureMethod_HMAC_SHA1 */ - private $sha1_method; + /** @var GuzzleHttp\Client */ + private $client; // API URLs const accessTokenURL = 'https://www.tumblr.com/oauth/access_token'; const authorizeURL = 'https://www.tumblr.com/oauth/authorize'; const requestTokenURL = 'https://www.tumblr.com/oauth/request_token'; - function __construct(string $consumer_key, string $consumer_secret) + function __construct(string $consumer_key, string $consumer_secret, string $oauth_token = '', string $oauth_token_secret = '') { - $this->sha1_method = new OAuthSignatureMethod_HMAC_SHA1(); - $this->consumer = new OAuthConsumer($consumer_key, $consumer_secret); + $this->consumer_key = $consumer_key; + $this->consumer_secret = $consumer_secret; + $this->oauth_token = $oauth_token; + $this->oauth_token_secret = $oauth_token_secret; + + if (empty($this->oauth_token) || empty($this->oauth_token_secret)) { + return; + } + + $stack = HandlerStack::create(); + + $middleware = new Oauth1([ + 'consumer_key' => $this->consumer_key, + 'consumer_secret' => $this->consumer_secret, + 'token' => $this->oauth_token, + 'token_secret' => $this->oauth_token_secret + ]); + $stack->push($middleware); + + $this->client = new Client([ + 'base_uri' => 'https://api.tumblr.com/v2/', + 'handler' => $stack + ]); } /** @@ -46,6 +73,9 @@ class TumblrOAuth function getRequestToken(string $oauth_callback): array { $request = $this->oAuthRequest(self::requestTokenURL, ['oauth_callback' => $oauth_callback]); + if (empty($request)) { + return []; + } return OAuthUtil::parse_parameters($request); } @@ -82,6 +112,9 @@ class TumblrOAuth } $request = $this->oAuthRequest(self::accessTokenURL, $parameters, $token); + if (empty($request)) { + return []; + } return OAuthUtil::parse_parameters($request); } @@ -95,14 +128,64 @@ class TumblrOAuth */ private function oAuthRequest(string $url, array $parameters, OAuthToken $token = null): string { - $request = OAuthRequest::from_consumer_and_token($this->consumer, 'GET', $url, $parameters, $token); - $request->sign_request($this->sha1_method, $this->consumer, $token); + $consumer = new OAuthConsumer($this->consumer_key, $this->consumer_secret); + $sha1_method = new OAuthSignatureMethod_HMAC_SHA1(); + + $request = OAuthRequest::from_consumer_and_token($consumer, 'GET', $url, $parameters, $token); + $request->sign_request($sha1_method, $consumer, $token); $curlResult = DI::httpClient()->get($request->to_url()); - $this->http_code = $curlResult->getReturnCode(); if ($curlResult->isSuccess()) { return $curlResult->getBody(); } return ''; } -} + + public function get(string $url, array $parameters = []): stdClass + { + if (!empty($parameters)) { + $url .= '?' . http_build_query($parameters); + } + + try { + $response = $this->client->get($url, ['auth' => 'oauth']); + } catch (RequestException $exception) { + $response = $exception->getResponse(); + Logger::notice('Get failed', ['code' => $exception->getCode(), 'message' => $exception->getMessage()]); + } + + return $this->formatResponse($response); + } + + public function post(string $url, array $parameter): stdClass + { + try { + $response = $this->client->post($url, ['auth' => 'oauth', 'json' => $parameter]); + } catch (RequestException $exception) { + $response = $exception->getResponse(); + Logger::notice('Post failed', ['code' => $exception->getCode(), 'message' => $exception->getMessage()]); + } + + return $this->formatResponse($response); + } + + private function formatResponse(ResponseInterface $response = null): stdClass + { + if (!is_null($response)) { + $content = $response->getBody()->getContents(); + if (!empty($content)) { + $result = json_decode($content); + } + } + + if (empty($result) || empty($result->meta)) { + $result = new stdClass; + $result->meta = new stdClass; + $result->meta->status = 500; + $result->meta->msg = ''; + $result->response = []; + $result->errors = []; + } + return $result; + } +} \ No newline at end of file diff --git a/tumblr/templates/connector_settings.tpl b/tumblr/templates/connector_settings.tpl index d28fab9d0..b5069e6e9 100644 --- a/tumblr/templates/connector_settings.tpl +++ b/tumblr/templates/connector_settings.tpl @@ -2,6 +2,7 @@ {{include file="field_checkbox.tpl" field=$enable}} {{include file="field_checkbox.tpl" field=$bydefault}} +{{include file="field_checkbox.tpl" field=$import}} {{if $page_select}} {{include file="field_select.tpl" field=$page_select}} diff --git a/tumblr/tumblr.php b/tumblr/tumblr.php index 8fba3cc6f..18d03a6a3 100644 --- a/tumblr/tumblr.php +++ b/tumblr/tumblr.php @@ -9,32 +9,40 @@ require_once __DIR__ . DIRECTORY_SEPARATOR . 'library' . DIRECTORY_SEPARATOR . 'tumblroauth.php'; +use Friendica\Content\PageInfo; use Friendica\Content\Text\BBCode; +use Friendica\Content\Text\HTML; use Friendica\Content\Text\NPF; +use Friendica\Core\Cache\Enum\Duration; use Friendica\Core\Hook; use Friendica\Core\Logger; +use Friendica\Core\Protocol; use Friendica\Core\Renderer; use Friendica\Core\System; +use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\Contact; use Friendica\Model\Item; +use Friendica\Model\ItemURI; use Friendica\Model\Photo; use Friendica\Model\Post; use Friendica\Model\Tag; +use Friendica\Protocol\Activity; use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; -use GuzzleHttp\Client; -use GuzzleHttp\Exception\RequestException; -use GuzzleHttp\HandlerStack; -use GuzzleHttp\Subscriber\Oauth\Oauth1; +use Friendica\Util\Strings; + +define('TUMBLR_DEFAULT_POLL_INTERVAL', 10); // given in minutes function tumblr_install() { - Hook::register('hook_fork', 'addon/tumblr/tumblr.php', 'tumblr_hook_fork'); - Hook::register('post_local', 'addon/tumblr/tumblr.php', 'tumblr_post_local'); - Hook::register('notifier_normal', 'addon/tumblr/tumblr.php', 'tumblr_send'); - Hook::register('jot_networks', 'addon/tumblr/tumblr.php', 'tumblr_jot_nets'); - Hook::register('connector_settings', 'addon/tumblr/tumblr.php', 'tumblr_settings'); - Hook::register('connector_settings_post', 'addon/tumblr/tumblr.php', 'tumblr_settings_post'); + Hook::register('hook_fork', __FILE__, 'tumblr_hook_fork'); + Hook::register('post_local', __FILE__, 'tumblr_post_local'); + Hook::register('notifier_normal', __FILE__, 'tumblr_send'); + Hook::register('jot_networks', __FILE__, 'tumblr_jot_nets'); + Hook::register('connector_settings', __FILE__, 'tumblr_settings'); + Hook::register('connector_settings_post', __FILE__, 'tumblr_settings_post'); + Hook::register('cron' , __FILE__, 'tumblr_cron'); } /** @@ -53,106 +61,84 @@ function tumblr_content() return ''; } - if (isset(DI::args()->getArgv()[1])) { - switch (DI::args()->getArgv()[1]) { - case 'connect': - $o = tumblr_connect(); - break; + if (!isset(DI::args()->getArgv()[1])) { + DI::baseUrl()->redirect('settings/connectors/tumblr'); + } - case 'callback': - $o = tumblr_callback(); - break; + switch (DI::args()->getArgv()[1]) { + case 'connect': + $o = tumblr_connect(); + break; - default: - $o = print_r(DI::args()->getArgv(), true); - break; - } - } else { - $o = tumblr_connect(); + case 'callback': + $o = tumblr_callback(); + break; + + default: + DI::baseUrl()->redirect('settings/connectors/tumblr'); + break; } return $o; } -function tumblr_addon_admin(string &$o) -{ - $t = Renderer::getMarkupTemplate('admin.tpl', 'addon/tumblr/'); - - $o = Renderer::replaceMacros($t, [ - '$submit' => DI::l10n()->t('Save Settings'), - // name, label, value, help, [extra values] - '$consumer_key' => ['consumer_key', DI::l10n()->t('Consumer Key'), DI::config()->get('tumblr', 'consumer_key'), ''], - '$consumer_secret' => ['consumer_secret', DI::l10n()->t('Consumer Secret'), DI::config()->get('tumblr', 'consumer_secret'), ''], - ]); -} - -function tumblr_addon_admin_post() -{ - DI::config()->set('tumblr', 'consumer_key', trim($_POST['consumer_key'] ?? '')); - DI::config()->set('tumblr', 'consumer_secret', trim($_POST['consumer_secret'] ?? '')); -} - function tumblr_connect() { - // Start a session. This is necessary to hold on to a few keys the callback script will also need - session_start(); - // Define the needed keys - $consumer_key = DI::config()->get('tumblr', 'consumer_key'); + $consumer_key = DI::config()->get('tumblr', 'consumer_key'); $consumer_secret = DI::config()->get('tumblr', 'consumer_secret'); + if (empty($consumer_key) || empty($consumer_secret)) { + DI::baseUrl()->redirect('settings/connectors/tumblr'); + } + // The callback URL is the script that gets called after the user authenticates with tumblr // In this example, it would be the included callback.php $callback_url = DI::baseUrl() . '/tumblr/callback'; - // Let's begin. First we need a Request Token. The request token is required to send the user + // Let's begin. First we need a Request Token. The request token is required to send the user // to Tumblr's login page. - // Create a new instance of the TumblrOAuth library. For this step, all we need to give the library is our + // Create a new instance of the TumblrOAuth library. For this step, all we need to give the library is our // Consumer Key and Consumer Secret $tum_oauth = new TumblrOAuth($consumer_key, $consumer_secret); - // Ask Tumblr for a Request Token. Specify the Callback URL here too (although this should be optional) + // Ask Tumblr for a Request Token. Specify the Callback URL here too (although this should be optional) $request_token = $tum_oauth->getRequestToken($callback_url); + if (empty($request_token)) { + // Give an error message + return DI::l10n()->t('Could not connect to Tumblr. Refresh the page or try again later.'); + } + // Store the request token and Request Token Secret as out callback.php script will need this DI::session()->set('request_token', $request_token['oauth_token']); DI::session()->set('request_token_secret', $request_token['oauth_token_secret']); - // Check the HTTP Code. It should be a 200 (OK), if it's anything else then something didn't work. - switch ($tum_oauth->http_code) { - case 200: - // Ask Tumblr to give us a special address to their login page - $url = $tum_oauth->getAuthorizeURL($request_token['oauth_token']); + // Ask Tumblr to give us a special address to their login page + $url = $tum_oauth->getAuthorizeURL($request_token['oauth_token']); - // Redirect the user to the login URL given to us by Tumblr - System::externalRedirect($url); + // Redirect the user to the login URL given to us by Tumblr + System::externalRedirect($url); - /* - * That's it for our side. The user is sent to a Tumblr Login page and - * asked to authroize our app. After that, Tumblr sends the user back to - * our Callback URL (callback.php) along with some information we need to get - * an access token. - */ - break; - - default: - // Give an error message - $o = 'Could not connect to Tumblr. Refresh the page or try again later.'; - } - - return $o; + /* + * That's it for our side. The user is sent to a Tumblr Login page and + * asked to authroize our app. After that, Tumblr sends the user back to + * our Callback URL (callback.php) along with some information we need to get + * an access token. + */ } function tumblr_callback() { - // Start a session, load the library - session_start(); - // Define the needed keys $consumer_key = DI::config()->get('tumblr', 'consumer_key'); $consumer_secret = DI::config()->get('tumblr', 'consumer_secret'); + if (empty($_REQUEST['oauth_verifier']) || empty($consumer_key) || empty($consumer_secret)) { + DI::baseUrl()->redirect('settings/connectors/tumblr'); + } + // Once the user approves your app at Tumblr, they are sent back to this script. // This script is passed two parameters in the URL, oauth_token (our Request Token) // and oauth_verifier (Key that we need to get Access Token). @@ -169,11 +155,8 @@ function tumblr_callback() DI::session()->remove('request_token'); DI::session()->remove('request_token_secret'); - // Make sure nothing went wrong. - if (200 == $tum_oauth->http_code) { - // good to go - } else { - return 'Unable to authenticate'; + if (empty($access_token)) { + return DI::l10n()->t('Unable to authenticate'); } // What's next? Now that we have an Access Token and Secret, we can make an API call. @@ -183,6 +166,76 @@ function tumblr_callback() DI::baseUrl()->redirect('settings/connectors/tumblr'); } +function tumblr_addon_admin(string &$o) +{ + $t = Renderer::getMarkupTemplate('admin.tpl', 'addon/tumblr/'); + + $o = Renderer::replaceMacros($t, [ + '$submit' => DI::l10n()->t('Save Settings'), + // name, label, value, help, [extra values] + '$consumer_key' => ['consumer_key', DI::l10n()->t('Consumer Key'), DI::config()->get('tumblr', 'consumer_key'), ''], + '$consumer_secret' => ['consumer_secret', DI::l10n()->t('Consumer Secret'), DI::config()->get('tumblr', 'consumer_secret'), ''], + ]); +} + +function tumblr_addon_admin_post() +{ + DI::config()->set('tumblr', 'consumer_key', trim($_POST['consumer_key'] ?? '')); + DI::config()->set('tumblr', 'consumer_secret', trim($_POST['consumer_secret'] ?? '')); +} + +function tumblr_settings(array &$data) +{ + if (!DI::userSession()->getLocalUserId()) { + return; + } + + $enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post', false); + $def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post_by_default', false); + $import = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'import', false); + + $cachekey = 'tumblr-blogs-' . DI::userSession()->getLocalUserId(); + $blogs = DI::cache()->get($cachekey); + if (empty($blogs)) { + $blogs = tumblr_get_blogs(DI::userSession()->getLocalUserId()); + if (!empty($blogs)) { + DI::cache()->set($cachekey, $blogs, Duration::HALF_HOUR); + } + } elseif (empty(tumblr_connection(DI::userSession()->getLocalUserId()))) { + $blogs = null; + DI::cache()->delete($cachekey); + } + + if (!empty($blogs)) { + $page = tumblr_get_page(DI::userSession()->getLocalUserId(), $blogs); + + $page_select = ['tumblr_page', DI::l10n()->t('Post to page:'), $page, '', $blogs]; + } + + $t = Renderer::getMarkupTemplate('connector_settings.tpl', 'addon/tumblr/'); + $html = Renderer::replaceMacros($t, [ + '$l10n' => [ + 'connect' => DI::l10n()->t('(Re-)Authenticate your tumblr page'), + 'noconnect' => DI::l10n()->t('You are not authenticated to tumblr'), + ], + + '$authenticate_url' => DI::baseUrl() . '/tumblr/connect', + + '$enable' => ['tumblr', DI::l10n()->t('Enable Tumblr Post Addon'), $enabled], + '$bydefault' => ['tumblr_bydefault', DI::l10n()->t('Post to Tumblr by default'), $def_enabled], + '$import' => ['tumblr_import', DI::l10n()->t('Import the remote timeline'), $import], + '$page_select' => $page_select ?? '', + ]); + + $data = [ + 'connector' => 'tumblr', + 'title' => DI::l10n()->t('Tumblr Export'), + 'image' => 'images/tumblr.png', + 'enabled' => $enabled, + 'html' => $html, + ]; +} + function tumblr_jot_nets(array &$jotnets_fields) { if (!DI::userSession()->getLocalUserId()) { @@ -201,51 +254,13 @@ function tumblr_jot_nets(array &$jotnets_fields) } } -function tumblr_settings(array &$data) -{ - if (!DI::userSession()->getLocalUserId()) { - return; - } - - $enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post', false); - $def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post_by_default', false); - - $blogs = tumblr_get_blogs(DI::userSession()->getLocalUserId()); - if (!empty($blogs)) { - $page = tumblr_get_page(DI::userSession()->getLocalUserId(), $blogs); - - $page_select = ['tumblr_page', DI::l10n()->t('Post to page:'), $page, '', $blogs]; - } - - $t = Renderer::getMarkupTemplate('connector_settings.tpl', 'addon/tumblr/'); - $html = Renderer::replaceMacros($t, [ - '$l10n' => [ - 'connect' => DI::l10n()->t('(Re-)Authenticate your tumblr page'), - 'noconnect' => DI::l10n()->t('You are not authenticated to tumblr'), - ], - - '$authenticate_url' => DI::baseUrl() . '/tumblr/connect', - - '$enable' => ['tumblr', DI::l10n()->t('Enable Tumblr Post Addon'), $enabled], - '$bydefault' => ['tumblr_bydefault', DI::l10n()->t('Post to Tumblr by default'), $def_enabled], - '$page_select' => $page_select ?? '', - ]); - - $data = [ - 'connector' => 'tumblr', - 'title' => DI::l10n()->t('Tumblr Export'), - 'image' => 'images/tumblr.png', - 'enabled' => $enabled, - 'html' => $html, - ]; -} - function tumblr_settings_post(array &$b) { if (!empty($_POST['tumblr-submit'])) { DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'tumblr', 'post', intval($_POST['tumblr'])); DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'tumblr', 'page', $_POST['tumblr_page']); DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'tumblr', 'post_by_default', intval($_POST['tumblr_bydefault'])); + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'tumblr', 'import', intval($_POST['tumblr_import'])); } } @@ -283,9 +298,9 @@ function tumblr_post_local(array &$b) } $tmbl_post = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post')); - $tmbl_enable = (($tmbl_post && !empty($_REQUEST['tumblr_enable'])) ? intval($_REQUEST['tumblr_enable']) : 0); + // if API is used, default to the chosen settings if ($b['api_source'] && intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post_by_default'))) { $tmbl_enable = 1; } @@ -395,13 +410,13 @@ function tumblr_send(array &$b) $page = tumblr_get_page($b['uid']); - $result = tumblr_post($connection, 'blog/' . $page . '/post', $params); + $result = $connection->post('blog/' . $page . '/post', $params); - if ($result['success']) { - Logger::info('success', ['blog' => $page, 'params' => $params]); + if ($result->meta->status < 400) { + Logger::info('Success (legacy)', ['blog' => $page, 'meta' => $result->meta, 'response' => $result->response]); return true; } else { - Logger::notice('error', ['blog' => $page, 'params' => $params, 'result' => $result['data']]); + Logger::notice('Error posting blog (legacy)', ['blog' => $page, 'meta' => $result->meta, 'response' => $result->response, 'errors' => $result->errors, 'params' => $params]); return false; } } @@ -416,7 +431,7 @@ function tumblr_send_npf(array $post): bool // "true" is returned, since the legacy function will fail as well. return true; } - + $post['body'] = Post\Media::addAttachmentsToBody($post['uri-id'], $post['body']); if (!empty($post['title'])) { $post['body'] = '[h1]' . $post['title'] . "[/h1]\n" . $post['body']; @@ -431,18 +446,374 @@ function tumblr_send_npf(array $post): bool 'interactability_reblog' => 'everyone' ]; - $result = tumblr_post($connection, 'blog/' . $page . '/posts', $params); + $result = $connection->post('blog/' . $page . '/posts', $params); - if ($result['success']) { - Logger::info('success', ['blog' => $page, 'params' => $params]); + if ($result->meta->status < 400) { + Logger::info('Success (NPF)', ['blog' => $page, 'meta' => $result->meta, 'response' => $result->response]); return true; } else { - Logger::notice('error', ['blog' => $page, 'params' => $params, 'result' => $result['data']]); + Logger::notice('Error posting blog (NPF)', ['blog' => $page, 'meta' => $result->meta, 'response' => $result->response, 'errors' => $result->errors, 'params' => $params]); return false; } } -function tumblr_connection(int $uid): ?GuzzleHttp\Client +function tumblr_cron() +{ + $last = DI::keyValue()->get('tumblr_last_poll'); + + $poll_interval = intval(DI::config()->get('tumblr', 'poll_interval')); + if (!$poll_interval) { + $poll_interval = TUMBLR_DEFAULT_POLL_INTERVAL; + } + + if ($last) { + $next = $last + ($poll_interval * 60); + if ($next > time()) { + Logger::notice('poll intervall not reached'); + return; + } + } + Logger::notice('cron_start'); + + $abandon_days = intval(DI::config()->get('system', 'account_abandon_days')); + if ($abandon_days < 1) { + $abandon_days = 0; + } + + $abandon_limit = date(DateTimeFormat::MYSQL, time() - $abandon_days * 86400); + + $pconfigs = DBA::selectToArray('pconfig', [], ['cat' => 'tumblr', 'k' => 'import', 'v' => true]); + foreach ($pconfigs as $pconfig) { + if ($abandon_days != 0) { + if (!DBA::exists('user', ["`uid` = ? AND `login_date` >= ?", $pconfig['uid'], $abandon_limit])) { + Logger::notice('abandoned account: timeline from user will not be imported', ['user' => $pconfig['uid']]); + continue; + } + } + + Logger::notice('importing timeline - start', ['user' => $pconfig['uid']]); + tumblr_fetch_dashboard($pconfig['uid']); + Logger::notice('importing timeline - done', ['user' => $pconfig['uid']]); + } + + Logger::notice('cron_end'); + + DI::keyValue()->set('tumblr_last_poll', time()); +} + +function tumblr_add_npf_data($html) +{ + $doc = new DOMDocument(); + + $doc->formatOutput = true; + @$doc->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8')); + $xpath = new DomXPath($doc); + $list = $xpath->query('//p[@class="npf_link"]'); + foreach ($list as $node) { + $data = tumblr_get_npf_data($node); + if (empty($data)) { + continue; + } + + tumblr_replace_with_npf($doc, $node, PageInfo::getFooterFromUrl($data['url'])); + } + + $list = $xpath->query('//figure[@data-provider="youtube"]'); + foreach ($list as $node) { + $attributes = tumblr_get_attributes($node); + if (empty($attributes['data-url'])) { + continue; + } + tumblr_replace_with_npf($doc, $node, '[youtube]' . $attributes['data-url'] . '[/youtube]'); + } + + return $doc->saveHTML(); +} + +function tumblr_get_attributes($node): array +{ + $attributes = []; + foreach ($node->attributes as $key => $attribute) { + $attributes[$key] = trim($attribute->value); + } + return $attributes; +} + +function tumblr_get_npf_data($node): array +{ + $attributes = tumblr_get_attributes($node); + if (empty($attributes['data-npf'])) { + return []; + } + + return json_decode($attributes['data-npf'], true); +} + +function tumblr_replace_with_npf($doc, $node, $replacement) +{ + $replace = $doc->createTextNode($replacement); + $node->parentNode->insertBefore($replace, $node); + $node->parentNode->removeChild($node); +} + +function tumblr_fetch_dashboard(int $uid) +{ + $page = tumblr_get_page($uid); + + $parameters = ['reblog_info' => false, 'notes_info' => false, 'npf' => false]; + + $last = DI::pConfig()->get($uid, 'tumblr', 'last_id'); + if (!empty($last)) { + $parameters['since_id'] = $last; + } + + $connection = tumblr_connection($uid); + $dashboard = $connection->get('user/dashboard', $parameters); + if ($dashboard->meta->status > 399) { + Logger::notice('Error fetching dashboard', ['meta' => $dashboard->meta, 'response' => $dashboard->response, 'errors' => $dashboard->errors]); + return []; + } + + if (empty($dashboard->response->posts)) { + return; + } + + foreach (array_reverse($dashboard->response->posts) as $post) { + $uri = 'tumblr::' . $post->id_string; + + if ($post->id > $last) { + $last = $post->id; + } + + if (Post::exists(['uri' => $uri, 'uid' => $uid]) || ($post->blog->uuid == $page)) { + DI::pConfig()->set($uid, 'tumblr', 'last_id', $last); + continue; + } + + $item = tumblr_get_header($post, $uri, $uid); + + $item = tumblr_get_content($item, $post); + item::insert($item); + + DI::pConfig()->set($uid, 'tumblr', 'last_id', $last); + } +} + +function tumblr_get_header(stdClass $post, string $uri, int $uid): array +{ + $contact = tumblr_get_contact($post->blog, $uid); + $item = [ + 'network' => Protocol::TUMBLR, + 'uid' => $uid, + 'wall' => false, + 'uri' => $uri, + 'private' => Item::UNLISTED, + 'verb' => Activity::POST, + 'contact-id' => $contact['id'], + 'author-name' => $contact['name'], + 'author-link' => $contact['url'], + 'author-avatar' => $contact['avatar'], + 'plink' => $post->post_url, + 'created' => date(DateTimeFormat::MYSQL, $post->timestamp) + ]; + + $item['owner-name'] = $item['author-name']; + $item['owner-link'] = $item['author-link']; + $item['owner-avatar'] = $item['author-avatar']; + + // @todo process $post->tags; + + return $item; +} + +function tumblr_get_content(array $item, stdClass $post): array +{ + switch ($post->type) { + case 'text': + $item['title'] = $post->title; + $item['body'] = HTML::toBBCode($post->body); + break; + + case 'quote': + if (empty($post->text)) { + $body = HTML::toBBCode($post->text) . "\n"; + } else { + $body = ''; + } + if (!empty($post->source_title) && !empty($post->source_url)) { + $body .= '[url=' . $post->source_url . ']' . $post->source_title . "[/url]:\n"; + } elseif (!empty($post->source_title)) { + $body .= $post->source_title . ":\n"; + } + $body .= '[quote]' . HTML::toBBCode($post->source) . '[/quote]'; + $item['body'] = $body; + break; + + case 'link': + $item['body'] = HTML::toBBCode($post->description) . "\n" . PageInfo::getFooterFromUrl($post->url); + break; + + case 'answer': + if (!empty($post->asking_name) && !empty($post->asking_url)) { + $body = '[url=' . $post->asking_url . ']' . $post->asking_name . "[/url]:\n"; + } elseif (!empty($post->asking_name)) { + $body = $post->asking_name . ":\n"; + } else { + $body = ''; + } + $body .= '[quote]' . HTML::toBBCode($post->question) . "[/quote]\n" . HTML::toBBCode($post->answer); + $item['body'] = $body; + break; + + case 'video': + $item['body'] = HTML::toBBCode($post->caption); + if (!empty($post->video_url)) { + $item['body'] .= "\n[video]" . $post->video_url . "[/video]\n"; + } elseif(!empty($post->thumbnail_url)) { + $item['body'] .= "\n[url=" . $post->permalink_url ."][img]" . $post->thumbnail_url . "[/img][/url]\n"; + } elseif(!empty($post->permalink_url)) { + $item['body'] .= "\n[url]" . $post->permalink_url ."[/url]\n"; + } elseif(!empty($post->source_url) && !empty($post->source_title)) { + $item['body'] .= "\n[url=" . $post->source_url ."]" . $post->source_title . "[/url]\n"; + } elseif(!empty($post->source_url)) { + $item['body'] .= "\n[url]" . $post->source_url ."[/url]\n"; + } + break; + + case 'audio': + $item['body'] = HTML::toBBCode($post->caption); + if(!empty($post->source_url) && !empty($post->source_title)) { + $item['body'] .= "\n[url=" . $post->source_url ."]" . $post->source_title . "[/url]\n"; + } elseif(!empty($post->source_url)) { + $item['body'] .= "\n[url]" . $post->source_url ."[/url]\n"; + } + break; + + case 'photo': + $item['body'] = HTML::toBBCode($post->caption); + foreach ($post->photos as $photo) { + if (!empty($photo->original_size)) { + $item['body'] .= "\n[img]" . $photo->original_size->url . "[/img]"; + } elseif (!empty($photo->alt_sizes)) { + $item['body'] .= "\n[img]" . $photo->alt_sizes[0]->url . "[/img]"; + } + } + break; + + case 'chat': + $item['title'] = $post->title; + $item['body'] = "\n[ul]"; + foreach ($post->dialogue as $line) { + $item['body'] .= "\n[li]" . $line->label . " " . $line->phrase . "[/li]"; + } + $item['body'] .= "[/ul]\n"; + break; + } + return $item; +} + +function tumblr_get_contact(stdClass $blog, int $uid) +{ + $condition = ['network' => Protocol::TUMBLR, 'uid' => $uid, 'poll' => 'tumblr::' . $blog->uuid]; + $contact = Contact::selectFirst([], $condition); + if (!empty($contact) && (strtotime($contact['updated']) >= $blog->updated)) { + return $contact; + } + if (empty($contact)) { + $cid = tumblr_insert_contact($blog, $uid); + } else { + $cid = $contact['id']; + } + + $condition['uid'] = 0; + + $contact = Contact::selectFirst([], $condition); + if (empty($contact)) { + $pcid = tumblr_insert_contact($blog, 0); + } else { + $pcid = $contact['id']; + } + + tumblr_update_contact($blog, $uid, $cid, $pcid); + + return Contact::getById($cid); +} + +function tumblr_insert_contact(stdClass $blog, int $uid) +{ + $baseurl = 'https://tumblr.com'; + $url = $baseurl . '/' . $blog->name; + + $fields = [ + 'uid' => $uid, + 'network' => Protocol::TUMBLR, + 'poll' => 'tumblr::' . $blog->uuid, + 'baseurl' => $baseurl, + 'priority' => 1, + 'writable' => false, // @todo Allow interaction at a later point in time + 'blocked' => false, + 'readonly' => false, + 'pending' => false, + 'url' => $url, + 'nurl' => Strings::normaliseLink($url), + 'alias' => $blog->url, + 'name' => $blog->title, + 'nick' => $blog->name, + 'addr' => $blog->name . '@tumblr.com', + 'about' => $blog->description, + 'updated' => date(DateTimeFormat::MYSQL, $blog->updated) + ]; + return Contact::insert($fields); +} + +function tumblr_update_contact(stdClass $blog, int $uid, int $cid, int $pcid) +{ + $connection = tumblr_connection($uid); + $info = $connection->get('blog/' . $blog->uuid . '/info'); + if ($info->meta->status > 399) { + Logger::notice('Error fetching dashboard', ['meta' => $info->meta, 'response' => $info->response, 'errors' => $info->errors]); + return; + } + + $avatar = $info->response->blog->avatar; + if (!empty($avatar)) { + Contact::updateAvatar($cid, $avatar[0]->url); + } + + $baseurl = 'https://tumblr.com'; + $url = $baseurl . '/' . $info->response->blog->name; + + if ($info->response->blog->followed && $info->response->blog->subscribed) { + $rel = Contact::FRIEND; + } elseif ($info->response->blog->followed && !$info->response->blog->subscribed) { + $rel = Contact::SHARING; + } elseif (!$info->response->blog->followed && $info->response->blog->subscribed) { + $rel = Contact::FOLLOWER; + } else { + $rel = Contact::NOTHING; + } + + $fields = [ + 'url' => $url, + 'nurl' => Strings::normaliseLink($url), + 'uri-id' => ItemURI::getIdByURI($url), + 'alias' => $info->response->blog->url, + 'name' => $info->response->blog->title, + 'nick' => $info->response->blog->name, + 'addr' => $info->response->blog->name . '@tumblr.com', + 'about' => $info->response->blog->description, + 'updated' => date(DateTimeFormat::MYSQL, $info->response->blog->updated), + 'header' => $info->response->blog->theme->header_image_focused, + 'rel' => $rel, + ]; + + Contact::update($fields, ['id' => $cid]); + + $fields['rel'] = Contact::NOTHING; + Contact::update($fields, ['id' => $pcid]); +} + +function tumblr_connection(int $uid): ?TumblrOAuth { $oauth_token = DI::pConfig()->get($uid, 'tumblr', 'oauth_token'); $oauth_token_secret = DI::pConfig()->get($uid, 'tumblr', 'oauth_token_secret'); @@ -454,25 +825,8 @@ function tumblr_connection(int $uid): ?GuzzleHttp\Client Logger::notice('Missing data, connection is not established', ['uid' => $uid]); return null; } - return tumblr_client($consumer_key, $consumer_secret, $oauth_token, $oauth_token_secret); -} -function tumblr_client(string $consumer_key, string $consumer_secret, string $oauth_token, string $oauth_token_secret): GuzzleHttp\Client -{ - $stack = HandlerStack::create(); - - $middleware = new Oauth1([ - 'consumer_key' => $consumer_key, - 'consumer_secret' => $consumer_secret, - 'token' => $oauth_token, - 'token_secret' => $oauth_token_secret - ]); - $stack->push($middleware); - - return new Client([ - 'base_uri' => 'https://api.tumblr.com/v2/', - 'handler' => $stack - ]); + return new TumblrOAuth($consumer_key, $consumer_secret, $oauth_token, $oauth_token_secret); } function tumblr_get_page(int $uid, array $blogs = []) @@ -503,44 +857,15 @@ function tumblr_get_blogs(int $uid) return []; } - $userinfo = tumblr_get($connection, 'user/info'); - if (empty($userinfo['success'])) { + $userinfo = $connection->get('user/info'); + if ($userinfo->meta->status > 299) { + Logger::notice('Error fetching blogs', ['meta' => $userinfo->meta, 'response' => $userinfo->response, 'errors' => $userinfo->errors]); return []; } $blogs = []; - foreach ($userinfo['data']->response->user->blogs as $blog) { + foreach ($userinfo->response->user->blogs as $blog) { $blogs[$blog->uuid] = $blog->name; } return $blogs; -} - -function tumblr_get($connection, string $url) -{ - try { - $res = $connection->get($url, ['auth' => 'oauth']); - - $success = true; - $data = json_decode($res->getBody()->getContents()); - } catch (RequestException $exception) { - $success = false; - $data = []; - Logger::notice('Request failed', ['code' => $exception->getCode(), 'message' => $exception->getMessage()]); - } - return ['success' => $success, 'data' => $data]; -} - -function tumblr_post($connection, string $url, array $parameter) -{ - try { - $res = $connection->post($url, ['auth' => 'oauth', 'json' => $parameter]); - - $success = true; - $data = json_decode($res->getBody()->getContents()); - } catch (RequestException $exception) { - $success = false; - $data = json_decode($exception->getResponse()->getBody()->getContents()); - Logger::notice('Post failed', ['code' => $exception->getCode(), 'message' => $exception->getMessage()]); - } - return ['success' => $success, 'data' => $data]; } \ No newline at end of file From 9c8e7a23a6e7fe54b6d80f174bb5692fc1aab741 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 18 Apr 2023 21:05:31 +0000 Subject: [PATCH 2/4] Add more types --- tumblr/tumblr.php | 50 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/tumblr/tumblr.php b/tumblr/tumblr.php index 18d03a6a3..606e73d1d 100644 --- a/tumblr/tumblr.php +++ b/tumblr/tumblr.php @@ -501,7 +501,7 @@ function tumblr_cron() DI::keyValue()->set('tumblr_last_poll', time()); } -function tumblr_add_npf_data($html) +function tumblr_add_npf_data(string $html, string $plink): string { $doc = new DOMDocument(); @@ -514,8 +514,18 @@ function tumblr_add_npf_data($html) if (empty($data)) { continue; } - - tumblr_replace_with_npf($doc, $node, PageInfo::getFooterFromUrl($data['url'])); + + tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); + } + + $list = $xpath->query('//div[@data-npf]'); + foreach ($list as $node) { + $data = tumblr_get_npf_data($node); + if (empty($data)) { + continue; + } + + tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); } $list = $xpath->query('//figure[@data-provider="youtube"]'); @@ -527,7 +537,29 @@ function tumblr_add_npf_data($html) tumblr_replace_with_npf($doc, $node, '[youtube]' . $attributes['data-url'] . '[/youtube]'); } - return $doc->saveHTML(); + return $doc->saveHTML(); +} + +function tumblr_get_type_replacement(array $data, string $plink): string +{ + switch ($data['type']) { + case 'poll': + $body = '[p][url=' . $plink. ']'. $data['question'] . '[/url][/p][ul]'; + foreach ($data['answers'] as $answer) { + $body .= '[li]' . $answer['answer_text'] . '[/li]'; + } + $body .= '[/ul]'; + break; + + case 'link': + $body = PageInfo::getFooterFromUrl($data['url']); + + default: + Logger::notice('Unknown type', ['type' => $data['type'], 'data' => $data, 'plink' => $plink]); + $body = ''; + } + + return $body; } function tumblr_get_attributes($node): array @@ -539,7 +571,7 @@ function tumblr_get_attributes($node): array return $attributes; } -function tumblr_get_npf_data($node): array +function tumblr_get_npf_data(DOMNode $node): array { $attributes = tumblr_get_attributes($node); if (empty($attributes['data-npf'])) { @@ -549,7 +581,7 @@ function tumblr_get_npf_data($node): array return json_decode($attributes['data-npf'], true); } -function tumblr_replace_with_npf($doc, $node, $replacement) +function tumblr_replace_with_npf(DOMDocument $doc, DOMNode $node, string $replacement) { $replace = $doc->createTextNode($replacement); $node->parentNode->insertBefore($replace, $node); @@ -631,7 +663,7 @@ function tumblr_get_content(array $item, stdClass $post): array switch ($post->type) { case 'text': $item['title'] = $post->title; - $item['body'] = HTML::toBBCode($post->body); + $item['body'] = HTML::toBBCode(tumblr_add_npf_data($post->body, $post->post_url)); break; case 'quote': @@ -829,7 +861,7 @@ function tumblr_connection(int $uid): ?TumblrOAuth return new TumblrOAuth($consumer_key, $consumer_secret, $oauth_token, $oauth_token_secret); } -function tumblr_get_page(int $uid, array $blogs = []) +function tumblr_get_page(int $uid, array $blogs = []): string { $page = DI::pConfig()->get($uid, 'tumblr', 'page'); @@ -850,7 +882,7 @@ function tumblr_get_page(int $uid, array $blogs = []) return ''; } -function tumblr_get_blogs(int $uid) +function tumblr_get_blogs(int $uid): array { $connection = tumblr_connection($uid); if (empty($connection)) { From 9e7f06ed4407cc177eab918cf5ec0a9c6ab80202 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 22 Apr 2023 10:01:09 +0000 Subject: [PATCH 3/4] Tumblr: Dashboard import and activities are working --- tumblr/lang/C/messages.po | 52 +++++++------- tumblr/tumblr.php | 143 ++++++++++++++++++++++++++++++++------ 2 files changed, 151 insertions(+), 44 deletions(-) diff --git a/tumblr/lang/C/messages.po b/tumblr/lang/C/messages.po index 83434406b..f3c2fc637 100644 --- a/tumblr/lang/C/messages.po +++ b/tumblr/lang/C/messages.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2021-11-21 19:17-0500\n" +"POT-Creation-Date: 2023-04-22 10:00+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -17,54 +17,58 @@ msgstr "" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -#: tumblr.php:39 +#: tumblr.php:60 msgid "Permission denied." msgstr "" -#: tumblr.php:69 +#: tumblr.php:111 +msgid "Could not connect to Tumblr. Refresh the page or try again later." +msgstr "" + +#: tumblr.php:159 +msgid "Unable to authenticate" +msgstr "" + +#: tumblr.php:174 msgid "Save Settings" msgstr "" -#: tumblr.php:71 +#: tumblr.php:176 msgid "Consumer Key" msgstr "" -#: tumblr.php:72 +#: tumblr.php:177 msgid "Consumer Secret" msgstr "" -#: tumblr.php:177 -msgid "You are now authenticated to tumblr." -msgstr "" - -#: tumblr.php:178 -msgid "return to the connector page" -msgstr "" - -#: tumblr.php:194 -msgid "Post to Tumblr" -msgstr "" - -#: tumblr.php:225 +#: tumblr.php:212 msgid "Post to page:" msgstr "" -#: tumblr.php:231 +#: tumblr.php:218 msgid "(Re-)Authenticate your tumblr page" msgstr "" -#: tumblr.php:232 +#: tumblr.php:219 msgid "You are not authenticated to tumblr" msgstr "" -#: tumblr.php:237 +#: tumblr.php:224 msgid "Enable Tumblr Post Addon" msgstr "" -#: tumblr.php:238 +#: tumblr.php:225 msgid "Post to Tumblr by default" msgstr "" -#: tumblr.php:244 -msgid "Tumblr Export" +#: tumblr.php:226 +msgid "Import the remote timeline" +msgstr "" + +#: tumblr.php:232 +msgid "Tumblr Import/Export" +msgstr "" + +#: tumblr.php:250 +msgid "Post to Tumblr" msgstr "" diff --git a/tumblr/tumblr.php b/tumblr/tumblr.php index 606e73d1d..88ae748bf 100644 --- a/tumblr/tumblr.php +++ b/tumblr/tumblr.php @@ -229,7 +229,7 @@ function tumblr_settings(array &$data) $data = [ 'connector' => 'tumblr', - 'title' => DI::l10n()->t('Tumblr Export'), + 'title' => DI::l10n()->t('Tumblr Import/Export'), 'image' => 'images/tumblr.png', 'enabled' => $enabled, 'html' => $html, @@ -272,10 +272,22 @@ function tumblr_hook_fork(array &$b) $post = $b['data']; - if ( - $post['deleted'] || $post['private'] || ($post['created'] !== $post['edited']) || - !strstr($post['postopts'] ?? '', 'tumblr') || ($post['parent'] != $post['id']) - ) { + // Editing is not supported by the addon + if (($post['created'] !== $post['edited']) && !$post['deleted']) { + DI::logger()->info('Editing is not supported by the addon'); + $b['execute'] = false; + return; + } + + if (DI::pConfig()->get($post['uid'], 'tumblr', 'import')) { + // Don't post if it isn't a reply to a tumblr post + if (($post['parent'] != $post['id']) && !Post::exists(['id' => $post['parent'], 'network' => Protocol::TUMBLR])) { + Logger::notice('No tumblr parent found', ['item' => $post['id']]); + $b['execute'] = false; + return; + } + } elseif (!strstr($post['postopts'] ?? '', 'tumblr') || ($post['parent'] != $post['id']) || $post['private']) { + DI::logger()->info('Activities are never exported when we don\'t import the tumblr timeline'); $b['execute'] = false; return; } @@ -283,8 +295,6 @@ function tumblr_hook_fork(array &$b) function tumblr_post_local(array &$b) { - // This can probably be changed to allow editing by pointing to a different API endpoint - if ($b['edit']) { return; } @@ -318,15 +328,63 @@ function tumblr_post_local(array &$b) function tumblr_send(array &$b) { - if ($b['deleted'] || $b['private'] || ($b['created'] !== $b['edited'])) { - return; - } - - if (!strstr($b['postopts'], 'tumblr')) { + if (($b['created'] !== $b['edited']) && !$b['deleted']) { return; } if ($b['gravity'] != Item::GRAVITY_PARENT) { + Logger::debug('Got comment', ['item' => $b]); + + $parent = tumblr_get_post_from_uri($b['thr-parent']); + if (empty($parent)) { + Logger::notice('No tumblr post', ['thr-parent' => $b['thr-parent']]); + return; + } + + Logger::debug('Parent found', ['parent' => $parent]); + + $connection = tumblr_connection($b['uid']); + if (empty($connection)) { + return; + } + + $page = tumblr_get_page($b['uid']); + + if ($b['gravity'] == Item::GRAVITY_COMMENT) { + Logger::notice('Commenting is not supported (yet)'); + } else { + if (($b['verb'] == Activity::LIKE) && !$b['deleted']) { + $params = ['id' => $parent['id'], 'reblog_key' => $parent['reblog_key']]; + $result = $connection->post('user/like', $params); + } elseif (($b['verb'] == Activity::LIKE) && $b['deleted']) { + $params = ['id' => $parent['id'], 'reblog_key' => $parent['reblog_key']]; + $result = $connection->post('user/unlike', $params); + } elseif (($b['verb'] == Activity::ANNOUNCE) && !$b['deleted']) { + $params = ['id' => $parent['id'], 'reblog_key' => $parent['reblog_key']]; + $result = $connection->post('blog/' . $page . '/post/reblog', $params); + } elseif (($b['verb'] == Activity::ANNOUNCE) && $b['deleted']) { + $announce = tumblr_get_post_from_uri($b['extid']); + if (empty($announce)) { + return; + } + $params = ['id' => $announce['id']]; + $result = $connection->post('blog/' . $page . '/post/delete', $params); + } else { + // Unsupported activity + return; + } + + if ($result->meta->status < 400) { + Logger::info('Successfully performed activity', ['verb' => $b['verb'], 'deleted' => $b['deleted'], 'meta' => $result->meta, 'response' => $result->response]); + if (!$b['deleted'] && !empty($result->response->id_string)) { + Item::update(['extid' => 'tumblr::' . $result->response->id_string], ['id' => $b['id']]); + } + } else { + Logger::notice('Error while performing activity', ['verb' => $b['verb'], 'deleted' => $b['deleted'], 'meta' => $result->meta, 'response' => $result->response, 'errors' => $result->errors, 'params' => $params]); + } + } + return; + } elseif ($b['private'] || !strstr($b['postopts'], 'tumblr')) { return; } @@ -421,6 +479,20 @@ function tumblr_send(array &$b) } } +function tumblr_get_post_from_uri(string $uri): array +{ + $parts = explode(':', $uri); + if (($parts[0] != 'tumblr') || empty($parts[2])) { + return []; + } + + $post ['id'] = $parts[2]; + $post['reblog_key'] = $parts[3] ?? ''; + + $post['reblog_key'] = str_replace('@t', '', $post['reblog_key']); // Temp + return $post; +} + function tumblr_send_npf(array $post): bool { $page = tumblr_get_page($post['uid']); @@ -537,6 +609,15 @@ function tumblr_add_npf_data(string $html, string $plink): string tumblr_replace_with_npf($doc, $node, '[youtube]' . $attributes['data-url'] . '[/youtube]'); } + $list = $xpath->query('//figure[@data-npf]'); + foreach ($list as $node) { + $data = tumblr_get_npf_data($node); + if (empty($data)) { + continue; + } + tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); + } + return $doc->saveHTML(); } @@ -552,8 +633,15 @@ function tumblr_get_type_replacement(array $data, string $plink): string break; case 'link': - $body = PageInfo::getFooterFromUrl($data['url']); + $body = PageInfo::getFooterFromUrl(str_replace('https://href.li/?', '', $data['url'])); + break; + case 'video': + if (!empty($data['url']) && ($data['provider'] == 'tumblr')) { + $body = '[video]' . $data['url'] . '[/video]'; + break; + } + default: Logger::notice('Unknown type', ['type' => $data['type'], 'data' => $data, 'plink' => $plink]); $body = ''; @@ -583,6 +671,9 @@ function tumblr_get_npf_data(DOMNode $node): array function tumblr_replace_with_npf(DOMDocument $doc, DOMNode $node, string $replacement) { + if (empty($replacement)) { + return; + } $replace = $doc->createTextNode($replacement); $node->parentNode->insertBefore($replace, $node); $node->parentNode->removeChild($node); @@ -611,12 +702,14 @@ function tumblr_fetch_dashboard(int $uid) } foreach (array_reverse($dashboard->response->posts) as $post) { - $uri = 'tumblr::' . $post->id_string; + $uri = 'tumblr::' . $post->id_string . ':' . $post->reblog_key; if ($post->id > $last) { $last = $post->id; } + Logger::debug('Importing post', ['uid' => $uid, 'created' => date(DateTimeFormat::MYSQL, $post->timestamp), 'uri' => $uri]); + if (Post::exists(['uri' => $uri, 'uid' => $uid]) || ($post->blog->uuid == $page)) { DI::pConfig()->set($uid, 'tumblr', 'last_id', $last); continue; @@ -625,7 +718,18 @@ function tumblr_fetch_dashboard(int $uid) $item = tumblr_get_header($post, $uri, $uid); $item = tumblr_get_content($item, $post); - item::insert($item); + + $id = item::insert($item); + + if ($id) { + $stored = Post::selectFirst(['uri-id'], ['id' => $id]); + + if (!empty($post->tags)) { + foreach ($post->tags as $tag) { + Tag::store($stored['uri-id'], Tag::HASHTAG, $tag); + } + } + } DI::pConfig()->set($uid, 'tumblr', 'last_id', $last); } @@ -653,8 +757,6 @@ function tumblr_get_header(stdClass $post, string $uri, int $uid): array $item['owner-link'] = $item['author-link']; $item['owner-avatar'] = $item['author-avatar']; - // @todo process $post->tags; - return $item; } @@ -782,7 +884,7 @@ function tumblr_insert_contact(stdClass $blog, int $uid) 'poll' => 'tumblr::' . $blog->uuid, 'baseurl' => $baseurl, 'priority' => 1, - 'writable' => false, // @todo Allow interaction at a later point in time + 'writable' => true, 'blocked' => false, 'readonly' => false, 'pending' => false, @@ -825,15 +927,16 @@ function tumblr_update_contact(stdClass $blog, int $uid, int $cid, int $pcid) $rel = Contact::NOTHING; } + $uri_id = ItemURI::getIdByURI($url); $fields = [ 'url' => $url, 'nurl' => Strings::normaliseLink($url), - 'uri-id' => ItemURI::getIdByURI($url), + 'uri-id' => $uri_id, 'alias' => $info->response->blog->url, 'name' => $info->response->blog->title, 'nick' => $info->response->blog->name, 'addr' => $info->response->blog->name . '@tumblr.com', - 'about' => $info->response->blog->description, + 'about' => BBCode::convertForUriId($uri_id, $info->response->blog->description, BBCode::CONNECTORS), 'updated' => date(DateTimeFormat::MYSQL, $info->response->blog->updated), 'header' => $info->response->blog->theme->header_image_focused, 'rel' => $rel, From da65314df5f67b00de97f3831f1e2424d9d37438 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 23 Apr 2023 10:26:19 +0000 Subject: [PATCH 4/4] Resructured code, added documentation --- tumblr/library/tumblroauth.php | 20 ++ tumblr/tumblr.php | 422 +++++++++++++++++++-------------- 2 files changed, 263 insertions(+), 179 deletions(-) diff --git a/tumblr/library/tumblroauth.php b/tumblr/library/tumblroauth.php index 162750f77..d7438f1d4 100644 --- a/tumblr/library/tumblroauth.php +++ b/tumblr/library/tumblroauth.php @@ -141,6 +141,13 @@ class TumblrOAuth return ''; } + /** + * OAuth get from a given url with given parameters + * + * @param string $url + * @param array $parameters + * @return stdClass + */ public function get(string $url, array $parameters = []): stdClass { if (!empty($parameters)) { @@ -157,6 +164,13 @@ class TumblrOAuth return $this->formatResponse($response); } + /** + * OAuth Post to a given url with given parameters + * + * @param string $url + * @param array $parameter + * @return stdClass + */ public function post(string $url, array $parameter): stdClass { try { @@ -169,6 +183,12 @@ class TumblrOAuth return $this->formatResponse($response); } + /** + * Convert the body in the given response to a class + * + * @param ResponseInterface|null $response + * @return stdClass + */ private function formatResponse(ResponseInterface $response = null): stdClass { if (!is_null($response)) { diff --git a/tumblr/tumblr.php b/tumblr/tumblr.php index 88ae748bf..6ee9dca6b 100644 --- a/tumblr/tumblr.php +++ b/tumblr/tumblr.php @@ -264,6 +264,50 @@ function tumblr_settings_post(array &$b) } } +function tumblr_cron() +{ + $last = DI::keyValue()->get('tumblr_last_poll'); + + $poll_interval = intval(DI::config()->get('tumblr', 'poll_interval')); + if (!$poll_interval) { + $poll_interval = TUMBLR_DEFAULT_POLL_INTERVAL; + } + + if ($last) { + $next = $last + ($poll_interval * 60); + if ($next > time()) { + Logger::notice('poll intervall not reached'); + return; + } + } + Logger::notice('cron_start'); + + $abandon_days = intval(DI::config()->get('system', 'account_abandon_days')); + if ($abandon_days < 1) { + $abandon_days = 0; + } + + $abandon_limit = date(DateTimeFormat::MYSQL, time() - $abandon_days * 86400); + + $pconfigs = DBA::selectToArray('pconfig', [], ['cat' => 'tumblr', 'k' => 'import', 'v' => true]); + foreach ($pconfigs as $pconfig) { + if ($abandon_days != 0) { + if (!DBA::exists('user', ["`uid` = ? AND `login_date` >= ?", $pconfig['uid'], $abandon_limit])) { + Logger::notice('abandoned account: timeline from user will not be imported', ['user' => $pconfig['uid']]); + continue; + } + } + + Logger::notice('importing timeline - start', ['user' => $pconfig['uid']]); + tumblr_fetch_dashboard($pconfig['uid']); + Logger::notice('importing timeline - done', ['user' => $pconfig['uid']]); + } + + Logger::notice('cron_end'); + + DI::keyValue()->set('tumblr_last_poll', time()); +} + function tumblr_hook_fork(array &$b) { if ($b['name'] != 'notifier_normal') { @@ -388,10 +432,13 @@ function tumblr_send(array &$b) return; } - if (tumblr_send_npf($b)) { - return; + if (!tumblr_send_npf($b)) { + tumblr_send_legacy($b); } +} +function tumblr_send_legacy(array $b) +{ $connection = tumblr_connection($b['uid']); if (empty($connection)) { return; @@ -472,27 +519,11 @@ function tumblr_send(array &$b) if ($result->meta->status < 400) { Logger::info('Success (legacy)', ['blog' => $page, 'meta' => $result->meta, 'response' => $result->response]); - return true; } else { Logger::notice('Error posting blog (legacy)', ['blog' => $page, 'meta' => $result->meta, 'response' => $result->response, 'errors' => $result->errors, 'params' => $params]); - return false; } } -function tumblr_get_post_from_uri(string $uri): array -{ - $parts = explode(':', $uri); - if (($parts[0] != 'tumblr') || empty($parts[2])) { - return []; - } - - $post ['id'] = $parts[2]; - $post['reblog_key'] = $parts[3] ?? ''; - - $post['reblog_key'] = str_replace('@t', '', $post['reblog_key']); // Temp - return $post; -} - function tumblr_send_npf(array $post): bool { $page = tumblr_get_page($post['uid']); @@ -529,156 +560,26 @@ function tumblr_send_npf(array $post): bool } } -function tumblr_cron() +function tumblr_get_post_from_uri(string $uri): array { - $last = DI::keyValue()->get('tumblr_last_poll'); - - $poll_interval = intval(DI::config()->get('tumblr', 'poll_interval')); - if (!$poll_interval) { - $poll_interval = TUMBLR_DEFAULT_POLL_INTERVAL; - } - - if ($last) { - $next = $last + ($poll_interval * 60); - if ($next > time()) { - Logger::notice('poll intervall not reached'); - return; - } - } - Logger::notice('cron_start'); - - $abandon_days = intval(DI::config()->get('system', 'account_abandon_days')); - if ($abandon_days < 1) { - $abandon_days = 0; - } - - $abandon_limit = date(DateTimeFormat::MYSQL, time() - $abandon_days * 86400); - - $pconfigs = DBA::selectToArray('pconfig', [], ['cat' => 'tumblr', 'k' => 'import', 'v' => true]); - foreach ($pconfigs as $pconfig) { - if ($abandon_days != 0) { - if (!DBA::exists('user', ["`uid` = ? AND `login_date` >= ?", $pconfig['uid'], $abandon_limit])) { - Logger::notice('abandoned account: timeline from user will not be imported', ['user' => $pconfig['uid']]); - continue; - } - } - - Logger::notice('importing timeline - start', ['user' => $pconfig['uid']]); - tumblr_fetch_dashboard($pconfig['uid']); - Logger::notice('importing timeline - done', ['user' => $pconfig['uid']]); - } - - Logger::notice('cron_end'); - - DI::keyValue()->set('tumblr_last_poll', time()); -} - -function tumblr_add_npf_data(string $html, string $plink): string -{ - $doc = new DOMDocument(); - - $doc->formatOutput = true; - @$doc->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8')); - $xpath = new DomXPath($doc); - $list = $xpath->query('//p[@class="npf_link"]'); - foreach ($list as $node) { - $data = tumblr_get_npf_data($node); - if (empty($data)) { - continue; - } - - tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); - } - - $list = $xpath->query('//div[@data-npf]'); - foreach ($list as $node) { - $data = tumblr_get_npf_data($node); - if (empty($data)) { - continue; - } - - tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); - } - - $list = $xpath->query('//figure[@data-provider="youtube"]'); - foreach ($list as $node) { - $attributes = tumblr_get_attributes($node); - if (empty($attributes['data-url'])) { - continue; - } - tumblr_replace_with_npf($doc, $node, '[youtube]' . $attributes['data-url'] . '[/youtube]'); - } - - $list = $xpath->query('//figure[@data-npf]'); - foreach ($list as $node) { - $data = tumblr_get_npf_data($node); - if (empty($data)) { - continue; - } - tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); - } - - return $doc->saveHTML(); -} - -function tumblr_get_type_replacement(array $data, string $plink): string -{ - switch ($data['type']) { - case 'poll': - $body = '[p][url=' . $plink. ']'. $data['question'] . '[/url][/p][ul]'; - foreach ($data['answers'] as $answer) { - $body .= '[li]' . $answer['answer_text'] . '[/li]'; - } - $body .= '[/ul]'; - break; - - case 'link': - $body = PageInfo::getFooterFromUrl(str_replace('https://href.li/?', '', $data['url'])); - break; - - case 'video': - if (!empty($data['url']) && ($data['provider'] == 'tumblr')) { - $body = '[video]' . $data['url'] . '[/video]'; - break; - } - - default: - Logger::notice('Unknown type', ['type' => $data['type'], 'data' => $data, 'plink' => $plink]); - $body = ''; - } - - return $body; -} - -function tumblr_get_attributes($node): array -{ - $attributes = []; - foreach ($node->attributes as $key => $attribute) { - $attributes[$key] = trim($attribute->value); - } - return $attributes; -} - -function tumblr_get_npf_data(DOMNode $node): array -{ - $attributes = tumblr_get_attributes($node); - if (empty($attributes['data-npf'])) { + $parts = explode(':', $uri); + if (($parts[0] != 'tumblr') || empty($parts[2])) { return []; } + + $post ['id'] = $parts[2]; + $post['reblog_key'] = $parts[3] ?? ''; - return json_decode($attributes['data-npf'], true); -} - -function tumblr_replace_with_npf(DOMDocument $doc, DOMNode $node, string $replacement) -{ - if (empty($replacement)) { - return; - } - $replace = $doc->createTextNode($replacement); - $node->parentNode->insertBefore($replace, $node); - $node->parentNode->removeChild($node); + $post['reblog_key'] = str_replace('@t', '', $post['reblog_key']); // Temp + return $post; } +/** + * Fetch the dashboard (timeline) for the given user + * + * @param integer $uid + * @return void + */ function tumblr_fetch_dashboard(int $uid) { $page = tumblr_get_page($uid); @@ -735,6 +636,14 @@ function tumblr_fetch_dashboard(int $uid) } } +/** + * Sets the initial data for the item array + * + * @param stdClass $post + * @param string $uri + * @param integer $uid + * @return array + */ function tumblr_get_header(stdClass $post, string $uri, int $uid): array { $contact = tumblr_get_contact($post->blog, $uid); @@ -760,6 +669,13 @@ function tumblr_get_header(stdClass $post, string $uri, int $uid): array return $item; } +/** + * Set the body according the given content type + * + * @param array $item + * @param stdClass $post + * @return array + */ function tumblr_get_content(array $item, stdClass $post): array { switch ($post->type) { @@ -846,7 +762,120 @@ function tumblr_get_content(array $item, stdClass $post): array return $item; } -function tumblr_get_contact(stdClass $blog, int $uid) +function tumblr_add_npf_data(string $html, string $plink): string +{ + $doc = new DOMDocument(); + + $doc->formatOutput = true; + @$doc->loadHTML(mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8')); + $xpath = new DomXPath($doc); + $list = $xpath->query('//p[@class="npf_link"]'); + foreach ($list as $node) { + $data = tumblr_get_npf_data($node); + if (empty($data)) { + continue; + } + + tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); + } + + $list = $xpath->query('//div[@data-npf]'); + foreach ($list as $node) { + $data = tumblr_get_npf_data($node); + if (empty($data)) { + continue; + } + + tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); + } + + $list = $xpath->query('//figure[@data-provider="youtube"]'); + foreach ($list as $node) { + $attributes = tumblr_get_attributes($node); + if (empty($attributes['data-url'])) { + continue; + } + tumblr_replace_with_npf($doc, $node, '[youtube]' . $attributes['data-url'] . '[/youtube]'); + } + + $list = $xpath->query('//figure[@data-npf]'); + foreach ($list as $node) { + $data = tumblr_get_npf_data($node); + if (empty($data)) { + continue; + } + tumblr_replace_with_npf($doc, $node, tumblr_get_type_replacement($data, $plink)); + } + + return $doc->saveHTML(); +} + +function tumblr_replace_with_npf(DOMDocument $doc, DOMNode $node, string $replacement) +{ + if (empty($replacement)) { + return; + } + $replace = $doc->createTextNode($replacement); + $node->parentNode->insertBefore($replace, $node); + $node->parentNode->removeChild($node); +} + +function tumblr_get_npf_data(DOMNode $node): array +{ + $attributes = tumblr_get_attributes($node); + if (empty($attributes['data-npf'])) { + return []; + } + + return json_decode($attributes['data-npf'], true); +} + +function tumblr_get_attributes($node): array +{ + $attributes = []; + foreach ($node->attributes as $key => $attribute) { + $attributes[$key] = trim($attribute->value); + } + return $attributes; +} + +function tumblr_get_type_replacement(array $data, string $plink): string +{ + switch ($data['type']) { + case 'poll': + $body = '[p][url=' . $plink. ']'. $data['question'] . '[/url][/p][ul]'; + foreach ($data['answers'] as $answer) { + $body .= '[li]' . $answer['answer_text'] . '[/li]'; + } + $body .= '[/ul]'; + break; + + case 'link': + $body = PageInfo::getFooterFromUrl(str_replace('https://href.li/?', '', $data['url'])); + break; + + case 'video': + if (!empty($data['url']) && ($data['provider'] == 'tumblr')) { + $body = '[video]' . $data['url'] . '[/video]'; + break; + } + + default: + Logger::notice('Unknown type', ['type' => $data['type'], 'data' => $data, 'plink' => $plink]); + $body = ''; + } + + return $body; +} + +/** + * Get a contact array for the given blog + * + * @param stdClass $blog + * @param integer $uid + * @return array + */ +function tumblr_get_contact(stdClass $blog, int $uid): array { $condition = ['network' => Protocol::TUMBLR, 'uid' => $uid, 'poll' => 'tumblr::' . $blog->uuid]; $contact = Contact::selectFirst([], $condition); @@ -873,6 +902,13 @@ function tumblr_get_contact(stdClass $blog, int $uid) return Contact::getById($cid); } +/** + * Create a new contact + * + * @param stdClass $blog + * @param integer $uid + * @return void + */ function tumblr_insert_contact(stdClass $blog, int $uid) { $baseurl = 'https://tumblr.com'; @@ -900,6 +936,15 @@ function tumblr_insert_contact(stdClass $blog, int $uid) return Contact::insert($fields); } +/** + * Updates the given contact for the given user and proviced contact ids + * + * @param stdClass $blog + * @param integer $uid + * @param integer $cid + * @param integer $pcid + * @return void + */ function tumblr_update_contact(stdClass $blog, int $uid, int $cid, int $pcid) { $connection = tumblr_connection($uid); @@ -948,22 +993,13 @@ function tumblr_update_contact(stdClass $blog, int $uid, int $cid, int $pcid) Contact::update($fields, ['id' => $pcid]); } -function tumblr_connection(int $uid): ?TumblrOAuth -{ - $oauth_token = DI::pConfig()->get($uid, 'tumblr', 'oauth_token'); - $oauth_token_secret = DI::pConfig()->get($uid, 'tumblr', 'oauth_token_secret'); - - $consumer_key = DI::config()->get('tumblr', 'consumer_key'); - $consumer_secret = DI::config()->get('tumblr', 'consumer_secret'); - - if (!$consumer_key || !$consumer_secret || !$oauth_token || !$oauth_token_secret) { - Logger::notice('Missing data, connection is not established', ['uid' => $uid]); - return null; - } - - return new TumblrOAuth($consumer_key, $consumer_secret, $oauth_token, $oauth_token_secret); -} - +/** + * Get the default page for posting. Detects the value if not provided or has got a bad value. + * + * @param integer $uid + * @param array $blogs + * @return string + */ function tumblr_get_page(int $uid, array $blogs = []): string { $page = DI::pConfig()->get($uid, 'tumblr', 'page'); @@ -985,6 +1021,12 @@ function tumblr_get_page(int $uid, array $blogs = []): string return ''; } +/** + * Get an array of blogs for the given user + * + * @param integer $uid + * @return array + */ function tumblr_get_blogs(int $uid): array { $connection = tumblr_connection($uid); @@ -1003,4 +1045,26 @@ function tumblr_get_blogs(int $uid): array $blogs[$blog->uuid] = $blog->name; } return $blogs; +} + +/** + * Creates a OAuth connection for the given user + * + * @param integer $uid + * @return TumblrOAuth|null + */ +function tumblr_connection(int $uid): ?TumblrOAuth +{ + $oauth_token = DI::pConfig()->get($uid, 'tumblr', 'oauth_token'); + $oauth_token_secret = DI::pConfig()->get($uid, 'tumblr', 'oauth_token_secret'); + + $consumer_key = DI::config()->get('tumblr', 'consumer_key'); + $consumer_secret = DI::config()->get('tumblr', 'consumer_secret'); + + if (!$consumer_key || !$consumer_secret || !$oauth_token || !$oauth_token_secret) { + Logger::notice('Missing data, connection is not established', ['uid' => $uid]); + return null; + } + + return new TumblrOAuth($consumer_key, $consumer_secret, $oauth_token, $oauth_token_secret); } \ No newline at end of file