From 3e5ac1ad5da5bdbfaa1f048cd8c48cf19f8f2c60 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:13 +0200 Subject: [PATCH 01/21] Add guzzlehttp/guzzle as composer requirement --- composer.json | 1 + composer.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 66ac9361d..2dd5dec7b 100644 --- a/composer.json +++ b/composer.json @@ -31,6 +31,7 @@ "divineomega/password_exposed": "^2.8", "ezyang/htmlpurifier": "^4.7", "friendica/json-ld": "^1.0", + "guzzlehttp/guzzle": "^6.5", "league/html-to-markdown": "^4.8", "level-2/dice": "^4", "lightopenid/lightopenid": "dev-master", diff --git a/composer.lock b/composer.lock index facde6d0b..5e8f1a20a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6dda88f5629c38f3d07b31e13ded57aa", + "content-hash": "7d6dee6e449da931e8fe209e61b2e78e", "packages": [ { "name": "asika/simple-console", From 7009d90addaa72028a3664625a33c6f786232068 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:13 +0200 Subject: [PATCH 02/21] Introduce IHTTPResult Interface as abstraction for CurlResult --- src/Network/CurlResult.php | 72 ++++++---------------------- src/Network/IHTTPResult.php | 93 +++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 57 deletions(-) create mode 100644 src/Network/IHTTPResult.php diff --git a/src/Network/CurlResult.php b/src/Network/CurlResult.php index 017f2c559..547bb4d28 100644 --- a/src/Network/CurlResult.php +++ b/src/Network/CurlResult.php @@ -29,7 +29,7 @@ use Friendica\Util\Network; /** * A content class for Curl call results */ -class CurlResult +class CurlResult implements IHTTPResult { /** * @var int HTTP return code or 0 if timeout or failure @@ -229,33 +229,19 @@ class CurlResult } } - /** - * Gets the Curl Code - * - * @return string The Curl Code - */ + /** {@inheritDoc} */ public function getReturnCode() { return $this->returnCode; } - /** - * Returns the Curl Content Type - * - * @return string the Curl Content Type - */ + /** {@inheritDoc} */ public function getContentType() { return $this->contentType; } - /** - * Returns the Curl headers - * - * @param string $field optional header field. Return all fields if empty - * - * @return string the Curl headers or the specified content of the header variable - */ + /** {@inheritDoc} */ public function getHeader(string $field = '') { if (empty($field)) { @@ -273,13 +259,7 @@ class CurlResult return ''; } - /** - * Check if a specified header exists - * - * @param string $field header field - * - * @return boolean "true" if header exists - */ + /** {@inheritDoc} */ public function inHeader(string $field) { $field = strtolower(trim($field)); @@ -289,11 +269,7 @@ class CurlResult return array_key_exists($field, $headers); } - /** - * Returns the Curl headers as an associated array - * - * @return array associated header array - */ + /** {@inheritDoc} */ public function getHeaderArray() { if (!empty($this->header_fields)) { @@ -313,73 +289,55 @@ class CurlResult return $this->header_fields; } - /** - * @return bool - */ + /** {@inheritDoc} */ public function isSuccess() { return $this->isSuccess; } - /** - * @return string - */ + /** {@inheritDoc} */ public function getUrl() { return $this->url; } - /** - * @return string - */ + /** {@inheritDoc} */ public function getRedirectUrl() { return $this->redirectUrl; } - /** - * @return string - */ + /** {@inheritDoc} */ public function getBody() { return $this->body; } - /** - * @return array - */ + /** {@inheritDoc} */ public function getInfo() { return $this->info; } - /** - * @return bool - */ + /** {@inheritDoc} */ public function isRedirectUrl() { return $this->isRedirectUrl; } - /** - * @return int - */ + /** {@inheritDoc} */ public function getErrorNumber() { return $this->errorNumber; } - /** - * @return string - */ + /** {@inheritDoc} */ public function getError() { return $this->error; } - /** - * @return bool - */ + /** {@inheritDoc} */ public function isTimeout() { return $this->isTimeout; diff --git a/src/Network/IHTTPResult.php b/src/Network/IHTTPResult.php new file mode 100644 index 000000000..be190c80c --- /dev/null +++ b/src/Network/IHTTPResult.php @@ -0,0 +1,93 @@ + Date: Fri, 20 Aug 2021 19:48:14 +0200 Subject: [PATCH 03/21] Replace IHTTPResult for CurlResult usages --- src/Model/GServer.php | 23 ++++++++++++----------- src/Network/CurlResult.php | 2 +- src/Network/IHTTPRequest.php | 6 +++--- tests/src/Core/InstallerTest.php | 26 +++++++++++++------------- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 7e1d178dd..eb99b1bbc 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -32,7 +32,7 @@ use Friendica\Database\Database; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Module\Register; -use Friendica\Network\CurlResult; +use Friendica\Network\IHTTPResult; use Friendica\Protocol\Relay; use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; @@ -171,7 +171,7 @@ class GServer if (($now - $contact_time) < (60 * 60 * 24)) { return DateTimeFormat::utc('now +1 day'); } - + // If the last contact was less than a week before then try again in a week if (($now - $contact_time) < (60 * 60 * 24 * 7)) { return DateTimeFormat::utc('now +1 week'); @@ -671,18 +671,19 @@ class GServer /** * Detect server type by using the nodeinfo data * - * @param string $url address of the server - * @param CurlResult $curlResult + * @param string $url address of the server + * @param IHTTPResult $httpResult + * * @return array Server data * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - private static function fetchNodeinfo(string $url, CurlResult $curlResult) + private static function fetchNodeinfo(string $url, IHTTPResult $httpResult) { - if (!$curlResult->isSuccess()) { + if (!$httpResult->isSuccess()) { return []; } - $nodeinfo = json_decode($curlResult->getBody(), true); + $nodeinfo = json_decode($httpResult->getBody(), true); if (!is_array($nodeinfo) || empty($nodeinfo['links'])) { return []; @@ -1748,8 +1749,8 @@ class GServer * * @param int $gsid Server id * @param int $protocol Protocol id - * @return void - * @throws Exception + * @return void + * @throws Exception */ public static function setProtocol(int $gsid, int $protocol) { @@ -1808,8 +1809,8 @@ class GServer * Fetch the protocol of the given server * * @param int $gsid Server id - * @return int - * @throws Exception + * @return int + * @throws Exception */ public static function getProtocol(int $gsid) { diff --git a/src/Network/CurlResult.php b/src/Network/CurlResult.php index 547bb4d28..6fb0153b6 100644 --- a/src/Network/CurlResult.php +++ b/src/Network/CurlResult.php @@ -101,7 +101,7 @@ class CurlResult implements IHTTPResult * * @param string $url optional URL * - * @return CurlResult a CURL with error response + * @return IHTTPResult a CURL with error response * @throws InternalServerErrorException */ public static function createErrorCurl($url = '') diff --git a/src/Network/IHTTPRequest.php b/src/Network/IHTTPRequest.php index e2ace44db..efe9f5f8b 100644 --- a/src/Network/IHTTPRequest.php +++ b/src/Network/IHTTPRequest.php @@ -52,7 +52,7 @@ interface IHTTPRequest * @param string $accept_content supply Accept: header with 'accept_content' as the value * @param string $cookiejar Path to cookie jar file * - * @return CurlResult With all relevant information, 'body' contains the actual fetched content. + * @return IHTTPResult With all relevant information, 'body' contains the actual fetched content. */ public function fetchFull(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = ''); @@ -80,7 +80,7 @@ interface IHTTPRequest * 'cookiejar' => path to cookie jar file * 'header' => header array * - * @return CurlResult + * @return IHTTPResult */ public function get(string $url, array $opts = []); @@ -92,7 +92,7 @@ interface IHTTPRequest * @param array $headers HTTP headers * @param int $timeout The timeout in seconds, default system config value or 60 seconds * - * @return CurlResult The content + * @return IHTTPResult The content */ public function post(string $url, $params, array $headers = [], int $timeout = 0); diff --git a/tests/src/Core/InstallerTest.php b/tests/src/Core/InstallerTest.php index 8cd1a3fad..8c72b7b2b 100644 --- a/tests/src/Core/InstallerTest.php +++ b/tests/src/Core/InstallerTest.php @@ -25,7 +25,7 @@ namespace Friendica\Core; use Dice\Dice; use Friendica\Core\Config\Cache; use Friendica\DI; -use Friendica\Network\CurlResult; +use Friendica\Network\IHTTPResult; use Friendica\Network\IHTTPRequest; use Friendica\Test\MockedTest; use Friendica\Test\Util\VFSTrait; @@ -319,14 +319,14 @@ class InstallerTest extends MockedTest $this->l10nMock->shouldReceive('t')->andReturnUsing(function ($args) { return $args; }); // Mocking the CURL Response - $curlResult = Mockery::mock(CurlResult::class); - $curlResult + $IHTTPResult = Mockery::mock(IHTTPResult::class); + $IHTTPResult ->shouldReceive('getReturnCode') ->andReturn('404'); - $curlResult + $IHTTPResult ->shouldReceive('getRedirectUrl') ->andReturn(''); - $curlResult + $IHTTPResult ->shouldReceive('getError') ->andReturn('test Error'); @@ -335,11 +335,11 @@ class InstallerTest extends MockedTest $networkMock ->shouldReceive('fetchFull') ->with('https://test/install/testrewrite') - ->andReturn($curlResult); + ->andReturn($IHTTPResult); $networkMock ->shouldReceive('fetchFull') ->with('http://test/install/testrewrite') - ->andReturn($curlResult); + ->andReturn($IHTTPResult); $this->dice->shouldReceive('create') ->with(IHTTPRequest::class) @@ -366,14 +366,14 @@ class InstallerTest extends MockedTest $this->l10nMock->shouldReceive('t')->andReturnUsing(function ($args) { return $args; }); // Mocking the failed CURL Response - $curlResultF = Mockery::mock(CurlResult::class); - $curlResultF + $IHTTPResultF = Mockery::mock(IHTTPResult::class); + $IHTTPResultF ->shouldReceive('getReturnCode') ->andReturn('404'); // Mocking the working CURL Response - $curlResultW = Mockery::mock(CurlResult::class); - $curlResultW + $IHTTPResultW = Mockery::mock(IHTTPResult::class); + $IHTTPResultW ->shouldReceive('getReturnCode') ->andReturn('204'); @@ -382,11 +382,11 @@ class InstallerTest extends MockedTest $networkMock ->shouldReceive('fetchFull') ->with('https://test/install/testrewrite') - ->andReturn($curlResultF); + ->andReturn($IHTTPResultF); $networkMock ->shouldReceive('fetchFull') ->with('http://test/install/testrewrite') - ->andReturn($curlResultW); + ->andReturn($IHTTPResultW); $this->dice->shouldReceive('create') ->with(IHTTPRequest::class) From a60ca4a1cf11376db7cc3eeb0f9366a746c40489 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:14 +0200 Subject: [PATCH 04/21] Use Guzzle for HTTPRequest and Result --- src/Network/GuzzleResponse.php | 152 +++++++++++++++++++++++++++++++++ src/Network/HTTPRequest.php | 128 ++++++++++++++------------- 2 files changed, 222 insertions(+), 58 deletions(-) create mode 100644 src/Network/GuzzleResponse.php diff --git a/src/Network/GuzzleResponse.php b/src/Network/GuzzleResponse.php new file mode 100644 index 000000000..c6cab7c9e --- /dev/null +++ b/src/Network/GuzzleResponse.php @@ -0,0 +1,152 @@ +. + * + */ + +namespace Friendica\Network; + +use Friendica\Core\Logger; +use Friendica\Core\System; +use Friendica\Network\HTTPException\NotImplementedException; +use GuzzleHttp\Psr7\Response; +use Psr\Http\Message\ResponseInterface; + +/** + * A content wrapper class for Guzzle call results + */ +class GuzzleResponse extends Response implements IHTTPResult, ResponseInterface +{ + /** @var string The URL */ + private $url; + /** @var boolean */ + private $isTimeout; + /** @var boolean */ + private $isSuccess; + /** + * @var int the error number or 0 (zero) if no error + */ + private $errorNumber; + + /** + * @var string the error message or '' (the empty string) if no + */ + private $error; + + public function __construct(ResponseInterface $response, string $url, $errorNumber = 0, $error = '') + { + parent::__construct($response->getStatusCode(), $response->getHeaders(), $response->getBody(), $response->getProtocolVersion(), $response->getReasonPhrase()); + $this->url = $url; + $this->error = $error; + $this->errorNumber = $errorNumber; + + $this->checkSuccess(); + } + + private function checkSuccess() + { + $this->isSuccess = ($this->getStatusCode() >= 200 && $this->getStatusCode() <= 299) || $this->errorNumber == 0; + + // Everything higher or equal 400 is not a success + if ($this->getReturnCode() >= 400) { + $this->isSuccess = false; + } + + if (!$this->isSuccess) { + Logger::notice('http error', ['url' => $this->url, 'code' => $this->getReturnCode(), 'error' => $this->error, 'callstack' => System::callstack(20)]); + Logger::debug('debug', ['info' => $this->getHeaders()]); + } + + if (!$this->isSuccess && $this->errorNumber == CURLE_OPERATION_TIMEDOUT) { + $this->isTimeout = true; + } else { + $this->isTimeout = false; + } + } + + /** {@inheritDoc} */ + public function getReturnCode() + { + return $this->getStatusCode(); + } + + /** {@inheritDoc} */ + public function getContentType() + { + return $this->getHeader('Content-Type'); + } + + /** {@inheritDoc} */ + public function inHeader(string $field) + { + return $this->hasHeader($field); + } + + /** {@inheritDoc} */ + public function getHeaderArray() + { + return $this->getHeaders(); + } + + /** {@inheritDoc} */ + public function isSuccess() + { + return $this->isSuccess; + } + + /** {@inheritDoc} */ + public function getUrl() + { + return $this->url; + } + + /** {@inheritDoc} */ + public function getRedirectUrl() + { + return $this->url; + } + + public function getInfo() + { + // TODO: Implement getInfo() method. + } + + /** {@inheritDoc} */ + public function isRedirectUrl() + { + throw new NotImplementedException(); + } + + /** {@inheritDoc} */ + public function getErrorNumber() + { + return $this->errorNumber; + } + + /** {@inheritDoc} */ + public function getError() + { + return $this->error; + } + + /** {@inheritDoc} */ + public function isTimeout() + { + return $this->isTimeout; + } +} diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index daf84dc9a..523efc698 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -28,6 +28,11 @@ use Friendica\Core\Config\IConfig; use Friendica\Core\System; use Friendica\Util\Network; use Friendica\Util\Profiler; +use GuzzleHttp\Client; +use GuzzleHttp\Exception\RequestException; +use Psr\Http\Message\RequestInterface; +use Psr\Http\Message\ResponseInterface; +use Psr\Http\Message\UriInterface; use Psr\Log\LoggerInterface; /** @@ -65,12 +70,8 @@ class HTTPRequest implements IHTTPRequest /** * {@inheritDoc} - * - * @param int $redirects The recursion counter for internal use - default 0 - * - * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public function get(string $url, array $opts = [], &$redirects = 0) + public function get(string $url, bool $binary = false, array $opts = []) { $this->profiler->startRecording('network'); @@ -103,120 +104,131 @@ class HTTPRequest implements IHTTPRequest return CurlResult::createErrorCurl($url); } - $ch = @curl_init($url); + $curlOptions = []; - if (($redirects > 8) || (!$ch)) { - $this->profiler->stopRecording(); - return CurlResult::createErrorCurl($url); - } - - @curl_setopt($ch, CURLOPT_HEADER, true); + $curlOptions[CURLOPT_HEADER] = true; if (!empty($opts['cookiejar'])) { - curl_setopt($ch, CURLOPT_COOKIEJAR, $opts["cookiejar"]); - curl_setopt($ch, CURLOPT_COOKIEFILE, $opts["cookiejar"]); + $curlOptions[CURLOPT_COOKIEJAR] = $opts["cookiejar"]; + $curlOptions[CURLOPT_COOKIEFILE] = $opts["cookiejar"]; } // These settings aren't needed. We're following the location already. - // @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - // @curl_setopt($ch, CURLOPT_MAXREDIRS, 5); + // $curlOptions[CURLOPT_FOLLOWLOCATION] =true; + // $curlOptions[CURLOPT_MAXREDIRS] = 5; if (!empty($opts['accept_content'])) { - curl_setopt( - $ch, - CURLOPT_HTTPHEADER, - ['Accept: ' . $opts['accept_content']] - ); + if (empty($curlOptions[CURLOPT_HTTPHEADER])) { + $curlOptions[CURLOPT_HTTPHEADER] = []; + } + array_push($curlOptions[CURLOPT_HTTPHEADER], 'Accept: ' . $opts['accept_content']); } if (!empty($opts['header'])) { - curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['header']); + if (empty($curlOptions[CURLOPT_HTTPHEADER])) { + $curlOptions[CURLOPT_HTTPHEADER] = []; + } + $curlOptions[CURLOPT_HTTPHEADER] = array_merge($opts['header'], $curlOptions[CURLOPT_HTTPHEADER]); } - @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - @curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent()); + $curlOptions[CURLOPT_RETURNTRANSFER] = true; + $curlOptions[CURLOPT_USERAGENT] = $this->getUserAgent(); $range = intval($this->config->get('system', 'curl_range_bytes', 0)); if ($range > 0) { - @curl_setopt($ch, CURLOPT_RANGE, '0-' . $range); + $curlOptions[CURLOPT_RANGE] = '0-' . $range; } // Without this setting it seems as if some webservers send compressed content // This seems to confuse curl so that it shows this uncompressed. /// @todo We could possibly set this value to "gzip" or something similar - curl_setopt($ch, CURLOPT_ENCODING, ''); + $curlOptions[CURLOPT_ENCODING] = ''; if (!empty($opts['headers'])) { $this->logger->notice('Wrong option \'headers\' used.'); - @curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['headers']); + if (empty($curlOptions[CURLOPT_HTTPHEADER])) { + $curlOptions[CURLOPT_HTTPHEADER] = []; + } + $curlOptions[CURLOPT_HTTPHEADER] = array_merge($opts['headers'], $curlOptions[CURLOPT_HTTPHEADER]); } if (!empty($opts['nobody'])) { - @curl_setopt($ch, CURLOPT_NOBODY, $opts['nobody']); + $curlOptions[CURLOPT_NOBODY] = $opts['nobody']; } - @curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); + $curlOptions[CURLOPT_CONNECTTIMEOUT] = 10; if (!empty($opts['timeout'])) { - @curl_setopt($ch, CURLOPT_TIMEOUT, $opts['timeout']); + $curlOptions[CURLOPT_TIMEOUT] = $opts['timeout']; } else { $curl_time = $this->config->get('system', 'curl_timeout', 60); - @curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time)); + $curlOptions[CURLOPT_TIMEOUT] = intval($curl_time); } // by default we will allow self-signed certs // but you can override this $check_cert = $this->config->get('system', 'verifyssl'); - @curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false)); + $curlOptions[CURLOPT_SSL_VERIFYPEER] = ($check_cert) ? true : false; if ($check_cert) { - @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); + $curlOptions[CURLOPT_SSL_VERIFYHOST] = 2; } $proxy = $this->config->get('system', 'proxy'); if (!empty($proxy)) { - @curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1); - @curl_setopt($ch, CURLOPT_PROXY, $proxy); + $curlOptions[CURLOPT_HTTPPROXYTUNNEL] = 1; + $curlOptions[CURLOPT_PROXY] = $proxy; $proxyuser = $this->config->get('system', 'proxyuser'); if (!empty($proxyuser)) { - @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser); + $curlOptions[CURLOPT_PROXYUSERPWD] = $proxyuser; } } if ($this->config->get('system', 'ipv4_resolve', false)) { - curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + $curlOptions[CURLOPT_IPRESOLVE] = CURL_IPRESOLVE_V4; } - $s = @curl_exec($ch); - $curl_info = @curl_getinfo($ch); - - // Special treatment for HTTP Code 416 - // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/416 - if (($curl_info['http_code'] == 416) && ($range > 0)) { - @curl_setopt($ch, CURLOPT_RANGE, ''); - $s = @curl_exec($ch); - $curl_info = @curl_getinfo($ch); + if ($binary) { + $curlOptions[CURLOPT_BINARYTRANSFER] = 1; } - $curlResponse = new CurlResult($url, $s, $curl_info, curl_errno($ch), curl_error($ch)); + $onRedirect = function( + RequestInterface $request, + ResponseInterface $response, + UriInterface $uri + ) { + $this->logger->notice('Curl redirect.', ['url' => $request->getUri(), 'to' => $uri]); + }; - if (!Network::isRedirectBlocked($url) && $curlResponse->isRedirectUrl()) { - $redirects++; - $this->logger->notice('Curl redirect.', ['url' => $url, 'to' => $curlResponse->getRedirectUrl()]); - @curl_close($ch); + $client = new Client([ + 'allow_redirect' => [ + 'max' => 8, + 'on_redirect' => $onRedirect, + 'track_redirect' => true, + 'strict' => true, + 'referer' => true, + ], + 'on_headers' => $onHeaders, + 'sink' => tempnam(get_temppath(), 'guzzle'), + 'curl' => $curlOptions + ]); + + try { + $response = $client->get($url); + return new GuzzleResponse($response, $url); + } catch (RequestException $exception) { + if ($exception->hasResponse()) { + return new GuzzleResponse($exception->getResponse(), $url, $exception->getCode(), $exception->getMessage()); + } else { + return new GuzzleResponse(null, $url, $exception->getCode(), $exception->getMessage()); + } + } finally { $this->profiler->stopRecording(); - return $this->get($curlResponse->getRedirectUrl(), $opts, $redirects); } - - @curl_close($ch); - - $this->profiler->stopRecording(); - - return $curlResponse; } /** From dee1899628380998d3a284e41bfc49ce0737dd2f Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:14 +0200 Subject: [PATCH 05/21] Fix IHTTPResult::getHeader/s() - Split functionality "getHeader()" and "getHeaders()" analog to IMessageInterface::getHeader/s() - Fix functionality at various places - Adapt CurlResultTest --- src/Network/CurlResult.php | 18 ++++++++++++------ src/Network/HTTPRequest.php | 3 +-- src/Network/IHTTPResult.php | 22 +++++++++++++++++++--- src/Protocol/DFRN.php | 2 +- src/Protocol/OStatus.php | 6 ++++-- src/Protocol/Salmon.php | 2 +- src/Util/ParseUrl.php | 3 +-- tests/datasets/curl/about.head.php | 20 ++++++++++++++++++++ tests/datasets/curl/about.redirect.php | 21 +++++++++++++++++++++ tests/src/Network/CurlResultTest.php | 14 +++++++++----- 10 files changed, 89 insertions(+), 22 deletions(-) create mode 100644 tests/datasets/curl/about.head.php create mode 100644 tests/datasets/curl/about.redirect.php diff --git a/src/Network/CurlResult.php b/src/Network/CurlResult.php index 6fb0153b6..227d26140 100644 --- a/src/Network/CurlResult.php +++ b/src/Network/CurlResult.php @@ -242,23 +242,29 @@ class CurlResult implements IHTTPResult } /** {@inheritDoc} */ - public function getHeader(string $field = '') + public function getHeader($header) { - if (empty($field)) { - return $this->header; + if (empty($header)) { + return ''; } - $field = strtolower(trim($field)); + $header = strtolower(trim($header)); $headers = $this->getHeaderArray(); - if (isset($headers[$field])) { - return $headers[$field]; + if (isset($headers[$header])) { + return $headers[$header]; } return ''; } + /** {@inheritDoc} */ + public function getHeaders() + { + return $this->getHeaderArray(); + } + /** {@inheritDoc} */ public function inHeader(string $field) { diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 523efc698..72311ea19 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -493,8 +493,7 @@ class HTTPRequest implements IHTTPRequest 'timeout' => $timeout, 'accept_content' => $accept_content, 'cookiejar' => $cookiejar - ], - $redirects + ] ); } diff --git a/src/Network/IHTTPResult.php b/src/Network/IHTTPResult.php index be190c80c..5904bcfa3 100644 --- a/src/Network/IHTTPResult.php +++ b/src/Network/IHTTPResult.php @@ -2,6 +2,8 @@ namespace Friendica\Network; +use Psr\Http\Message\MessageInterface; + /** * Temporary class to map Friendica used variables based on PSR-7 HTTPResponse */ @@ -23,15 +25,25 @@ interface IHTTPResult /** * Returns the headers + * @see MessageInterface::getHeader() * - * @param string $field optional header field. Return all fields if empty + * @param string $header optional header field. Return all fields if empty * * @return string the headers or the specified content of the header variable */ - public function getHeader(string $field = ''); + public function getHeader($header); + + /** + * Returns all headers + * @see MessageInterface::getHeaders() + * + * @return string[][] + */ + public function getHeaders(); /** * Check if a specified header exists + * @see MessageInterface::hasHeader() * * @param string $field header field * @@ -41,8 +53,10 @@ interface IHTTPResult /** * Returns the headers as an associated array + * @see MessageInterface::getHeaders() + * @deprecated * - * @return array associated header array + * @return string[][] associated header array */ public function getHeaderArray(); @@ -62,6 +76,8 @@ interface IHTTPResult public function getRedirectUrl(); /** + * @see MessageInterface::getBody() + * * @return string */ public function getBody(); diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 832e2bde2..cd63291ee 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -985,7 +985,7 @@ class DFRN return -9; // timed out } - if (($curl_stat == 503) && (stristr($postResult->getHeader(), 'retry-after'))) { + if (($curl_stat == 503) && $postResult->inHeader('retry-after')) { return -10; } diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index e813cc250..8d818471b 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -735,7 +735,8 @@ class OStatus $xml = ''; - if (stristr($curlResult->getHeader(), 'Content-Type: application/atom+xml')) { + if ($curlResult->inHeader('Content-Type') && + stristr($curlResult->getHeader('Content-Type'), 'application/atom+xml')) { $xml = $curlResult->getBody(); } @@ -928,7 +929,8 @@ class OStatus $xml = ''; - if (stristr($curlResult->getHeader(), 'Content-Type: application/atom+xml')) { + if ($curlResult->inHeader('Content-Type') && + stristr($curlResult->getHeader('Content-Type'), 'application/atom+xml')) { Logger::log('Directly fetched XML for URI ' . $related_uri, Logger::DEBUG); $xml = $curlResult->getBody(); } diff --git a/src/Protocol/Salmon.php b/src/Protocol/Salmon.php index 81b1f7528..53367f6d0 100644 --- a/src/Protocol/Salmon.php +++ b/src/Protocol/Salmon.php @@ -215,7 +215,7 @@ class Salmon return -1; } - if (($return_code == 503) && (stristr($postResult->getHeader(), 'retry-after'))) { + if (($return_code == 503) && $postResult->inHeader('retry-after')) { return -1; } diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index d5c9f02ad..75210af0f 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -232,7 +232,6 @@ class ParseUrl } } - $header = $curlResult->getHeader(); $body = $curlResult->getBody(); if ($do_oembed) { @@ -273,7 +272,7 @@ class ParseUrl $charset = ''; // Look for a charset, first in headers // Expected form: Content-Type: text/html; charset=ISO-8859-4 - if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $header, $matches)) { + if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) { $charset = trim(trim(trim(array_pop($matches)), ';,')); } diff --git a/tests/datasets/curl/about.head.php b/tests/datasets/curl/about.head.php new file mode 100644 index 000000000..d0be0feb4 --- /dev/null +++ b/tests/datasets/curl/about.head.php @@ -0,0 +1,20 @@ + '', + 'date' => 'Thu, 11 Oct 2018 18:43:54 GMT', + 'content-type' => 'text/html; charset=utf-8', + 'vary' => 'Accept-Encoding', + 'server' => 'Mastodon', + 'x-frame-options' => 'SAMEORIGIN', + 'x-content-type-options' => 'nosniff', + 'x-xss-protection' => '1; mode=block', + 'etag' => 'W/"706e6c48957e1d46ecf9d7597a7880af"', + 'cache-control' => 'max-age=0, private, must-revalidate', + 'set-cookie' => '_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly', + 'x-request-id' => 'a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784', + 'x-runtime' => '0.049566', + 'strict-transport-security' => 'max-age=31536000; includeSubDomains; preload', + 'referrer-policy' => 'same-origin', + 'content-security-policy' => "frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de", +]; diff --git a/tests/datasets/curl/about.redirect.php b/tests/datasets/curl/about.redirect.php new file mode 100644 index 000000000..5ae3fd88f --- /dev/null +++ b/tests/datasets/curl/about.redirect.php @@ -0,0 +1,21 @@ + '', + 'date' => 'Thu, 11 Oct 2018 18:43:54 GMT', + 'content-type' => 'text/html; charset=utf-8', + 'vary' => 'Accept-Encoding', + 'server' => 'Mastodon', + 'location' => 'https://test.other/some/', + 'x-frame-options' => 'SAMEORIGIN', + 'x-content-type-options' => 'nosniff', + 'x-xss-protection' => '1; mode=block', + 'etag' => 'W/"706e6c48957e1d46ecf9d7597a7880af"', + 'cache-control' => 'max-age=0, private, must-revalidate', + 'set-cookie' => '_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly', + 'x-request-id' => 'a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784', + 'x-runtime' => '0.049566', + 'strict-transport-security' => 'max-age=31536000; includeSubDomains; preload', + 'referrer-policy' => 'same-origin', + 'content-security-policy' => "frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de", +]; diff --git a/tests/src/Network/CurlResultTest.php b/tests/src/Network/CurlResultTest.php index 090479642..c28dc5f1b 100644 --- a/tests/src/Network/CurlResultTest.php +++ b/tests/src/Network/CurlResultTest.php @@ -53,6 +53,7 @@ class CurlResultTest extends TestCase public function testNormal() { $header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head'); + $headerArray = include(__DIR__ . '/../../datasets/curl/about.head.php'); $body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body'); @@ -65,7 +66,7 @@ class CurlResultTest extends TestCase self::assertTrue($curlResult->isSuccess()); self::assertFalse($curlResult->isTimeout()); self::assertFalse($curlResult->isRedirectUrl()); - self::assertSame($header, $curlResult->getHeader()); + self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('https://test.local', $curlResult->getUrl()); @@ -80,6 +81,7 @@ class CurlResultTest extends TestCase public function testRedirect() { $header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head'); + $headerArray = include(__DIR__ . '/../../datasets/curl/about.head.php'); $body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body'); @@ -93,7 +95,7 @@ class CurlResultTest extends TestCase self::assertTrue($curlResult->isSuccess()); self::assertFalse($curlResult->isTimeout()); self::assertTrue($curlResult->isRedirectUrl()); - self::assertSame($header, $curlResult->getHeader()); + self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('https://test.local/test/it', $curlResult->getUrl()); @@ -106,6 +108,7 @@ class CurlResultTest extends TestCase public function testTimeout() { $header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head'); + $headerArray = include(__DIR__ . '/../../datasets/curl/about.head.php'); $body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body'); @@ -119,7 +122,7 @@ class CurlResultTest extends TestCase self::assertFalse($curlResult->isSuccess()); self::assertTrue($curlResult->isTimeout()); self::assertFalse($curlResult->isRedirectUrl()); - self::assertSame($header, $curlResult->getHeader()); + self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('https://test.local/test/it', $curlResult->getRedirectUrl()); @@ -134,6 +137,7 @@ class CurlResultTest extends TestCase public function testRedirectHeader() { $header = file_get_contents(__DIR__ . '/../../datasets/curl/about.redirect'); + $headerArray = include(__DIR__ . '/../../datasets/curl/about.redirect.php'); $body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body'); @@ -146,7 +150,7 @@ class CurlResultTest extends TestCase self::assertTrue($curlResult->isSuccess()); self::assertFalse($curlResult->isTimeout()); self::assertTrue($curlResult->isRedirectUrl()); - self::assertSame($header, $curlResult->getHeader()); + self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('https://test.local/test/it?key=value', $curlResult->getUrl()); @@ -204,7 +208,7 @@ class CurlResultTest extends TestCase 'url' => 'https://test.local' ]); - self::assertNotEmpty($curlResult->getHeader()); + self::assertNotEmpty($curlResult->getHeaders()); self::assertEmpty($curlResult->getHeader('wrongHeader')); } } From 89f718ec72adfe0e7f67ae7e2b850960c5147889 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:16 +0200 Subject: [PATCH 06/21] Use CurlResult for failed HTTPRequests (legacy usage) --- src/Network/HTTPRequest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 72311ea19..3bc58c19b 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -224,7 +224,7 @@ class HTTPRequest implements IHTTPRequest if ($exception->hasResponse()) { return new GuzzleResponse($exception->getResponse(), $url, $exception->getCode(), $exception->getMessage()); } else { - return new GuzzleResponse(null, $url, $exception->getCode(), $exception->getMessage()); + return new CurlResult($url, '', ['http_code' => $exception->getCode()], $exception->getCode(), $exception->getMessage()); } } finally { $this->profiler->stopRecording(); From 3c074ab315ecb3fb5864eb85a9b2d0ce35d4aa5a Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:20 +0200 Subject: [PATCH 07/21] HTTPRequest: Replace getInfo() with new parameter 'content_length' --- src/Network/CurlResult.php | 6 ------ src/Network/GuzzleResponse.php | 5 ----- src/Network/HTTPRequest.php | 13 +++++++++++-- src/Network/IHTTPRequest.php | 1 + src/Network/IHTTPResult.php | 5 ----- src/Network/Probe.php | 7 +------ src/Util/ParseUrl.php | 2 +- 7 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/Network/CurlResult.php b/src/Network/CurlResult.php index 227d26140..65c110338 100644 --- a/src/Network/CurlResult.php +++ b/src/Network/CurlResult.php @@ -319,12 +319,6 @@ class CurlResult implements IHTTPResult return $this->body; } - /** {@inheritDoc} */ - public function getInfo() - { - return $this->info; - } - /** {@inheritDoc} */ public function isRedirectUrl() { diff --git a/src/Network/GuzzleResponse.php b/src/Network/GuzzleResponse.php index c6cab7c9e..69e88b402 100644 --- a/src/Network/GuzzleResponse.php +++ b/src/Network/GuzzleResponse.php @@ -121,11 +121,6 @@ class GuzzleResponse extends Response implements IHTTPResult, ResponseInterface return $this->url; } - public function getInfo() - { - // TODO: Implement getInfo() method. - } - /** {@inheritDoc} */ public function isRedirectUrl() { diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 3bc58c19b..cf9b53a92 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -30,6 +30,7 @@ use Friendica\Util\Network; use Friendica\Util\Profiler; use GuzzleHttp\Client; use GuzzleHttp\Exception\RequestException; +use GuzzleHttp\Exception\TransferException; use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\UriInterface; @@ -204,6 +205,13 @@ class HTTPRequest implements IHTTPRequest $this->logger->notice('Curl redirect.', ['url' => $request->getUri(), 'to' => $uri]); }; + $onHeaders = function (ResponseInterface $response) use ($opts) { + if (!empty($opts['content_length']) && + $response->getHeaderLine('Content-Length') > $opts['content_length']) { + throw new TransferException('The file is too big!'); + } + }; + $client = new Client([ 'allow_redirect' => [ 'max' => 8, @@ -220,8 +228,9 @@ class HTTPRequest implements IHTTPRequest try { $response = $client->get($url); return new GuzzleResponse($response, $url); - } catch (RequestException $exception) { - if ($exception->hasResponse()) { + } catch (TransferException $exception) { + if ($exception instanceof RequestException && + $exception->hasResponse()) { return new GuzzleResponse($exception->getResponse(), $url, $exception->getCode(), $exception->getMessage()); } else { return new CurlResult($url, '', ['http_code' => $exception->getCode()], $exception->getCode(), $exception->getMessage()); diff --git a/src/Network/IHTTPRequest.php b/src/Network/IHTTPRequest.php index efe9f5f8b..b496c7feb 100644 --- a/src/Network/IHTTPRequest.php +++ b/src/Network/IHTTPRequest.php @@ -79,6 +79,7 @@ interface IHTTPRequest * 'timeout' => int Timeout in seconds, default system config value or 60 seconds * 'cookiejar' => path to cookie jar file * 'header' => header array + * 'content_length' => int maximum File content length * * @return IHTTPResult */ diff --git a/src/Network/IHTTPResult.php b/src/Network/IHTTPResult.php index 5904bcfa3..77ee86976 100644 --- a/src/Network/IHTTPResult.php +++ b/src/Network/IHTTPResult.php @@ -82,11 +82,6 @@ interface IHTTPResult */ public function getBody(); - /** - * @return array - */ - public function getInfo(); - /** * @return boolean */ diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 8cee67940..111580800 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -424,16 +424,11 @@ class Probe */ private static function getHideStatus($url) { - $curlResult = DI::httpRequest()->get($url); + $curlResult = DI::httpRequest()->get($url, false, ['content_length' => 1000000]); if (!$curlResult->isSuccess()) { return false; } - // If the file is too large then exit - if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) { - return false; - } - // If it isn't a HTML file then exit if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { return false; diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index 75210af0f..d95897b1d 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -213,7 +213,7 @@ class ParseUrl return $siteinfo; } - $curlResult = DI::httpRequest()->get($url); + $curlResult = DI::httpRequest()->get($url, false, ['content_length' => 1000000]); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { return $siteinfo; } From 803c1d71de9e6d13da9576660c9ab8b8bb2c19f3 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:20 +0200 Subject: [PATCH 08/21] Fix IHTTPResult::getHeader() - Now returns a string array, like expected - Fix usages - Fix dataset --- src/Content/Text/BBCode.php | 4 +-- src/Model/Photo.php | 5 ++-- src/Model/User.php | 6 ++--- src/Network/CurlResult.php | 10 +++++--- src/Network/IHTTPResult.php | 4 +-- src/Network/Probe.php | 2 +- src/Protocol/OStatus.php | 4 +-- src/Util/Images.php | 18 ++++++++------ src/Util/ParseUrl.php | 6 +++-- tests/datasets/curl/about.head.php | 32 ++++++++++++------------ tests/datasets/curl/about.redirect.php | 34 +++++++++++++------------- 11 files changed, 67 insertions(+), 58 deletions(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index cd433bdbb..ce6b36081 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -499,8 +499,8 @@ class BBCode } $i = $curlResult->getBody(); - $type = $curlResult->getContentType(); - $type = Images::getMimeTypeByData($i, $mtch[1], $type); + $contType = $curlResult->getContentType(); + $type = Images::getMimeTypeByData($i, $mtch[1], $contType); if ($i) { $Image = new Image($i, $type); diff --git a/src/Model/Photo.php b/src/Model/Photo.php index a59c30aca..20d2c8a7a 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -492,16 +492,17 @@ class Photo if (!empty($image_url)) { $ret = DI::httpRequest()->get($image_url); $img_str = $ret->getBody(); - $type = $ret->getContentType(); + $contType = $ret->getContentType(); } else { $img_str = ''; + $contType = []; } if ($quit_on_error && ($img_str == "")) { return false; } - $type = Images::getMimeTypeByData($img_str, $image_url, $type); + $type = Images::getMimeTypeByData($img_str, $image_url, $contType); $Image = new Image($img_str, $type); if ($Image->isValid()) { diff --git a/src/Model/User.php b/src/Model/User.php index 49423ce9e..48a13e6dd 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -1095,13 +1095,13 @@ class User $curlResult = DI::httpRequest()->get($photo); if ($curlResult->isSuccess()) { $img_str = $curlResult->getBody(); - $type = $curlResult->getContentType(); + $contType = $curlResult->getContentType(); } else { $img_str = ''; - $type = ''; + $contType = []; } - $type = Images::getMimeTypeByData($img_str, $photo, $type); + $type = Images::getMimeTypeByData($img_str, $photo, $contType); $Image = new Image($img_str, $type); if ($Image->isValid()) { diff --git a/src/Network/CurlResult.php b/src/Network/CurlResult.php index 65c110338..1c74634db 100644 --- a/src/Network/CurlResult.php +++ b/src/Network/CurlResult.php @@ -245,7 +245,7 @@ class CurlResult implements IHTTPResult public function getHeader($header) { if (empty($header)) { - return ''; + return []; } $header = strtolower(trim($header)); @@ -256,7 +256,7 @@ class CurlResult implements IHTTPResult return $headers[$header]; } - return ''; + return []; } /** {@inheritDoc} */ @@ -289,7 +289,11 @@ class CurlResult implements IHTTPResult $parts = explode(':', $line); $headerfield = strtolower(trim(array_shift($parts))); $headerdata = trim(implode(':', $parts)); - $this->header_fields[$headerfield] = $headerdata; + if (empty($this->header_fields[$headerfield])) { + $this->header_fields[$headerfield] = [$headerdata]; + } elseif (!in_array($headerdata, $this->header_fields[$headerfield])) { + $this->header_fields[$headerfield][] = $headerdata; + } } return $this->header_fields; diff --git a/src/Network/IHTTPResult.php b/src/Network/IHTTPResult.php index 77ee86976..acee2dde9 100644 --- a/src/Network/IHTTPResult.php +++ b/src/Network/IHTTPResult.php @@ -19,7 +19,7 @@ interface IHTTPResult /** * Returns the Content Type * - * @return string the Content Type + * @return string[] the Content Types */ public function getContentType(); @@ -29,7 +29,7 @@ interface IHTTPResult * * @param string $header optional header field. Return all fields if empty * - * @return string the headers or the specified content of the header variable + * @return string[] the headers or the specified content of the header variable */ public function getHeader($header); diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 111580800..ef899b6c1 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -430,7 +430,7 @@ class Probe } // If it isn't a HTML file then exit - if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { + if (!in_array('html', $curlResult->getContentType())) { return false; } diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index 8d818471b..215c08fa0 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -736,7 +736,7 @@ class OStatus $xml = ''; if ($curlResult->inHeader('Content-Type') && - stristr($curlResult->getHeader('Content-Type'), 'application/atom+xml')) { + in_array('application/atom+xml', $curlResult->getHeader('Content-Type'))) { $xml = $curlResult->getBody(); } @@ -930,7 +930,7 @@ class OStatus $xml = ''; if ($curlResult->inHeader('Content-Type') && - stristr($curlResult->getHeader('Content-Type'), 'application/atom+xml')) { + in_array('application/atom+xml', $curlResult->getHeader('Content-Type'))) { Logger::log('Directly fetched XML for URI ' . $related_uri, Logger::DEBUG); $xml = $curlResult->getBody(); } diff --git a/src/Util/Images.php b/src/Util/Images.php index bf84ee6c2..c1ac731b6 100644 --- a/src/Util/Images.php +++ b/src/Util/Images.php @@ -75,23 +75,25 @@ class Images /** * Fetch image mimetype from the image data or guessing from the file name * - * @param string $image_data Image data - * @param string $filename File name (for guessing the type via the extension) - * @param string $mime default mime type + * @param string $image_data Image data + * @param string $filename File name (for guessing the type via the extension) + * @param string[] $mimeTypes possible mime types * * @return string * @throws \Exception */ - public static function getMimeTypeByData(string $image_data, string $filename = '', string $mime = '') + public static function getMimeTypeByData(string $image_data, string $filename = '', array $mimeTypes = []) { - if (substr($mime, 0, 6) == 'image/') { - Logger::info('Using default mime type', ['filename' => $filename, 'mime' => $mime]); - return $mime; + foreach ($mimeTypes as $mimeType) { + if (substr($mimeType, 0, 6) == 'image/') { + Logger::info('Using default mime type', ['filename' => $filename, 'mime' => $mimeTypes]); + return $mimeType; + } } $image = @getimagesizefromstring($image_data); if (!empty($image['mime'])) { - Logger::info('Mime type detected via data', ['filename' => $filename, 'default' => $mime, 'mime' => $image['mime']]); + Logger::info('Mime type detected via data', ['filename' => $filename, 'default' => $mimeTypes, 'mime' => $image['mime']]); return $image['mime']; } diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index d95897b1d..79c3928d7 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -272,8 +272,10 @@ class ParseUrl $charset = ''; // Look for a charset, first in headers // Expected form: Content-Type: text/html; charset=ISO-8859-4 - if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) { - $charset = trim(trim(trim(array_pop($matches)), ';,')); + foreach ($curlResult->getContentType() as $type) { + if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $type, $matches)) { + $charset = trim(trim(trim(array_pop($matches)), ';,')); + } } // Then in body that gets precedence diff --git a/tests/datasets/curl/about.head.php b/tests/datasets/curl/about.head.php index d0be0feb4..b7773b81a 100644 --- a/tests/datasets/curl/about.head.php +++ b/tests/datasets/curl/about.head.php @@ -1,20 +1,20 @@ '', - 'date' => 'Thu, 11 Oct 2018 18:43:54 GMT', - 'content-type' => 'text/html; charset=utf-8', - 'vary' => 'Accept-Encoding', - 'server' => 'Mastodon', - 'x-frame-options' => 'SAMEORIGIN', - 'x-content-type-options' => 'nosniff', - 'x-xss-protection' => '1; mode=block', - 'etag' => 'W/"706e6c48957e1d46ecf9d7597a7880af"', - 'cache-control' => 'max-age=0, private, must-revalidate', - 'set-cookie' => '_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly', - 'x-request-id' => 'a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784', - 'x-runtime' => '0.049566', - 'strict-transport-security' => 'max-age=31536000; includeSubDomains; preload', - 'referrer-policy' => 'same-origin', - 'content-security-policy' => "frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de", + 'http/2 200' => [''], + 'date' => ['Thu, 11 Oct 2018 18:43:54 GMT'], + 'content-type' => ['text/html; charset=utf-8'], + 'vary' => ['Accept-Encoding'], + 'server' => ['Mastodon'], + 'x-frame-options' => ['DENY', 'SAMEORIGIN'], + 'x-content-type-options' => ['nosniff'], + 'x-xss-protection' => ['1; mode=block'], + 'etag' => ['W/"706e6c48957e1d46ecf9d7597a7880af"'], + 'cache-control' => ['max-age=0, private, must-revalidate'], + 'set-cookie' => ['_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly'], + 'x-request-id' => ['a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784'], + 'x-runtime' => ['0.049566'], + 'strict-transport-security' => ['max-age=31536000; includeSubDomains; preload'], + 'referrer-policy' => ['same-origin'], + 'content-security-policy' => ["frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de"], ]; diff --git a/tests/datasets/curl/about.redirect.php b/tests/datasets/curl/about.redirect.php index 5ae3fd88f..f01689aad 100644 --- a/tests/datasets/curl/about.redirect.php +++ b/tests/datasets/curl/about.redirect.php @@ -1,21 +1,21 @@ '', - 'date' => 'Thu, 11 Oct 2018 18:43:54 GMT', - 'content-type' => 'text/html; charset=utf-8', - 'vary' => 'Accept-Encoding', - 'server' => 'Mastodon', - 'location' => 'https://test.other/some/', - 'x-frame-options' => 'SAMEORIGIN', - 'x-content-type-options' => 'nosniff', - 'x-xss-protection' => '1; mode=block', - 'etag' => 'W/"706e6c48957e1d46ecf9d7597a7880af"', - 'cache-control' => 'max-age=0, private, must-revalidate', - 'set-cookie' => '_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly', - 'x-request-id' => 'a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784', - 'x-runtime' => '0.049566', - 'strict-transport-security' => 'max-age=31536000; includeSubDomains; preload', - 'referrer-policy' => 'same-origin', - 'content-security-policy' => "frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de", + 'http/2 301' => [''], + 'date' => ['Thu, 11 Oct 2018 18:43:54 GMT'], + 'content-type' => ['text/html; charset=utf-8'], + 'vary' => ['Accept-Encoding'], + 'server' => ['Mastodon'], + 'location' => ['https://test.other/some/'], + 'x-frame-options' => ['DENY', 'SAMEORIGIN'], + 'x-content-type-options' => ['nosniff'], + 'x-xss-protection' => ['1; mode=block'], + 'etag' => ['W/"706e6c48957e1d46ecf9d7597a7880af"'], + 'cache-control' => ['max-age=0, private, must-revalidate'], + 'set-cookie' => ['_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly'], + 'x-request-id' => ['a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784'], + 'x-runtime' => ['0.049566'], + 'strict-transport-security' => ['max-age=31536000; includeSubDomains; preload'], + 'referrer-policy' => ['same-origin'], + 'content-security-policy' => ["frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de"], ]; From 82f767f2ee200720d0e6fefa3724e972146803f3 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:20 +0200 Subject: [PATCH 09/21] Fix Content-Type for `CurlResult` class --- src/Network/CurlResult.php | 6 +++--- tests/src/Network/CurlResultTest.php | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Network/CurlResult.php b/src/Network/CurlResult.php index 1c74634db..9d93c8bd8 100644 --- a/src/Network/CurlResult.php +++ b/src/Network/CurlResult.php @@ -37,7 +37,7 @@ class CurlResult implements IHTTPResult private $returnCode; /** - * @var string the content type of the Curl call + * @var string[] the content type of the Curl call */ private $contentType; @@ -223,9 +223,9 @@ class CurlResult implements IHTTPResult private function checkInfo() { if (isset($this->info['content_type'])) { - $this->contentType = $this->info['content_type']; + $this->contentType = [$this->info['content_type']]; } else { - $this->contentType = ''; + $this->contentType = []; } } diff --git a/tests/src/Network/CurlResultTest.php b/tests/src/Network/CurlResultTest.php index c28dc5f1b..19fc8c3ec 100644 --- a/tests/src/Network/CurlResultTest.php +++ b/tests/src/Network/CurlResultTest.php @@ -68,7 +68,7 @@ class CurlResultTest extends TestCase self::assertFalse($curlResult->isRedirectUrl()); self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); - self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); + self::assertSame(['text/html; charset=utf-8'], $curlResult->getContentType()); self::assertSame('https://test.local', $curlResult->getUrl()); self::assertSame('https://test.local', $curlResult->getRedirectUrl()); } @@ -97,7 +97,7 @@ class CurlResultTest extends TestCase self::assertTrue($curlResult->isRedirectUrl()); self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); - self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); + self::assertSame(['text/html; charset=utf-8'], $curlResult->getContentType()); self::assertSame('https://test.local/test/it', $curlResult->getUrl()); self::assertSame('https://test.other/test/it', $curlResult->getRedirectUrl()); } @@ -124,7 +124,7 @@ class CurlResultTest extends TestCase self::assertFalse($curlResult->isRedirectUrl()); self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); - self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); + self::assertSame(['text/html; charset=utf-8'], $curlResult->getContentType()); self::assertSame('https://test.local/test/it', $curlResult->getRedirectUrl()); self::assertSame('Tested error', $curlResult->getError()); } @@ -152,7 +152,7 @@ class CurlResultTest extends TestCase self::assertTrue($curlResult->isRedirectUrl()); self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); - self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); + self::assertSame(['text/html; charset=utf-8'], $curlResult->getContentType()); self::assertSame('https://test.local/test/it?key=value', $curlResult->getUrl()); self::assertSame('https://test.other/some/?key=value', $curlResult->getRedirectUrl()); } From 26401e88b87646cabc21eb7a095b3ac8ebd76fe9 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:21 +0200 Subject: [PATCH 10/21] IHTTPResult::getContentType is now a string again --- src/Model/Photo.php | 2 +- src/Model/User.php | 2 +- src/Network/CurlResult.php | 6 +++--- src/Network/GuzzleResponse.php | 2 +- src/Network/IHTTPResult.php | 2 +- src/Network/Probe.php | 2 +- src/Util/Images.php | 18 ++++++++---------- src/Util/ParseUrl.php | 6 ++---- tests/src/Network/CurlResultTest.php | 8 ++++---- 9 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/Model/Photo.php b/src/Model/Photo.php index 20d2c8a7a..2a2d02f86 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -495,7 +495,7 @@ class Photo $contType = $ret->getContentType(); } else { $img_str = ''; - $contType = []; + $contType = ''; } if ($quit_on_error && ($img_str == "")) { diff --git a/src/Model/User.php b/src/Model/User.php index 48a13e6dd..34a7b78a4 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -1098,7 +1098,7 @@ class User $contType = $curlResult->getContentType(); } else { $img_str = ''; - $contType = []; + $contType = ''; } $type = Images::getMimeTypeByData($img_str, $photo, $contType); diff --git a/src/Network/CurlResult.php b/src/Network/CurlResult.php index 9d93c8bd8..1c74634db 100644 --- a/src/Network/CurlResult.php +++ b/src/Network/CurlResult.php @@ -37,7 +37,7 @@ class CurlResult implements IHTTPResult private $returnCode; /** - * @var string[] the content type of the Curl call + * @var string the content type of the Curl call */ private $contentType; @@ -223,9 +223,9 @@ class CurlResult implements IHTTPResult private function checkInfo() { if (isset($this->info['content_type'])) { - $this->contentType = [$this->info['content_type']]; + $this->contentType = $this->info['content_type']; } else { - $this->contentType = []; + $this->contentType = ''; } } diff --git a/src/Network/GuzzleResponse.php b/src/Network/GuzzleResponse.php index 69e88b402..72d87ae9f 100644 --- a/src/Network/GuzzleResponse.php +++ b/src/Network/GuzzleResponse.php @@ -88,7 +88,7 @@ class GuzzleResponse extends Response implements IHTTPResult, ResponseInterface /** {@inheritDoc} */ public function getContentType() { - return $this->getHeader('Content-Type'); + return implode($this->getHeader('Content-Type')); } /** {@inheritDoc} */ diff --git a/src/Network/IHTTPResult.php b/src/Network/IHTTPResult.php index acee2dde9..38a117628 100644 --- a/src/Network/IHTTPResult.php +++ b/src/Network/IHTTPResult.php @@ -19,7 +19,7 @@ interface IHTTPResult /** * Returns the Content Type * - * @return string[] the Content Types + * @return string the Content Type */ public function getContentType(); diff --git a/src/Network/Probe.php b/src/Network/Probe.php index ef899b6c1..111580800 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -430,7 +430,7 @@ class Probe } // If it isn't a HTML file then exit - if (!in_array('html', $curlResult->getContentType())) { + if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { return false; } diff --git a/src/Util/Images.php b/src/Util/Images.php index c1ac731b6..1dcca2e39 100644 --- a/src/Util/Images.php +++ b/src/Util/Images.php @@ -75,25 +75,23 @@ class Images /** * Fetch image mimetype from the image data or guessing from the file name * - * @param string $image_data Image data - * @param string $filename File name (for guessing the type via the extension) - * @param string[] $mimeTypes possible mime types + * @param string $image_data Image data + * @param string $filename File name (for guessing the type via the extension) + * @param string $mimeType possible mime type * * @return string * @throws \Exception */ - public static function getMimeTypeByData(string $image_data, string $filename = '', array $mimeTypes = []) + public static function getMimeTypeByData(string $image_data, string $filename = '', string $mimeType = '') { - foreach ($mimeTypes as $mimeType) { - if (substr($mimeType, 0, 6) == 'image/') { - Logger::info('Using default mime type', ['filename' => $filename, 'mime' => $mimeTypes]); - return $mimeType; - } + if (substr($mimeType, 0, 6) == 'image/') { + Logger::info('Using default mime type', ['filename' => $filename, 'mime' => $mimeType]); + return $mimeType; } $image = @getimagesizefromstring($image_data); if (!empty($image['mime'])) { - Logger::info('Mime type detected via data', ['filename' => $filename, 'default' => $mimeTypes, 'mime' => $image['mime']]); + Logger::info('Mime type detected via data', ['filename' => $filename, 'default' => $mimeType, 'mime' => $image['mime']]); return $image['mime']; } diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index 79c3928d7..d95897b1d 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -272,10 +272,8 @@ class ParseUrl $charset = ''; // Look for a charset, first in headers // Expected form: Content-Type: text/html; charset=ISO-8859-4 - foreach ($curlResult->getContentType() as $type) { - if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $type, $matches)) { - $charset = trim(trim(trim(array_pop($matches)), ';,')); - } + if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) { + $charset = trim(trim(trim(array_pop($matches)), ';,')); } // Then in body that gets precedence diff --git a/tests/src/Network/CurlResultTest.php b/tests/src/Network/CurlResultTest.php index 19fc8c3ec..c28dc5f1b 100644 --- a/tests/src/Network/CurlResultTest.php +++ b/tests/src/Network/CurlResultTest.php @@ -68,7 +68,7 @@ class CurlResultTest extends TestCase self::assertFalse($curlResult->isRedirectUrl()); self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); - self::assertSame(['text/html; charset=utf-8'], $curlResult->getContentType()); + self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('https://test.local', $curlResult->getUrl()); self::assertSame('https://test.local', $curlResult->getRedirectUrl()); } @@ -97,7 +97,7 @@ class CurlResultTest extends TestCase self::assertTrue($curlResult->isRedirectUrl()); self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); - self::assertSame(['text/html; charset=utf-8'], $curlResult->getContentType()); + self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('https://test.local/test/it', $curlResult->getUrl()); self::assertSame('https://test.other/test/it', $curlResult->getRedirectUrl()); } @@ -124,7 +124,7 @@ class CurlResultTest extends TestCase self::assertFalse($curlResult->isRedirectUrl()); self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); - self::assertSame(['text/html; charset=utf-8'], $curlResult->getContentType()); + self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('https://test.local/test/it', $curlResult->getRedirectUrl()); self::assertSame('Tested error', $curlResult->getError()); } @@ -152,7 +152,7 @@ class CurlResultTest extends TestCase self::assertTrue($curlResult->isRedirectUrl()); self::assertSame($headerArray, $curlResult->getHeaders()); self::assertSame($body, $curlResult->getBody()); - self::assertSame(['text/html; charset=utf-8'], $curlResult->getContentType()); + self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('https://test.local/test/it?key=value', $curlResult->getUrl()); self::assertSame('https://test.other/some/?key=value', $curlResult->getRedirectUrl()); } From 1995feff619c23de0e0f34248d37a49e90862a88 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:21 +0200 Subject: [PATCH 11/21] Fix redirect logging --- src/Network/HTTPRequest.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index cf9b53a92..d47e969d5 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -197,12 +197,14 @@ class HTTPRequest implements IHTTPRequest $curlOptions[CURLOPT_BINARYTRANSFER] = 1; } + $logger = $this->logger; + $onRedirect = function( RequestInterface $request, ResponseInterface $response, UriInterface $uri - ) { - $this->logger->notice('Curl redirect.', ['url' => $request->getUri(), 'to' => $uri]); + ) use ($logger) { + $logger->notice('Curl redirect.', ['url' => $request->getUri(), 'to' => $uri]); }; $onHeaders = function (ResponseInterface $response) use ($opts) { From 50e2478189a05b4e7048fb2574a22bc25ffa2719 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:21 +0200 Subject: [PATCH 12/21] Remove unnecessary exception message (avoid log flooding) --- src/Network/HTTPRequest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index d47e969d5..e68c0a8f4 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -233,9 +233,9 @@ class HTTPRequest implements IHTTPRequest } catch (TransferException $exception) { if ($exception instanceof RequestException && $exception->hasResponse()) { - return new GuzzleResponse($exception->getResponse(), $url, $exception->getCode(), $exception->getMessage()); + return new GuzzleResponse($exception->getResponse(), $url, $exception->getCode(), ''); } else { - return new CurlResult($url, '', ['http_code' => $exception->getCode()], $exception->getCode(), $exception->getMessage()); + return new CurlResult($url, '', ['http_code' => $exception->getCode()], $exception->getCode(), ''); } } finally { $this->profiler->stopRecording(); From 756f57f1988ae84b2b4f16febe6c0b7ac8e871b6 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:22 +0200 Subject: [PATCH 13/21] Revert "Use last entry for Content-Type --- src/Network/GuzzleResponse.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Network/GuzzleResponse.php b/src/Network/GuzzleResponse.php index 72d87ae9f..18155d1d2 100644 --- a/src/Network/GuzzleResponse.php +++ b/src/Network/GuzzleResponse.php @@ -88,7 +88,13 @@ class GuzzleResponse extends Response implements IHTTPResult, ResponseInterface /** {@inheritDoc} */ public function getContentType() { - return implode($this->getHeader('Content-Type')); + $contentTypes = $this->getHeader('Content-Type') ?? []; + $countTypes = count($contentTypes); + if ($countTypes > 0) { + return $contentTypes[$countTypes - 1]; + } else { + return ''; + } } /** {@inheritDoc} */ From a338e4cbff7c0e9b238bf53b4b369398a822b45d Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 19:48:23 +0200 Subject: [PATCH 14/21] Update src/Network/GuzzleResponse.php --- src/Network/GuzzleResponse.php | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Network/GuzzleResponse.php b/src/Network/GuzzleResponse.php index 18155d1d2..b1bbaa965 100644 --- a/src/Network/GuzzleResponse.php +++ b/src/Network/GuzzleResponse.php @@ -89,12 +89,7 @@ class GuzzleResponse extends Response implements IHTTPResult, ResponseInterface public function getContentType() { $contentTypes = $this->getHeader('Content-Type') ?? []; - $countTypes = count($contentTypes); - if ($countTypes > 0) { - return $contentTypes[$countTypes - 1]; - } else { - return ''; - } + return array_pop($contentTypes) ?? ''; } /** {@inheritDoc} */ From 06371d29a679e3eed2a4c5cbb7b2b15ee82ede23 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 20:03:42 +0200 Subject: [PATCH 15/21] Cleanup --- src/Content/Text/BBCode.php | 4 ++-- src/Model/Photo.php | 6 +++--- src/Model/User.php | 6 +++--- src/Network/Probe.php | 5 +++++ src/Util/Images.php | 12 ++++++------ 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index ce6b36081..cd433bdbb 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -499,8 +499,8 @@ class BBCode } $i = $curlResult->getBody(); - $contType = $curlResult->getContentType(); - $type = Images::getMimeTypeByData($i, $mtch[1], $contType); + $type = $curlResult->getContentType(); + $type = Images::getMimeTypeByData($i, $mtch[1], $type); if ($i) { $Image = new Image($i, $type); diff --git a/src/Model/Photo.php b/src/Model/Photo.php index 2a2d02f86..23e0b9a38 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -492,17 +492,17 @@ class Photo if (!empty($image_url)) { $ret = DI::httpRequest()->get($image_url); $img_str = $ret->getBody(); - $contType = $ret->getContentType(); + $type = $ret->getContentType(); } else { $img_str = ''; - $contType = ''; + $type = ''; } if ($quit_on_error && ($img_str == "")) { return false; } - $type = Images::getMimeTypeByData($img_str, $image_url, $contType); + $type = Images::getMimeTypeByData($img_str, $image_url, $type); $Image = new Image($img_str, $type); if ($Image->isValid()) { diff --git a/src/Model/User.php b/src/Model/User.php index 34a7b78a4..49423ce9e 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -1095,13 +1095,13 @@ class User $curlResult = DI::httpRequest()->get($photo); if ($curlResult->isSuccess()) { $img_str = $curlResult->getBody(); - $contType = $curlResult->getContentType(); + $type = $curlResult->getContentType(); } else { $img_str = ''; - $contType = ''; + $type = ''; } - $type = Images::getMimeTypeByData($img_str, $photo, $contType); + $type = Images::getMimeTypeByData($img_str, $photo, $type); $Image = new Image($img_str, $type); if ($Image->isValid()) { diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 111580800..c547afcd4 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -429,6 +429,11 @@ class Probe return false; } + // If the file is too large then exit + if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) { + return false; + } + // If it isn't a HTML file then exit if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { return false; diff --git a/src/Util/Images.php b/src/Util/Images.php index 1dcca2e39..bf84ee6c2 100644 --- a/src/Util/Images.php +++ b/src/Util/Images.php @@ -77,21 +77,21 @@ class Images * * @param string $image_data Image data * @param string $filename File name (for guessing the type via the extension) - * @param string $mimeType possible mime type + * @param string $mime default mime type * * @return string * @throws \Exception */ - public static function getMimeTypeByData(string $image_data, string $filename = '', string $mimeType = '') + public static function getMimeTypeByData(string $image_data, string $filename = '', string $mime = '') { - if (substr($mimeType, 0, 6) == 'image/') { - Logger::info('Using default mime type', ['filename' => $filename, 'mime' => $mimeType]); - return $mimeType; + if (substr($mime, 0, 6) == 'image/') { + Logger::info('Using default mime type', ['filename' => $filename, 'mime' => $mime]); + return $mime; } $image = @getimagesizefromstring($image_data); if (!empty($image['mime'])) { - Logger::info('Mime type detected via data', ['filename' => $filename, 'default' => $mimeType, 'mime' => $image['mime']]); + Logger::info('Mime type detected via data', ['filename' => $filename, 'default' => $mime, 'mime' => $image['mime']]); return $image['mime']; } From 65ca164487af3890affe224e14b090cc654a2bd0 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 20:05:41 +0200 Subject: [PATCH 16/21] A lot of Fixings --- src/Content/Text/BBCode.php | 4 ++-- src/Model/Post/Link.php | 2 +- src/Model/Post/Media.php | 4 ++-- src/Network/GuzzleResponse.php | 6 ++++++ src/Network/HTTPRequest.php | 7 +------ src/Network/Probe.php | 7 +------ src/Util/ParseUrl.php | 11 +++-------- 7 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index cd433bdbb..f52a2137d 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1195,7 +1195,7 @@ class BBCode if (is_null($text)) { $curlResult = DI::httpRequest()->head($match[1], ['timeout' => DI::config()->get('system', 'xrd_timeout')]); if ($curlResult->isSuccess()) { - $mimetype = $curlResult->getHeader('Content-Type'); + $mimetype = $curlResult->getHeader('Content-Type')[0] ?? ''; } else { $mimetype = ''; } @@ -1266,7 +1266,7 @@ class BBCode $curlResult = DI::httpRequest()->head($match[1], ['timeout' => DI::config()->get('system', 'xrd_timeout')]); if ($curlResult->isSuccess()) { - $mimetype = $curlResult->getHeader('Content-Type'); + $mimetype = $curlResult->getHeader('Content-Type')[0] ?? ''; } else { $mimetype = ''; } diff --git a/src/Model/Post/Link.php b/src/Model/Post/Link.php index 64840a9c9..edd341162 100644 --- a/src/Model/Post/Link.php +++ b/src/Model/Post/Link.php @@ -90,7 +90,7 @@ class Link $curlResult = DI::httpRequest()->head($url, ['timeout' => $timeout]); if ($curlResult->isSuccess()) { if (empty($media['mimetype'])) { - return $curlResult->getHeader('Content-Type'); + return $curlResult->getHeader('Content-Type')[0] ?? ''; } } return ''; diff --git a/src/Model/Post/Media.php b/src/Model/Post/Media.php index fae71a953..c57b1cabe 100644 --- a/src/Model/Post/Media.php +++ b/src/Model/Post/Media.php @@ -170,10 +170,10 @@ class Media $curlResult = DI::httpRequest()->head($media['url'], ['timeout' => $timeout]); if ($curlResult->isSuccess()) { if (empty($media['mimetype'])) { - $media['mimetype'] = $curlResult->getHeader('Content-Type'); + $media['mimetype'] = $curlResult->getHeader('Content-Type')[0] ?? ''; } if (empty($media['size'])) { - $media['size'] = (int)$curlResult->getHeader('Content-Length'); + $media['size'] = (int)$curlResult->getHeader('Content-Length')[0] ?? 0; } } else { Logger::notice('Could not fetch head', ['media' => $media]); diff --git a/src/Network/GuzzleResponse.php b/src/Network/GuzzleResponse.php index b1bbaa965..1bb2ee4c5 100644 --- a/src/Network/GuzzleResponse.php +++ b/src/Network/GuzzleResponse.php @@ -145,4 +145,10 @@ class GuzzleResponse extends Response implements IHTTPResult, ResponseInterface { return $this->isTimeout; } + + /// @todo - fix mismatching use of "getBody()" as string here and parent "getBody()" as streaminterface + public function getBody() + { + return parent::getBody()->getContents(); + } } diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index e68c0a8f4..18c060fc5 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -72,7 +72,7 @@ class HTTPRequest implements IHTTPRequest /** * {@inheritDoc} */ - public function get(string $url, bool $binary = false, array $opts = []) + public function get(string $url, array $opts = []) { $this->profiler->startRecording('network'); @@ -193,10 +193,6 @@ class HTTPRequest implements IHTTPRequest $curlOptions[CURLOPT_IPRESOLVE] = CURL_IPRESOLVE_V4; } - if ($binary) { - $curlOptions[CURLOPT_BINARYTRANSFER] = 1; - } - $logger = $this->logger; $onRedirect = function( @@ -223,7 +219,6 @@ class HTTPRequest implements IHTTPRequest 'referer' => true, ], 'on_headers' => $onHeaders, - 'sink' => tempnam(get_temppath(), 'guzzle'), 'curl' => $curlOptions ]); diff --git a/src/Network/Probe.php b/src/Network/Probe.php index c547afcd4..4cacfedf4 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -424,16 +424,11 @@ class Probe */ private static function getHideStatus($url) { - $curlResult = DI::httpRequest()->get($url, false, ['content_length' => 1000000]); + $curlResult = DI::httpRequest()->get($url, ['content_length' => 1000000]); if (!$curlResult->isSuccess()) { return false; } - // If the file is too large then exit - if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) { - return false; - } - // If it isn't a HTML file then exit if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { return false; diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index d95897b1d..915a143a0 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -63,7 +63,7 @@ class ParseUrl return []; } - $contenttype = $curlResult->getHeader('Content-Type'); + $contenttype = $curlResult->getHeader('Content-Type')[0] ?? ''; if (empty($contenttype)) { return []; } @@ -213,19 +213,14 @@ class ParseUrl return $siteinfo; } - $curlResult = DI::httpRequest()->get($url, false, ['content_length' => 1000000]); + $curlResult = DI::httpRequest()->get($url, ['content_length' => 1000000]); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { return $siteinfo; } $siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS); - // If the file is too large then exit - if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) { - return $siteinfo; - } - - if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')) { + if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')[0] ?? '') { if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) { $maxAge = max(86400, (int)array_pop($matches)); $siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds"); From 653d49cb92206aab7e21931759a1442bcb118a05 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 22:00:28 +0200 Subject: [PATCH 17/21] Fix getHeader() --- src/Model/Post/Media.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Model/Post/Media.php b/src/Model/Post/Media.php index c57b1cabe..6f3ca2344 100644 --- a/src/Model/Post/Media.php +++ b/src/Model/Post/Media.php @@ -173,7 +173,7 @@ class Media $media['mimetype'] = $curlResult->getHeader('Content-Type')[0] ?? ''; } if (empty($media['size'])) { - $media['size'] = (int)$curlResult->getHeader('Content-Length')[0] ?? 0; + $media['size'] = (int)($curlResult->getHeader('Content-Length')[0] ?? 0); } } else { Logger::notice('Could not fetch head', ['media' => $media]); From 9eba32226c9ac1a67bdb4f5b6310e780ff5d5f42 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 22:11:36 +0200 Subject: [PATCH 18/21] make CS happy :) --- src/Network/GuzzleResponse.php | 2 +- tests/datasets/curl/about.head.php | 32 ++++++++++++------------ tests/datasets/curl/about.redirect.php | 34 +++++++++++++------------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/Network/GuzzleResponse.php b/src/Network/GuzzleResponse.php index 1bb2ee4c5..b68f2e843 100644 --- a/src/Network/GuzzleResponse.php +++ b/src/Network/GuzzleResponse.php @@ -68,7 +68,7 @@ class GuzzleResponse extends Response implements IHTTPResult, ResponseInterface } if (!$this->isSuccess) { - Logger::notice('http error', ['url' => $this->url, 'code' => $this->getReturnCode(), 'error' => $this->error, 'callstack' => System::callstack(20)]); + Logger::notice('http error', ['url' => $this->url, 'code' => $this->getReturnCode(), 'error' => $this->error, 'callstack' => System::callstack(20)]); Logger::debug('debug', ['info' => $this->getHeaders()]); } diff --git a/tests/datasets/curl/about.head.php b/tests/datasets/curl/about.head.php index b7773b81a..2369bb65c 100644 --- a/tests/datasets/curl/about.head.php +++ b/tests/datasets/curl/about.head.php @@ -1,20 +1,20 @@ [''], - 'date' => ['Thu, 11 Oct 2018 18:43:54 GMT'], - 'content-type' => ['text/html; charset=utf-8'], - 'vary' => ['Accept-Encoding'], - 'server' => ['Mastodon'], - 'x-frame-options' => ['DENY', 'SAMEORIGIN'], - 'x-content-type-options' => ['nosniff'], - 'x-xss-protection' => ['1; mode=block'], - 'etag' => ['W/"706e6c48957e1d46ecf9d7597a7880af"'], - 'cache-control' => ['max-age=0, private, must-revalidate'], - 'set-cookie' => ['_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly'], - 'x-request-id' => ['a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784'], - 'x-runtime' => ['0.049566'], - 'strict-transport-security' => ['max-age=31536000; includeSubDomains; preload'], - 'referrer-policy' => ['same-origin'], - 'content-security-policy' => ["frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de"], + 'http/2 200' => [''], + 'date' => ['Thu, 11 Oct 2018 18:43:54 GMT'], + 'content-type' => ['text/html; charset=utf-8'], + 'vary' => ['Accept-Encoding'], + 'server' => ['Mastodon'], + 'x-frame-options' => ['DENY', 'SAMEORIGIN'], + 'x-content-type-options' => ['nosniff'], + 'x-xss-protection' => ['1; mode=block'], + 'etag' => ['W/"706e6c48957e1d46ecf9d7597a7880af"'], + 'cache-control' => ['max-age=0, private, must-revalidate'], + 'set-cookie' => ['_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly'], + 'x-request-id' => ['a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784'], + 'x-runtime' => ['0.049566'], + 'strict-transport-security' => ['max-age=31536000; includeSubDomains; preload'], + 'referrer-policy' => ['same-origin'], + 'content-security-policy' => ["frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de"], ]; diff --git a/tests/datasets/curl/about.redirect.php b/tests/datasets/curl/about.redirect.php index f01689aad..63ae12637 100644 --- a/tests/datasets/curl/about.redirect.php +++ b/tests/datasets/curl/about.redirect.php @@ -1,21 +1,21 @@ [''], - 'date' => ['Thu, 11 Oct 2018 18:43:54 GMT'], - 'content-type' => ['text/html; charset=utf-8'], - 'vary' => ['Accept-Encoding'], - 'server' => ['Mastodon'], - 'location' => ['https://test.other/some/'], - 'x-frame-options' => ['DENY', 'SAMEORIGIN'], - 'x-content-type-options' => ['nosniff'], - 'x-xss-protection' => ['1; mode=block'], - 'etag' => ['W/"706e6c48957e1d46ecf9d7597a7880af"'], - 'cache-control' => ['max-age=0, private, must-revalidate'], - 'set-cookie' => ['_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly'], - 'x-request-id' => ['a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784'], - 'x-runtime' => ['0.049566'], - 'strict-transport-security' => ['max-age=31536000; includeSubDomains; preload'], - 'referrer-policy' => ['same-origin'], - 'content-security-policy' => ["frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de"], + 'http/2 301' => [''], + 'date' => ['Thu, 11 Oct 2018 18:43:54 GMT'], + 'content-type' => ['text/html; charset=utf-8'], + 'vary' => ['Accept-Encoding'], + 'server' => ['Mastodon'], + 'location' => ['https://test.other/some/'], + 'x-frame-options' => ['DENY', 'SAMEORIGIN'], + 'x-content-type-options' => ['nosniff'], + 'x-xss-protection' => ['1; mode=block'], + 'etag' => ['W/"706e6c48957e1d46ecf9d7597a7880af"'], + 'cache-control' => ['max-age=0, private, must-revalidate'], + 'set-cookie' => ['_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly'], + 'x-request-id' => ['a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784'], + 'x-runtime' => ['0.049566'], + 'strict-transport-security' => ['max-age=31536000; includeSubDomains; preload'], + 'referrer-policy' => ['same-origin'], + 'content-security-policy' => ["frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de"], ]; From 8f13319c73a4b3b865a13105eca4bee53c2eecd8 Mon Sep 17 00:00:00 2001 From: Philipp Date: Fri, 20 Aug 2021 22:30:54 +0200 Subject: [PATCH 19/21] remove unused $redirect parameter --- src/Network/HTTPRequest.php | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 18c060fc5..320ff04c1 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -472,12 +472,8 @@ class HTTPRequest implements IHTTPRequest /** * {@inheritDoc} - * - * @param int $redirects The recursion counter for internal use - default 0 - * - * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public function fetch(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '', &$redirects = 0) + public function fetch(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '') { $ret = $this->fetchFull($url, $timeout, $accept_content, $cookiejar, $redirects); @@ -486,12 +482,8 @@ class HTTPRequest implements IHTTPRequest /** * {@inheritDoc} - * - * @param int $redirects The recursion counter for internal use - default 0 - * - * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public function fetchFull(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '', &$redirects = 0) + public function fetchFull(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '') { return $this->get( $url, From c3eca0cfae29ff710496894cc391e695d2f77fce Mon Sep 17 00:00:00 2001 From: Philipp Date: Sun, 22 Aug 2021 22:43:28 +0200 Subject: [PATCH 20/21] Remove legacy header-parsing for guzzle --- src/Network/HTTPRequest.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 320ff04c1..e33b0ed59 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -107,8 +107,6 @@ class HTTPRequest implements IHTTPRequest $curlOptions = []; - $curlOptions[CURLOPT_HEADER] = true; - if (!empty($opts['cookiejar'])) { $curlOptions[CURLOPT_COOKIEJAR] = $opts["cookiejar"]; $curlOptions[CURLOPT_COOKIEFILE] = $opts["cookiejar"]; @@ -475,7 +473,7 @@ class HTTPRequest implements IHTTPRequest */ public function fetch(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '') { - $ret = $this->fetchFull($url, $timeout, $accept_content, $cookiejar, $redirects); + $ret = $this->fetchFull($url, $timeout, $accept_content, $cookiejar); return $ret->getBody(); } From ff2d85b703bbdea270c7cf4611643ebeb69c3de3 Mon Sep 17 00:00:00 2001 From: Philipp Date: Sun, 22 Aug 2021 22:49:55 +0200 Subject: [PATCH 21/21] Initialize $curlOptions[CURLOPT_HTTPHEADER] outside of the checks --- src/Network/HTTPRequest.php | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index e33b0ed59..b08e91832 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -108,7 +108,7 @@ class HTTPRequest implements IHTTPRequest $curlOptions = []; if (!empty($opts['cookiejar'])) { - $curlOptions[CURLOPT_COOKIEJAR] = $opts["cookiejar"]; + $curlOptions[CURLOPT_COOKIEJAR] = $opts["cookiejar"]; $curlOptions[CURLOPT_COOKIEFILE] = $opts["cookiejar"]; } @@ -116,22 +116,18 @@ class HTTPRequest implements IHTTPRequest // $curlOptions[CURLOPT_FOLLOWLOCATION] =true; // $curlOptions[CURLOPT_MAXREDIRS] = 5; + $curlOptions[CURLOPT_HTTPHEADER] = []; + if (!empty($opts['accept_content'])) { - if (empty($curlOptions[CURLOPT_HTTPHEADER])) { - $curlOptions[CURLOPT_HTTPHEADER] = []; - } array_push($curlOptions[CURLOPT_HTTPHEADER], 'Accept: ' . $opts['accept_content']); } if (!empty($opts['header'])) { - if (empty($curlOptions[CURLOPT_HTTPHEADER])) { - $curlOptions[CURLOPT_HTTPHEADER] = []; - } $curlOptions[CURLOPT_HTTPHEADER] = array_merge($opts['header'], $curlOptions[CURLOPT_HTTPHEADER]); } $curlOptions[CURLOPT_RETURNTRANSFER] = true; - $curlOptions[CURLOPT_USERAGENT] = $this->getUserAgent(); + $curlOptions[CURLOPT_USERAGENT] = $this->getUserAgent(); $range = intval($this->config->get('system', 'curl_range_bytes', 0)); @@ -146,9 +142,6 @@ class HTTPRequest implements IHTTPRequest if (!empty($opts['headers'])) { $this->logger->notice('Wrong option \'headers\' used.'); - if (empty($curlOptions[CURLOPT_HTTPHEADER])) { - $curlOptions[CURLOPT_HTTPHEADER] = []; - } $curlOptions[CURLOPT_HTTPHEADER] = array_merge($opts['headers'], $curlOptions[CURLOPT_HTTPHEADER]); }