Merge pull request #10619 from nupplaphil/task/guzzlehttp

PSR-7 Part 1: Use Guzzle PSR-7 library for HTTPRequest->get()
This commit is contained in:
Hypolite Petovan 2021-08-22 18:11:30 -04:00 committed by GitHub
commit 2108be7e07
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 461 additions and 201 deletions

View file

@ -31,6 +31,7 @@
"divineomega/password_exposed": "^2.8", "divineomega/password_exposed": "^2.8",
"ezyang/htmlpurifier": "^4.7", "ezyang/htmlpurifier": "^4.7",
"friendica/json-ld": "^1.0", "friendica/json-ld": "^1.0",
"guzzlehttp/guzzle": "^6.5",
"league/html-to-markdown": "^4.8", "league/html-to-markdown": "^4.8",
"level-2/dice": "^4", "level-2/dice": "^4",
"lightopenid/lightopenid": "dev-master", "lightopenid/lightopenid": "dev-master",

2
composer.lock generated
View file

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"content-hash": "6dda88f5629c38f3d07b31e13ded57aa", "content-hash": "7d6dee6e449da931e8fe209e61b2e78e",
"packages": [ "packages": [
{ {
"name": "asika/simple-console", "name": "asika/simple-console",

View file

@ -1195,7 +1195,7 @@ class BBCode
if (is_null($text)) { if (is_null($text)) {
$curlResult = DI::httpRequest()->head($match[1], ['timeout' => DI::config()->get('system', 'xrd_timeout')]); $curlResult = DI::httpRequest()->head($match[1], ['timeout' => DI::config()->get('system', 'xrd_timeout')]);
if ($curlResult->isSuccess()) { if ($curlResult->isSuccess()) {
$mimetype = $curlResult->getHeader('Content-Type'); $mimetype = $curlResult->getHeader('Content-Type')[0] ?? '';
} else { } else {
$mimetype = ''; $mimetype = '';
} }
@ -1266,7 +1266,7 @@ class BBCode
$curlResult = DI::httpRequest()->head($match[1], ['timeout' => DI::config()->get('system', 'xrd_timeout')]); $curlResult = DI::httpRequest()->head($match[1], ['timeout' => DI::config()->get('system', 'xrd_timeout')]);
if ($curlResult->isSuccess()) { if ($curlResult->isSuccess()) {
$mimetype = $curlResult->getHeader('Content-Type'); $mimetype = $curlResult->getHeader('Content-Type')[0] ?? '';
} else { } else {
$mimetype = ''; $mimetype = '';
} }

View file

@ -32,7 +32,7 @@ use Friendica\Database\Database;
use Friendica\Database\DBA; use Friendica\Database\DBA;
use Friendica\DI; use Friendica\DI;
use Friendica\Module\Register; use Friendica\Module\Register;
use Friendica\Network\CurlResult; use Friendica\Network\IHTTPResult;
use Friendica\Protocol\Relay; use Friendica\Protocol\Relay;
use Friendica\Util\DateTimeFormat; use Friendica\Util\DateTimeFormat;
use Friendica\Util\Network; use Friendica\Util\Network;
@ -672,17 +672,18 @@ class GServer
* Detect server type by using the nodeinfo data * Detect server type by using the nodeinfo data
* *
* @param string $url address of the server * @param string $url address of the server
* @param CurlResult $curlResult * @param IHTTPResult $httpResult
*
* @return array Server data * @return array Server data
* @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/ */
private static function fetchNodeinfo(string $url, CurlResult $curlResult) private static function fetchNodeinfo(string $url, IHTTPResult $httpResult)
{ {
if (!$curlResult->isSuccess()) { if (!$httpResult->isSuccess()) {
return []; return [];
} }
$nodeinfo = json_decode($curlResult->getBody(), true); $nodeinfo = json_decode($httpResult->getBody(), true);
if (!is_array($nodeinfo) || empty($nodeinfo['links'])) { if (!is_array($nodeinfo) || empty($nodeinfo['links'])) {
return []; return [];

View file

@ -495,6 +495,7 @@ class Photo
$type = $ret->getContentType(); $type = $ret->getContentType();
} else { } else {
$img_str = ''; $img_str = '';
$type = '';
} }
if ($quit_on_error && ($img_str == "")) { if ($quit_on_error && ($img_str == "")) {

View file

@ -90,7 +90,7 @@ class Link
$curlResult = DI::httpRequest()->head($url, ['timeout' => $timeout]); $curlResult = DI::httpRequest()->head($url, ['timeout' => $timeout]);
if ($curlResult->isSuccess()) { if ($curlResult->isSuccess()) {
if (empty($media['mimetype'])) { if (empty($media['mimetype'])) {
return $curlResult->getHeader('Content-Type'); return $curlResult->getHeader('Content-Type')[0] ?? '';
} }
} }
return ''; return '';

View file

@ -170,10 +170,10 @@ class Media
$curlResult = DI::httpRequest()->head($media['url'], ['timeout' => $timeout]); $curlResult = DI::httpRequest()->head($media['url'], ['timeout' => $timeout]);
if ($curlResult->isSuccess()) { if ($curlResult->isSuccess()) {
if (empty($media['mimetype'])) { if (empty($media['mimetype'])) {
$media['mimetype'] = $curlResult->getHeader('Content-Type'); $media['mimetype'] = $curlResult->getHeader('Content-Type')[0] ?? '';
} }
if (empty($media['size'])) { if (empty($media['size'])) {
$media['size'] = (int)$curlResult->getHeader('Content-Length'); $media['size'] = (int)($curlResult->getHeader('Content-Length')[0] ?? 0);
} }
} else { } else {
Logger::notice('Could not fetch head', ['media' => $media]); Logger::notice('Could not fetch head', ['media' => $media]);

View file

@ -29,7 +29,7 @@ use Friendica\Util\Network;
/** /**
* A content class for Curl call results * A content class for Curl call results
*/ */
class CurlResult class CurlResult implements IHTTPResult
{ {
/** /**
* @var int HTTP return code or 0 if timeout or failure * @var int HTTP return code or 0 if timeout or failure
@ -101,7 +101,7 @@ class CurlResult
* *
* @param string $url optional URL * @param string $url optional URL
* *
* @return CurlResult a CURL with error response * @return IHTTPResult a CURL with error response
* @throws InternalServerErrorException * @throws InternalServerErrorException
*/ */
public static function createErrorCurl($url = '') public static function createErrorCurl($url = '')
@ -229,57 +229,43 @@ class CurlResult
} }
} }
/** /** {@inheritDoc} */
* Gets the Curl Code
*
* @return string The Curl Code
*/
public function getReturnCode() public function getReturnCode()
{ {
return $this->returnCode; return $this->returnCode;
} }
/** /** {@inheritDoc} */
* Returns the Curl Content Type
*
* @return string the Curl Content Type
*/
public function getContentType() public function getContentType()
{ {
return $this->contentType; return $this->contentType;
} }
/** /** {@inheritDoc} */
* Returns the Curl headers public function getHeader($header)
*
* @param string $field optional header field. Return all fields if empty
*
* @return string the Curl headers or the specified content of the header variable
*/
public function getHeader(string $field = '')
{ {
if (empty($field)) { if (empty($header)) {
return $this->header; return [];
} }
$field = strtolower(trim($field)); $header = strtolower(trim($header));
$headers = $this->getHeaderArray(); $headers = $this->getHeaderArray();
if (isset($headers[$field])) { if (isset($headers[$header])) {
return $headers[$field]; return $headers[$header];
} }
return ''; return [];
} }
/** /** {@inheritDoc} */
* Check if a specified header exists public function getHeaders()
* {
* @param string $field header field return $this->getHeaderArray();
* }
* @return boolean "true" if header exists
*/ /** {@inheritDoc} */
public function inHeader(string $field) public function inHeader(string $field)
{ {
$field = strtolower(trim($field)); $field = strtolower(trim($field));
@ -289,11 +275,7 @@ class CurlResult
return array_key_exists($field, $headers); return array_key_exists($field, $headers);
} }
/** /** {@inheritDoc} */
* Returns the Curl headers as an associated array
*
* @return array associated header array
*/
public function getHeaderArray() public function getHeaderArray()
{ {
if (!empty($this->header_fields)) { if (!empty($this->header_fields)) {
@ -307,79 +289,59 @@ class CurlResult
$parts = explode(':', $line); $parts = explode(':', $line);
$headerfield = strtolower(trim(array_shift($parts))); $headerfield = strtolower(trim(array_shift($parts)));
$headerdata = trim(implode(':', $parts)); $headerdata = trim(implode(':', $parts));
$this->header_fields[$headerfield] = $headerdata; if (empty($this->header_fields[$headerfield])) {
$this->header_fields[$headerfield] = [$headerdata];
} elseif (!in_array($headerdata, $this->header_fields[$headerfield])) {
$this->header_fields[$headerfield][] = $headerdata;
}
} }
return $this->header_fields; return $this->header_fields;
} }
/** /** {@inheritDoc} */
* @return bool
*/
public function isSuccess() public function isSuccess()
{ {
return $this->isSuccess; return $this->isSuccess;
} }
/** /** {@inheritDoc} */
* @return string
*/
public function getUrl() public function getUrl()
{ {
return $this->url; return $this->url;
} }
/** /** {@inheritDoc} */
* @return string
*/
public function getRedirectUrl() public function getRedirectUrl()
{ {
return $this->redirectUrl; return $this->redirectUrl;
} }
/** /** {@inheritDoc} */
* @return string
*/
public function getBody() public function getBody()
{ {
return $this->body; return $this->body;
} }
/** /** {@inheritDoc} */
* @return array
*/
public function getInfo()
{
return $this->info;
}
/**
* @return bool
*/
public function isRedirectUrl() public function isRedirectUrl()
{ {
return $this->isRedirectUrl; return $this->isRedirectUrl;
} }
/** /** {@inheritDoc} */
* @return int
*/
public function getErrorNumber() public function getErrorNumber()
{ {
return $this->errorNumber; return $this->errorNumber;
} }
/** /** {@inheritDoc} */
* @return string
*/
public function getError() public function getError()
{ {
return $this->error; return $this->error;
} }
/** /** {@inheritDoc} */
* @return bool
*/
public function isTimeout() public function isTimeout()
{ {
return $this->isTimeout; return $this->isTimeout;

View file

@ -0,0 +1,154 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Network;
use Friendica\Core\Logger;
use Friendica\Core\System;
use Friendica\Network\HTTPException\NotImplementedException;
use GuzzleHttp\Psr7\Response;
use Psr\Http\Message\ResponseInterface;
/**
* A content wrapper class for Guzzle call results
*/
class GuzzleResponse extends Response implements IHTTPResult, ResponseInterface
{
/** @var string The URL */
private $url;
/** @var boolean */
private $isTimeout;
/** @var boolean */
private $isSuccess;
/**
* @var int the error number or 0 (zero) if no error
*/
private $errorNumber;
/**
* @var string the error message or '' (the empty string) if no
*/
private $error;
public function __construct(ResponseInterface $response, string $url, $errorNumber = 0, $error = '')
{
parent::__construct($response->getStatusCode(), $response->getHeaders(), $response->getBody(), $response->getProtocolVersion(), $response->getReasonPhrase());
$this->url = $url;
$this->error = $error;
$this->errorNumber = $errorNumber;
$this->checkSuccess();
}
private function checkSuccess()
{
$this->isSuccess = ($this->getStatusCode() >= 200 && $this->getStatusCode() <= 299) || $this->errorNumber == 0;
// Everything higher or equal 400 is not a success
if ($this->getReturnCode() >= 400) {
$this->isSuccess = false;
}
if (!$this->isSuccess) {
Logger::notice('http error', ['url' => $this->url, 'code' => $this->getReturnCode(), 'error' => $this->error, 'callstack' => System::callstack(20)]);
Logger::debug('debug', ['info' => $this->getHeaders()]);
}
if (!$this->isSuccess && $this->errorNumber == CURLE_OPERATION_TIMEDOUT) {
$this->isTimeout = true;
} else {
$this->isTimeout = false;
}
}
/** {@inheritDoc} */
public function getReturnCode()
{
return $this->getStatusCode();
}
/** {@inheritDoc} */
public function getContentType()
{
$contentTypes = $this->getHeader('Content-Type') ?? [];
return array_pop($contentTypes) ?? '';
}
/** {@inheritDoc} */
public function inHeader(string $field)
{
return $this->hasHeader($field);
}
/** {@inheritDoc} */
public function getHeaderArray()
{
return $this->getHeaders();
}
/** {@inheritDoc} */
public function isSuccess()
{
return $this->isSuccess;
}
/** {@inheritDoc} */
public function getUrl()
{
return $this->url;
}
/** {@inheritDoc} */
public function getRedirectUrl()
{
return $this->url;
}
/** {@inheritDoc} */
public function isRedirectUrl()
{
throw new NotImplementedException();
}
/** {@inheritDoc} */
public function getErrorNumber()
{
return $this->errorNumber;
}
/** {@inheritDoc} */
public function getError()
{
return $this->error;
}
/** {@inheritDoc} */
public function isTimeout()
{
return $this->isTimeout;
}
/// @todo - fix mismatching use of "getBody()" as string here and parent "getBody()" as streaminterface
public function getBody()
{
return parent::getBody()->getContents();
}
}

View file

@ -28,6 +28,12 @@ use Friendica\Core\Config\IConfig;
use Friendica\Core\System; use Friendica\Core\System;
use Friendica\Util\Network; use Friendica\Util\Network;
use Friendica\Util\Profiler; use Friendica\Util\Profiler;
use GuzzleHttp\Client;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Exception\TransferException;
use Psr\Http\Message\RequestInterface;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\UriInterface;
use Psr\Log\LoggerInterface; use Psr\Log\LoggerInterface;
/** /**
@ -65,12 +71,8 @@ class HTTPRequest implements IHTTPRequest
/** /**
* {@inheritDoc} * {@inheritDoc}
*
* @param int $redirects The recursion counter for internal use - default 0
*
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/ */
public function get(string $url, array $opts = [], &$redirects = 0) public function get(string $url, array $opts = [])
{ {
$this->profiler->startRecording('network'); $this->profiler->startRecording('network');
@ -103,120 +105,127 @@ class HTTPRequest implements IHTTPRequest
return CurlResult::createErrorCurl($url); return CurlResult::createErrorCurl($url);
} }
$ch = @curl_init($url); $curlOptions = [];
if (($redirects > 8) || (!$ch)) {
$this->profiler->stopRecording();
return CurlResult::createErrorCurl($url);
}
@curl_setopt($ch, CURLOPT_HEADER, true);
if (!empty($opts['cookiejar'])) { if (!empty($opts['cookiejar'])) {
curl_setopt($ch, CURLOPT_COOKIEJAR, $opts["cookiejar"]); $curlOptions[CURLOPT_COOKIEJAR] = $opts["cookiejar"];
curl_setopt($ch, CURLOPT_COOKIEFILE, $opts["cookiejar"]); $curlOptions[CURLOPT_COOKIEFILE] = $opts["cookiejar"];
} }
// These settings aren't needed. We're following the location already. // These settings aren't needed. We're following the location already.
// @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); // $curlOptions[CURLOPT_FOLLOWLOCATION] =true;
// @curl_setopt($ch, CURLOPT_MAXREDIRS, 5); // $curlOptions[CURLOPT_MAXREDIRS] = 5;
$curlOptions[CURLOPT_HTTPHEADER] = [];
if (!empty($opts['accept_content'])) { if (!empty($opts['accept_content'])) {
curl_setopt( array_push($curlOptions[CURLOPT_HTTPHEADER], 'Accept: ' . $opts['accept_content']);
$ch,
CURLOPT_HTTPHEADER,
['Accept: ' . $opts['accept_content']]
);
} }
if (!empty($opts['header'])) { if (!empty($opts['header'])) {
curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['header']); $curlOptions[CURLOPT_HTTPHEADER] = array_merge($opts['header'], $curlOptions[CURLOPT_HTTPHEADER]);
} }
@curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); $curlOptions[CURLOPT_RETURNTRANSFER] = true;
@curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent()); $curlOptions[CURLOPT_USERAGENT] = $this->getUserAgent();
$range = intval($this->config->get('system', 'curl_range_bytes', 0)); $range = intval($this->config->get('system', 'curl_range_bytes', 0));
if ($range > 0) { if ($range > 0) {
@curl_setopt($ch, CURLOPT_RANGE, '0-' . $range); $curlOptions[CURLOPT_RANGE] = '0-' . $range;
} }
// Without this setting it seems as if some webservers send compressed content // Without this setting it seems as if some webservers send compressed content
// This seems to confuse curl so that it shows this uncompressed. // This seems to confuse curl so that it shows this uncompressed.
/// @todo We could possibly set this value to "gzip" or something similar /// @todo We could possibly set this value to "gzip" or something similar
curl_setopt($ch, CURLOPT_ENCODING, ''); $curlOptions[CURLOPT_ENCODING] = '';
if (!empty($opts['headers'])) { if (!empty($opts['headers'])) {
$this->logger->notice('Wrong option \'headers\' used.'); $this->logger->notice('Wrong option \'headers\' used.');
@curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['headers']); $curlOptions[CURLOPT_HTTPHEADER] = array_merge($opts['headers'], $curlOptions[CURLOPT_HTTPHEADER]);
} }
if (!empty($opts['nobody'])) { if (!empty($opts['nobody'])) {
@curl_setopt($ch, CURLOPT_NOBODY, $opts['nobody']); $curlOptions[CURLOPT_NOBODY] = $opts['nobody'];
} }
@curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10); $curlOptions[CURLOPT_CONNECTTIMEOUT] = 10;
if (!empty($opts['timeout'])) { if (!empty($opts['timeout'])) {
@curl_setopt($ch, CURLOPT_TIMEOUT, $opts['timeout']); $curlOptions[CURLOPT_TIMEOUT] = $opts['timeout'];
} else { } else {
$curl_time = $this->config->get('system', 'curl_timeout', 60); $curl_time = $this->config->get('system', 'curl_timeout', 60);
@curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time)); $curlOptions[CURLOPT_TIMEOUT] = intval($curl_time);
} }
// by default we will allow self-signed certs // by default we will allow self-signed certs
// but you can override this // but you can override this
$check_cert = $this->config->get('system', 'verifyssl'); $check_cert = $this->config->get('system', 'verifyssl');
@curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false)); $curlOptions[CURLOPT_SSL_VERIFYPEER] = ($check_cert) ? true : false;
if ($check_cert) { if ($check_cert) {
@curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); $curlOptions[CURLOPT_SSL_VERIFYHOST] = 2;
} }
$proxy = $this->config->get('system', 'proxy'); $proxy = $this->config->get('system', 'proxy');
if (!empty($proxy)) { if (!empty($proxy)) {
@curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1); $curlOptions[CURLOPT_HTTPPROXYTUNNEL] = 1;
@curl_setopt($ch, CURLOPT_PROXY, $proxy); $curlOptions[CURLOPT_PROXY] = $proxy;
$proxyuser = $this->config->get('system', 'proxyuser'); $proxyuser = $this->config->get('system', 'proxyuser');
if (!empty($proxyuser)) { if (!empty($proxyuser)) {
@curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser); $curlOptions[CURLOPT_PROXYUSERPWD] = $proxyuser;
} }
} }
if ($this->config->get('system', 'ipv4_resolve', false)) { if ($this->config->get('system', 'ipv4_resolve', false)) {
curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); $curlOptions[CURLOPT_IPRESOLVE] = CURL_IPRESOLVE_V4;
} }
$s = @curl_exec($ch); $logger = $this->logger;
$curl_info = @curl_getinfo($ch);
// Special treatment for HTTP Code 416 $onRedirect = function(
// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/416 RequestInterface $request,
if (($curl_info['http_code'] == 416) && ($range > 0)) { ResponseInterface $response,
@curl_setopt($ch, CURLOPT_RANGE, ''); UriInterface $uri
$s = @curl_exec($ch); ) use ($logger) {
$curl_info = @curl_getinfo($ch); $logger->notice('Curl redirect.', ['url' => $request->getUri(), 'to' => $uri]);
};
$onHeaders = function (ResponseInterface $response) use ($opts) {
if (!empty($opts['content_length']) &&
$response->getHeaderLine('Content-Length') > $opts['content_length']) {
throw new TransferException('The file is too big!');
} }
};
$curlResponse = new CurlResult($url, $s, $curl_info, curl_errno($ch), curl_error($ch)); $client = new Client([
'allow_redirect' => [
'max' => 8,
'on_redirect' => $onRedirect,
'track_redirect' => true,
'strict' => true,
'referer' => true,
],
'on_headers' => $onHeaders,
'curl' => $curlOptions
]);
if (!Network::isRedirectBlocked($url) && $curlResponse->isRedirectUrl()) { try {
$redirects++; $response = $client->get($url);
$this->logger->notice('Curl redirect.', ['url' => $url, 'to' => $curlResponse->getRedirectUrl()]); return new GuzzleResponse($response, $url);
@curl_close($ch); } catch (TransferException $exception) {
if ($exception instanceof RequestException &&
$exception->hasResponse()) {
return new GuzzleResponse($exception->getResponse(), $url, $exception->getCode(), '');
} else {
return new CurlResult($url, '', ['http_code' => $exception->getCode()], $exception->getCode(), '');
}
} finally {
$this->profiler->stopRecording(); $this->profiler->stopRecording();
return $this->get($curlResponse->getRedirectUrl(), $opts, $redirects);
} }
@curl_close($ch);
$this->profiler->stopRecording();
return $curlResponse;
} }
/** /**
@ -454,26 +463,18 @@ class HTTPRequest implements IHTTPRequest
/** /**
* {@inheritDoc} * {@inheritDoc}
*
* @param int $redirects The recursion counter for internal use - default 0
*
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/ */
public function fetch(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '', &$redirects = 0) public function fetch(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '')
{ {
$ret = $this->fetchFull($url, $timeout, $accept_content, $cookiejar, $redirects); $ret = $this->fetchFull($url, $timeout, $accept_content, $cookiejar);
return $ret->getBody(); return $ret->getBody();
} }
/** /**
* {@inheritDoc} * {@inheritDoc}
*
* @param int $redirects The recursion counter for internal use - default 0
*
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
*/ */
public function fetchFull(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '', &$redirects = 0) public function fetchFull(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '')
{ {
return $this->get( return $this->get(
$url, $url,
@ -481,8 +482,7 @@ class HTTPRequest implements IHTTPRequest
'timeout' => $timeout, 'timeout' => $timeout,
'accept_content' => $accept_content, 'accept_content' => $accept_content,
'cookiejar' => $cookiejar 'cookiejar' => $cookiejar
], ]
$redirects
); );
} }

View file

@ -52,7 +52,7 @@ interface IHTTPRequest
* @param string $accept_content supply Accept: header with 'accept_content' as the value * @param string $accept_content supply Accept: header with 'accept_content' as the value
* @param string $cookiejar Path to cookie jar file * @param string $cookiejar Path to cookie jar file
* *
* @return CurlResult With all relevant information, 'body' contains the actual fetched content. * @return IHTTPResult With all relevant information, 'body' contains the actual fetched content.
*/ */
public function fetchFull(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = ''); public function fetchFull(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = '');
@ -79,8 +79,9 @@ interface IHTTPRequest
* 'timeout' => int Timeout in seconds, default system config value or 60 seconds * 'timeout' => int Timeout in seconds, default system config value or 60 seconds
* 'cookiejar' => path to cookie jar file * 'cookiejar' => path to cookie jar file
* 'header' => header array * 'header' => header array
* 'content_length' => int maximum File content length
* *
* @return CurlResult * @return IHTTPResult
*/ */
public function get(string $url, array $opts = []); public function get(string $url, array $opts = []);
@ -92,7 +93,7 @@ interface IHTTPRequest
* @param array $headers HTTP headers * @param array $headers HTTP headers
* @param int $timeout The timeout in seconds, default system config value or 60 seconds * @param int $timeout The timeout in seconds, default system config value or 60 seconds
* *
* @return CurlResult The content * @return IHTTPResult The content
*/ */
public function post(string $url, $params, array $headers = [], int $timeout = 0); public function post(string $url, $params, array $headers = [], int $timeout = 0);

104
src/Network/IHTTPResult.php Normal file
View file

@ -0,0 +1,104 @@
<?php
namespace Friendica\Network;
use Psr\Http\Message\MessageInterface;
/**
* Temporary class to map Friendica used variables based on PSR-7 HTTPResponse
*/
interface IHTTPResult
{
/**
* Gets the Return Code
*
* @return string The Return Code
*/
public function getReturnCode();
/**
* Returns the Content Type
*
* @return string the Content Type
*/
public function getContentType();
/**
* Returns the headers
* @see MessageInterface::getHeader()
*
* @param string $header optional header field. Return all fields if empty
*
* @return string[] the headers or the specified content of the header variable
*/
public function getHeader($header);
/**
* Returns all headers
* @see MessageInterface::getHeaders()
*
* @return string[][]
*/
public function getHeaders();
/**
* Check if a specified header exists
* @see MessageInterface::hasHeader()
*
* @param string $field header field
*
* @return boolean "true" if header exists
*/
public function inHeader(string $field);
/**
* Returns the headers as an associated array
* @see MessageInterface::getHeaders()
* @deprecated
*
* @return string[][] associated header array
*/
public function getHeaderArray();
/**
* @return bool
*/
public function isSuccess();
/**
* @return string
*/
public function getUrl();
/**
* @return string
*/
public function getRedirectUrl();
/**
* @see MessageInterface::getBody()
*
* @return string
*/
public function getBody();
/**
* @return boolean
*/
public function isRedirectUrl();
/**
* @return integer
*/
public function getErrorNumber();
/**
* @return string
*/
public function getError();
/**
* @return boolean
*/
public function isTimeout();
}

View file

@ -424,16 +424,11 @@ class Probe
*/ */
private static function getHideStatus($url) private static function getHideStatus($url)
{ {
$curlResult = DI::httpRequest()->get($url); $curlResult = DI::httpRequest()->get($url, ['content_length' => 1000000]);
if (!$curlResult->isSuccess()) { if (!$curlResult->isSuccess()) {
return false; return false;
} }
// If the file is too large then exit
if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) {
return false;
}
// If it isn't a HTML file then exit // If it isn't a HTML file then exit
if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) { if (($curlResult->getContentType() != '') && !strstr(strtolower($curlResult->getContentType()), 'html')) {
return false; return false;

View file

@ -985,7 +985,7 @@ class DFRN
return -9; // timed out return -9; // timed out
} }
if (($curl_stat == 503) && (stristr($postResult->getHeader(), 'retry-after'))) { if (($curl_stat == 503) && $postResult->inHeader('retry-after')) {
return -10; return -10;
} }

View file

@ -735,7 +735,8 @@ class OStatus
$xml = ''; $xml = '';
if (stristr($curlResult->getHeader(), 'Content-Type: application/atom+xml')) { if ($curlResult->inHeader('Content-Type') &&
in_array('application/atom+xml', $curlResult->getHeader('Content-Type'))) {
$xml = $curlResult->getBody(); $xml = $curlResult->getBody();
} }
@ -928,7 +929,8 @@ class OStatus
$xml = ''; $xml = '';
if (stristr($curlResult->getHeader(), 'Content-Type: application/atom+xml')) { if ($curlResult->inHeader('Content-Type') &&
in_array('application/atom+xml', $curlResult->getHeader('Content-Type'))) {
Logger::log('Directly fetched XML for URI ' . $related_uri, Logger::DEBUG); Logger::log('Directly fetched XML for URI ' . $related_uri, Logger::DEBUG);
$xml = $curlResult->getBody(); $xml = $curlResult->getBody();
} }

View file

@ -215,7 +215,7 @@ class Salmon
return -1; return -1;
} }
if (($return_code == 503) && (stristr($postResult->getHeader(), 'retry-after'))) { if (($return_code == 503) && $postResult->inHeader('retry-after')) {
return -1; return -1;
} }

View file

@ -63,7 +63,7 @@ class ParseUrl
return []; return [];
} }
$contenttype = $curlResult->getHeader('Content-Type'); $contenttype = $curlResult->getHeader('Content-Type')[0] ?? '';
if (empty($contenttype)) { if (empty($contenttype)) {
return []; return [];
} }
@ -213,26 +213,20 @@ class ParseUrl
return $siteinfo; return $siteinfo;
} }
$curlResult = DI::httpRequest()->get($url); $curlResult = DI::httpRequest()->get($url, ['content_length' => 1000000]);
if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
return $siteinfo; return $siteinfo;
} }
$siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS); $siteinfo['expires'] = DateTimeFormat::utc(self::DEFAULT_EXPIRATION_SUCCESS);
// If the file is too large then exit if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')[0] ?? '') {
if (($curlResult->getInfo()['download_content_length'] ?? 0) > 1000000) {
return $siteinfo;
}
if ($cacheControlHeader = $curlResult->getHeader('Cache-Control')) {
if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) { if (preg_match('/max-age=([0-9]+)/i', $cacheControlHeader, $matches)) {
$maxAge = max(86400, (int)array_pop($matches)); $maxAge = max(86400, (int)array_pop($matches));
$siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds"); $siteinfo['expires'] = DateTimeFormat::utc("now + $maxAge seconds");
} }
} }
$header = $curlResult->getHeader();
$body = $curlResult->getBody(); $body = $curlResult->getBody();
if ($do_oembed) { if ($do_oembed) {
@ -273,7 +267,7 @@ class ParseUrl
$charset = ''; $charset = '';
// Look for a charset, first in headers // Look for a charset, first in headers
// Expected form: Content-Type: text/html; charset=ISO-8859-4 // Expected form: Content-Type: text/html; charset=ISO-8859-4
if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $header, $matches)) { if (preg_match('/charset=([a-z0-9-_.\/]+)/i', $curlResult->getContentType(), $matches)) {
$charset = trim(trim(trim(array_pop($matches)), ';,')); $charset = trim(trim(trim(array_pop($matches)), ';,'));
} }

View file

@ -0,0 +1,20 @@
<?php
return [
'http/2 200' => [''],
'date' => ['Thu, 11 Oct 2018 18:43:54 GMT'],
'content-type' => ['text/html; charset=utf-8'],
'vary' => ['Accept-Encoding'],
'server' => ['Mastodon'],
'x-frame-options' => ['DENY', 'SAMEORIGIN'],
'x-content-type-options' => ['nosniff'],
'x-xss-protection' => ['1; mode=block'],
'etag' => ['W/"706e6c48957e1d46ecf9d7597a7880af"'],
'cache-control' => ['max-age=0, private, must-revalidate'],
'set-cookie' => ['_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly'],
'x-request-id' => ['a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784'],
'x-runtime' => ['0.049566'],
'strict-transport-security' => ['max-age=31536000; includeSubDomains; preload'],
'referrer-policy' => ['same-origin'],
'content-security-policy' => ["frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de"],
];

View file

@ -0,0 +1,21 @@
<?php
return [
'http/2 301' => [''],
'date' => ['Thu, 11 Oct 2018 18:43:54 GMT'],
'content-type' => ['text/html; charset=utf-8'],
'vary' => ['Accept-Encoding'],
'server' => ['Mastodon'],
'location' => ['https://test.other/some/'],
'x-frame-options' => ['DENY', 'SAMEORIGIN'],
'x-content-type-options' => ['nosniff'],
'x-xss-protection' => ['1; mode=block'],
'etag' => ['W/"706e6c48957e1d46ecf9d7597a7880af"'],
'cache-control' => ['max-age=0, private, must-revalidate'],
'set-cookie' => ['_mastodon_session=v3kcy%2FW3aZYBBvZUohuwksEKwzYIyEUlEuJ1KqTAfWPKvVQq%2F4UuJ39zp621VyfpQNlvY46TL%2FYutzXowSLYQBNFCJcrEiF04aU0TdtHls9zynMiyeHhoVgCijOXWXNt9%2FCmpQ49RkNEujkv9NaJ0cum32MCVZKjE9%2BMKmLM%2F8ZygZeLBGJ7sg%3D%3D--QGIiU0%2FpXc3Aym8F--he2iRRPePOdtEs3z%2BufSXg%3D%3D; path=/; secure; HttpOnly'],
'x-request-id' => ['a0c0b8e7-cd60-4efa-b79b-cf1b0d5a0784'],
'x-runtime' => ['0.049566'],
'strict-transport-security' => ['max-age=31536000; includeSubDomains; preload'],
'referrer-policy' => ['same-origin'],
'content-security-policy' => ["frame-ancestors 'none'; script-src 'self'; object-src 'self'; img-src * data: blob:; media-src 'self' data:; font-src 'self' data: https://fonts.gstatic.com/; connect-src 'self' blob: wss://mastodonten.de"],
];

View file

@ -25,7 +25,7 @@ namespace Friendica\Core;
use Dice\Dice; use Dice\Dice;
use Friendica\Core\Config\Cache; use Friendica\Core\Config\Cache;
use Friendica\DI; use Friendica\DI;
use Friendica\Network\CurlResult; use Friendica\Network\IHTTPResult;
use Friendica\Network\IHTTPRequest; use Friendica\Network\IHTTPRequest;
use Friendica\Test\MockedTest; use Friendica\Test\MockedTest;
use Friendica\Test\Util\VFSTrait; use Friendica\Test\Util\VFSTrait;
@ -319,14 +319,14 @@ class InstallerTest extends MockedTest
$this->l10nMock->shouldReceive('t')->andReturnUsing(function ($args) { return $args; }); $this->l10nMock->shouldReceive('t')->andReturnUsing(function ($args) { return $args; });
// Mocking the CURL Response // Mocking the CURL Response
$curlResult = Mockery::mock(CurlResult::class); $IHTTPResult = Mockery::mock(IHTTPResult::class);
$curlResult $IHTTPResult
->shouldReceive('getReturnCode') ->shouldReceive('getReturnCode')
->andReturn('404'); ->andReturn('404');
$curlResult $IHTTPResult
->shouldReceive('getRedirectUrl') ->shouldReceive('getRedirectUrl')
->andReturn(''); ->andReturn('');
$curlResult $IHTTPResult
->shouldReceive('getError') ->shouldReceive('getError')
->andReturn('test Error'); ->andReturn('test Error');
@ -335,11 +335,11 @@ class InstallerTest extends MockedTest
$networkMock $networkMock
->shouldReceive('fetchFull') ->shouldReceive('fetchFull')
->with('https://test/install/testrewrite') ->with('https://test/install/testrewrite')
->andReturn($curlResult); ->andReturn($IHTTPResult);
$networkMock $networkMock
->shouldReceive('fetchFull') ->shouldReceive('fetchFull')
->with('http://test/install/testrewrite') ->with('http://test/install/testrewrite')
->andReturn($curlResult); ->andReturn($IHTTPResult);
$this->dice->shouldReceive('create') $this->dice->shouldReceive('create')
->with(IHTTPRequest::class) ->with(IHTTPRequest::class)
@ -366,14 +366,14 @@ class InstallerTest extends MockedTest
$this->l10nMock->shouldReceive('t')->andReturnUsing(function ($args) { return $args; }); $this->l10nMock->shouldReceive('t')->andReturnUsing(function ($args) { return $args; });
// Mocking the failed CURL Response // Mocking the failed CURL Response
$curlResultF = Mockery::mock(CurlResult::class); $IHTTPResultF = Mockery::mock(IHTTPResult::class);
$curlResultF $IHTTPResultF
->shouldReceive('getReturnCode') ->shouldReceive('getReturnCode')
->andReturn('404'); ->andReturn('404');
// Mocking the working CURL Response // Mocking the working CURL Response
$curlResultW = Mockery::mock(CurlResult::class); $IHTTPResultW = Mockery::mock(IHTTPResult::class);
$curlResultW $IHTTPResultW
->shouldReceive('getReturnCode') ->shouldReceive('getReturnCode')
->andReturn('204'); ->andReturn('204');
@ -382,11 +382,11 @@ class InstallerTest extends MockedTest
$networkMock $networkMock
->shouldReceive('fetchFull') ->shouldReceive('fetchFull')
->with('https://test/install/testrewrite') ->with('https://test/install/testrewrite')
->andReturn($curlResultF); ->andReturn($IHTTPResultF);
$networkMock $networkMock
->shouldReceive('fetchFull') ->shouldReceive('fetchFull')
->with('http://test/install/testrewrite') ->with('http://test/install/testrewrite')
->andReturn($curlResultW); ->andReturn($IHTTPResultW);
$this->dice->shouldReceive('create') $this->dice->shouldReceive('create')
->with(IHTTPRequest::class) ->with(IHTTPRequest::class)

View file

@ -53,6 +53,7 @@ class CurlResultTest extends TestCase
public function testNormal() public function testNormal()
{ {
$header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head'); $header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head');
$headerArray = include(__DIR__ . '/../../datasets/curl/about.head.php');
$body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body'); $body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body');
@ -65,7 +66,7 @@ class CurlResultTest extends TestCase
self::assertTrue($curlResult->isSuccess()); self::assertTrue($curlResult->isSuccess());
self::assertFalse($curlResult->isTimeout()); self::assertFalse($curlResult->isTimeout());
self::assertFalse($curlResult->isRedirectUrl()); self::assertFalse($curlResult->isRedirectUrl());
self::assertSame($header, $curlResult->getHeader()); self::assertSame($headerArray, $curlResult->getHeaders());
self::assertSame($body, $curlResult->getBody()); self::assertSame($body, $curlResult->getBody());
self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('text/html; charset=utf-8', $curlResult->getContentType());
self::assertSame('https://test.local', $curlResult->getUrl()); self::assertSame('https://test.local', $curlResult->getUrl());
@ -80,6 +81,7 @@ class CurlResultTest extends TestCase
public function testRedirect() public function testRedirect()
{ {
$header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head'); $header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head');
$headerArray = include(__DIR__ . '/../../datasets/curl/about.head.php');
$body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body'); $body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body');
@ -93,7 +95,7 @@ class CurlResultTest extends TestCase
self::assertTrue($curlResult->isSuccess()); self::assertTrue($curlResult->isSuccess());
self::assertFalse($curlResult->isTimeout()); self::assertFalse($curlResult->isTimeout());
self::assertTrue($curlResult->isRedirectUrl()); self::assertTrue($curlResult->isRedirectUrl());
self::assertSame($header, $curlResult->getHeader()); self::assertSame($headerArray, $curlResult->getHeaders());
self::assertSame($body, $curlResult->getBody()); self::assertSame($body, $curlResult->getBody());
self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('text/html; charset=utf-8', $curlResult->getContentType());
self::assertSame('https://test.local/test/it', $curlResult->getUrl()); self::assertSame('https://test.local/test/it', $curlResult->getUrl());
@ -106,6 +108,7 @@ class CurlResultTest extends TestCase
public function testTimeout() public function testTimeout()
{ {
$header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head'); $header = file_get_contents(__DIR__ . '/../../datasets/curl/about.head');
$headerArray = include(__DIR__ . '/../../datasets/curl/about.head.php');
$body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body'); $body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body');
@ -119,7 +122,7 @@ class CurlResultTest extends TestCase
self::assertFalse($curlResult->isSuccess()); self::assertFalse($curlResult->isSuccess());
self::assertTrue($curlResult->isTimeout()); self::assertTrue($curlResult->isTimeout());
self::assertFalse($curlResult->isRedirectUrl()); self::assertFalse($curlResult->isRedirectUrl());
self::assertSame($header, $curlResult->getHeader()); self::assertSame($headerArray, $curlResult->getHeaders());
self::assertSame($body, $curlResult->getBody()); self::assertSame($body, $curlResult->getBody());
self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('text/html; charset=utf-8', $curlResult->getContentType());
self::assertSame('https://test.local/test/it', $curlResult->getRedirectUrl()); self::assertSame('https://test.local/test/it', $curlResult->getRedirectUrl());
@ -134,6 +137,7 @@ class CurlResultTest extends TestCase
public function testRedirectHeader() public function testRedirectHeader()
{ {
$header = file_get_contents(__DIR__ . '/../../datasets/curl/about.redirect'); $header = file_get_contents(__DIR__ . '/../../datasets/curl/about.redirect');
$headerArray = include(__DIR__ . '/../../datasets/curl/about.redirect.php');
$body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body'); $body = file_get_contents(__DIR__ . '/../../datasets/curl/about.body');
@ -146,7 +150,7 @@ class CurlResultTest extends TestCase
self::assertTrue($curlResult->isSuccess()); self::assertTrue($curlResult->isSuccess());
self::assertFalse($curlResult->isTimeout()); self::assertFalse($curlResult->isTimeout());
self::assertTrue($curlResult->isRedirectUrl()); self::assertTrue($curlResult->isRedirectUrl());
self::assertSame($header, $curlResult->getHeader()); self::assertSame($headerArray, $curlResult->getHeaders());
self::assertSame($body, $curlResult->getBody()); self::assertSame($body, $curlResult->getBody());
self::assertSame('text/html; charset=utf-8', $curlResult->getContentType()); self::assertSame('text/html; charset=utf-8', $curlResult->getContentType());
self::assertSame('https://test.local/test/it?key=value', $curlResult->getUrl()); self::assertSame('https://test.local/test/it?key=value', $curlResult->getUrl());
@ -204,7 +208,7 @@ class CurlResultTest extends TestCase
'url' => 'https://test.local' 'url' => 'https://test.local'
]); ]);
self::assertNotEmpty($curlResult->getHeader()); self::assertNotEmpty($curlResult->getHeaders());
self::assertEmpty($curlResult->getHeader('wrongHeader')); self::assertEmpty($curlResult->getHeader('wrongHeader'));
} }
} }