From 61da51c2d51342bb338cba67191b50442488eeb3 Mon Sep 17 00:00:00 2001 From: Philipp Date: Sun, 18 Oct 2020 22:15:20 +0200 Subject: [PATCH 1/5] Add HTTPRequest::head() function --- mod/parse_url.php | 2 +- src/Network/HTTPRequest.php | 11 +++++++++++ src/Network/IHTTPRequest.php | 23 +++++++++++++++++------ src/Util/HTTPSignature.php | 6 +++++- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/mod/parse_url.php b/mod/parse_url.php index ede557ff2..3c931fdb1 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -85,7 +85,7 @@ function parse_url_content(App $a) // Check if the URL is an image, video or audio file. If so format // the URL with the corresponding BBCode media tag // Fetch the header of the URL - $curlResponse = DI::httpRequest()->get($url, ['novalidate' => true, 'nobody' => true]); + $curlResponse = DI::httpRequest()->head($url, ['novalidate' => true]); if ($curlResponse->isSuccess()) { // Convert the header fields into an array diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 644229527..df62ea454 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -52,6 +52,17 @@ class HTTPRequest implements IHTTPRequest $this->baseUrl = $baseUrl->get(); } + /** {@inheritDoc} + * + * @throws HTTPException\InternalServerErrorException + */ + public function head(string $url, array $opts = []) + { + $opts['nobody'] = true; + + return $this->get($url, $opts); + } + /** * {@inheritDoc} * diff --git a/src/Network/IHTTPRequest.php b/src/Network/IHTTPRequest.php index d6bf98105..a1c7730ca 100644 --- a/src/Network/IHTTPRequest.php +++ b/src/Network/IHTTPRequest.php @@ -29,12 +29,10 @@ interface IHTTPRequest /** * Fetches the content of an URL * - * If binary flag is true, return binary results. * Set the cookiejar argument to a string (e.g. "/tmp/friendica-cookies.txt") * to preserve cookies from one request to the next. * * @param string $url URL to fetch - * TRUE if asked to return binary results (file download) * @param int $timeout Timeout in seconds, default system config value or 60 seconds * @param string $accept_content supply Accept: header with 'accept_content' as the value * @param string $cookiejar Path to cookie jar file @@ -50,7 +48,6 @@ interface IHTTPRequest * all the information collected during the fetch. * * @param string $url URL to fetch - * TRUE if asked to return binary results (file download) * @param int $timeout Timeout in seconds, default system config value or 60 seconds * @param string $accept_content supply Accept: header with 'accept_content' as the value * @param string $cookiejar Path to cookie jar file @@ -60,16 +57,30 @@ interface IHTTPRequest public function fetchFull(string $url, int $timeout = 0, string $accept_content = '', string $cookiejar = ''); /** - * Send a GET to an URL. + * Send a HEAD to an URL. + * + * @param string $url URL to fetch + * @param array $opts (optional parameters) assoziative array with: + * 'accept_content' => supply Accept: header with 'accept_content' as the value + * 'timeout' => int Timeout in seconds, default system config value or 60 seconds + * 'http_auth' => username:password + * 'novalidate' => do not validate SSL certs, default is to validate using our CA list + * 'cookiejar' => path to cookie jar file + * 'header' => header array + * + * @return CurlResult + */ + public function head(string $url, array $opts = []); + + /** + * Send a GET to an URL. * * @param string $url URL to fetch - * TRUE if asked to return binary results (file download) * @param array $opts (optional parameters) assoziative array with: * 'accept_content' => supply Accept: header with 'accept_content' as the value * 'timeout' => int Timeout in seconds, default system config value or 60 seconds * 'http_auth' => username:password * 'novalidate' => do not validate SSL certs, default is to validate using our CA list - * 'nobody' => only return the header * 'cookiejar' => path to cookie jar file * 'header' => header array * diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index e3ed5d079..61e95a52c 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -449,7 +449,11 @@ class HTTPSignature $curl_opts = $opts; $curl_opts['header'] = $headers; - $curlResult = DI::httpRequest()->get($request, $curl_opts); + if ($opts['nobody']) { + $curlResult = DI::httpRequest()->head($request, $curl_opts); + } else { + $curlResult = DI::httpRequest()->get($request, $curl_opts); + } $return_code = $curlResult->getReturnCode(); Logger::log('Fetched for user ' . $uid . ' from ' . $request . ' returned ' . $return_code, Logger::DEBUG); From 54d78f9ce174eee354081426172cdd875bf15e3d Mon Sep 17 00:00:00 2001 From: Philipp Date: Sun, 18 Oct 2020 22:19:03 +0200 Subject: [PATCH 2/5] Removed completely un-used 'http_auth' option from HTTPRequest --- src/Network/IHTTPRequest.php | 2 -- src/Util/HTTPSignature.php | 1 - 2 files changed, 3 deletions(-) diff --git a/src/Network/IHTTPRequest.php b/src/Network/IHTTPRequest.php index a1c7730ca..6c839dd3d 100644 --- a/src/Network/IHTTPRequest.php +++ b/src/Network/IHTTPRequest.php @@ -63,7 +63,6 @@ interface IHTTPRequest * @param array $opts (optional parameters) assoziative array with: * 'accept_content' => supply Accept: header with 'accept_content' as the value * 'timeout' => int Timeout in seconds, default system config value or 60 seconds - * 'http_auth' => username:password * 'novalidate' => do not validate SSL certs, default is to validate using our CA list * 'cookiejar' => path to cookie jar file * 'header' => header array @@ -79,7 +78,6 @@ interface IHTTPRequest * @param array $opts (optional parameters) assoziative array with: * 'accept_content' => supply Accept: header with 'accept_content' as the value * 'timeout' => int Timeout in seconds, default system config value or 60 seconds - * 'http_auth' => username:password * 'novalidate' => do not validate SSL certs, default is to validate using our CA list * 'cookiejar' => path to cookie jar file * 'header' => header array diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index 61e95a52c..4d6ae7e90 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -403,7 +403,6 @@ class HTTPSignature * @param array $opts (optional parameters) assoziative array with: * 'accept_content' => supply Accept: header with 'accept_content' as the value * 'timeout' => int Timeout in seconds, default system config value or 60 seconds - * 'http_auth' => username:password * 'novalidate' => do not validate SSL certs, default is to validate using our CA list * 'nobody' => only return the header * 'cookiejar' => path to cookie jar file From c19f1a83ce29d3fba347f956fb97704f598c6380 Mon Sep 17 00:00:00 2001 From: Philipp Date: Sun, 18 Oct 2020 22:23:07 +0200 Subject: [PATCH 3/5] Remove option 'novalidate' flag from HTTPRequest options --- mod/parse_url.php | 2 +- src/Network/IHTTPRequest.php | 2 -- src/Util/HTTPSignature.php | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/mod/parse_url.php b/mod/parse_url.php index 3c931fdb1..1904d9219 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -85,7 +85,7 @@ function parse_url_content(App $a) // Check if the URL is an image, video or audio file. If so format // the URL with the corresponding BBCode media tag // Fetch the header of the URL - $curlResponse = DI::httpRequest()->head($url, ['novalidate' => true]); + $curlResponse = DI::httpRequest()->head($url); if ($curlResponse->isSuccess()) { // Convert the header fields into an array diff --git a/src/Network/IHTTPRequest.php b/src/Network/IHTTPRequest.php index 6c839dd3d..8927941e8 100644 --- a/src/Network/IHTTPRequest.php +++ b/src/Network/IHTTPRequest.php @@ -63,7 +63,6 @@ interface IHTTPRequest * @param array $opts (optional parameters) assoziative array with: * 'accept_content' => supply Accept: header with 'accept_content' as the value * 'timeout' => int Timeout in seconds, default system config value or 60 seconds - * 'novalidate' => do not validate SSL certs, default is to validate using our CA list * 'cookiejar' => path to cookie jar file * 'header' => header array * @@ -78,7 +77,6 @@ interface IHTTPRequest * @param array $opts (optional parameters) assoziative array with: * 'accept_content' => supply Accept: header with 'accept_content' as the value * 'timeout' => int Timeout in seconds, default system config value or 60 seconds - * 'novalidate' => do not validate SSL certs, default is to validate using our CA list * 'cookiejar' => path to cookie jar file * 'header' => header array * diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index 4d6ae7e90..2b98b1a32 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -403,7 +403,6 @@ class HTTPSignature * @param array $opts (optional parameters) assoziative array with: * 'accept_content' => supply Accept: header with 'accept_content' as the value * 'timeout' => int Timeout in seconds, default system config value or 60 seconds - * 'novalidate' => do not validate SSL certs, default is to validate using our CA list * 'nobody' => only return the header * 'cookiejar' => path to cookie jar file * From a74d88c4ee20d0991c1ec27b6a0fb9f59d8e7f9b Mon Sep 17 00:00:00 2001 From: Philipp Date: Sun, 18 Oct 2020 22:31:26 +0200 Subject: [PATCH 4/5] Remove 'headers' option occurrences and add a warning if used. --- src/Model/GServer.php | 2 +- src/Module/Magic.php | 12 ++++++------ src/Network/HTTPRequest.php | 1 + src/Util/HTTPSignature.php | 10 +++++----- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 009816d81..3a2a0bd79 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1639,7 +1639,7 @@ class GServer if (!empty($accesstoken)) { $api = 'https://instances.social/api/1.0/instances/list?count=0'; $header = ['Authorization: Bearer '.$accesstoken]; - $curlResult = DI::httpRequest()->get($api, ['headers' => $header]); + $curlResult = DI::httpRequest()->get($api, ['header' => $header]); if ($curlResult->isSuccess()) { $servers = json_decode($curlResult->getBody(), true); diff --git a/src/Module/Magic.php b/src/Module/Magic.php index fc37d91b4..af8ff3605 100644 --- a/src/Module/Magic.php +++ b/src/Module/Magic.php @@ -88,19 +88,19 @@ class Magic extends BaseModule $exp = explode('/profile/', $contact['url']); $basepath = $exp[0]; - $headers = []; - $headers['Accept'] = 'application/x-dfrn+json, application/x-zot+json'; - $headers['X-Open-Web-Auth'] = Strings::getRandomHex(); + $header = []; + $header['Accept'] = 'application/x-dfrn+json, application/x-zot+json'; + $header['X-Open-Web-Auth'] = Strings::getRandomHex(); // Create a header that is signed with the local users private key. - $headers = HTTPSignature::createSig( - $headers, + $header = HTTPSignature::createSig( + $header, $user['prvkey'], 'acct:' . $user['nickname'] . '@' . DI::baseUrl()->getHostname() . (DI::baseUrl()->getUrlPath() ? '/' . DI::baseUrl()->getUrlPath() : '') ); // Try to get an authentication token from the other instance. - $curlResult = DI::httpRequest()->get($basepath . '/owa', ['headers' => $headers]); + $curlResult = DI::httpRequest()->get($basepath . '/owa', ['header' => $header]); if ($curlResult->isSuccess()) { $j = json_decode($curlResult->getBody(), true); diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index df62ea454..93ce86c87 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -141,6 +141,7 @@ class HTTPRequest implements IHTTPRequest curl_setopt($ch, CURLOPT_ENCODING, ''); if (!empty($opts['headers'])) { + $this->logger->warning('Wrong option \'headers\' used.'); @curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['headers']); } diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index 2b98b1a32..e3244fade 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -411,7 +411,7 @@ class HTTPSignature */ public static function fetchRaw($request, $uid = 0, $binary = false, $opts = []) { - $headers = []; + $header = []; if (!empty($uid)) { $owner = User::getOwnerDataById($uid); @@ -431,21 +431,21 @@ class HTTPSignature $path = parse_url($request, PHP_URL_PATH); $date = DateTimeFormat::utcNow(DateTimeFormat::HTTP); - $headers = ['Date: ' . $date, 'Host: ' . $host]; + $header = ['Date: ' . $date, 'Host: ' . $host]; $signed_data = "(request-target): get " . $path . "\ndate: ". $date . "\nhost: " . $host; $signature = base64_encode(Crypto::rsaSign($signed_data, $owner['uprvkey'], 'sha256')); - $headers[] = 'Signature: keyId="' . $owner['url'] . '#main-key' . '",algorithm="rsa-sha256",headers="(request-target) date host",signature="' . $signature . '"'; + $header[] = 'Signature: keyId="' . $owner['url'] . '#main-key' . '",algorithm="rsa-sha256",headers="(request-target) date host",signature="' . $signature . '"'; } if (!empty($opts['accept_content'])) { - $headers[] = 'Accept: ' . $opts['accept_content']; + $header[] = 'Accept: ' . $opts['accept_content']; } $curl_opts = $opts; - $curl_opts['header'] = $headers; + $curl_opts['header'] = $header; if ($opts['nobody']) { $curlResult = DI::httpRequest()->head($request, $curl_opts); From d524a9569f6ee1d33072dd7a1b6af5c09645a693 Mon Sep 17 00:00:00 2001 From: Philipp Date: Sun, 18 Oct 2020 22:32:36 +0200 Subject: [PATCH 5/5] lower loglevel --- src/Network/HTTPRequest.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 93ce86c87..a0d940414 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -141,7 +141,7 @@ class HTTPRequest implements IHTTPRequest curl_setopt($ch, CURLOPT_ENCODING, ''); if (!empty($opts['headers'])) { - $this->logger->warning('Wrong option \'headers\' used.'); + $this->logger->notice('Wrong option \'headers\' used.'); @curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['headers']); } @@ -184,8 +184,6 @@ class HTTPRequest implements IHTTPRequest curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); } - $logger = $this->logger; - $s = @curl_exec($ch); $curl_info = @curl_getinfo($ch);