From 5344efef71f3a2978d50f1cf24771df28afc5aaa Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:07:05 +0100 Subject: [PATCH 01/17] Move post/curl/fetchUrl/fetchUrlFull to own class "Network\HTTPRequest" --- mod/dfrn_confirm.php | 4 +- mod/dfrn_poll.php | 28 +-- mod/dfrn_request.php | 5 +- mod/match.php | 4 +- mod/oexchange.php | 4 +- mod/ostatus_subscribe.php | 4 +- mod/parse_url.php | 4 +- mod/pubsubhubbub.php | 4 +- mod/redir.php | 4 +- src/Content/OEmbed.php | 7 +- src/Content/Text/BBCode.php | 9 +- src/Core/Installer.php | 6 +- src/Core/Protocol.php | 4 +- src/Core/Search.php | 5 +- src/Core/Worker.php | 4 +- src/Model/APContact.php | 6 +- src/Model/GContact.php | 11 +- src/Model/GServer.php | 53 +++-- src/Model/Photo.php | 4 +- src/Model/Profile.php | 3 +- src/Model/User.php | 3 +- src/Module/Admin/Summary.php | 6 +- src/Module/Debug/Feed.php | 4 +- src/Module/Magic.php | 4 +- src/Network/HTTPRequest.php | 364 +++++++++++++++++++++++++++++++ src/Network/Probe.php | 22 +- src/Protocol/ActivityPub.php | 6 +- src/Protocol/DFRN.php | 7 +- src/Protocol/Diaspora.php | 5 +- src/Protocol/OStatus.php | 16 +- src/Protocol/PortableContact.php | 12 +- src/Protocol/Salmon.php | 10 +- src/Util/ExAuth.php | 3 +- src/Util/HTTPSignature.php | 9 +- src/Util/Images.php | 3 +- src/Util/Network.php | 331 ---------------------------- src/Util/ParseUrl.php | 3 +- src/Worker/CheckVersion.php | 4 +- src/Worker/CronJobs.php | 6 +- src/Worker/Directory.php | 4 +- src/Worker/OnePoll.php | 10 +- src/Worker/PubSubPublish.php | 4 +- src/Worker/SearchDirectory.php | 4 +- 43 files changed, 528 insertions(+), 485 deletions(-) create mode 100644 src/Network/HTTPRequest.php diff --git a/mod/dfrn_confirm.php b/mod/dfrn_confirm.php index 8b87bae5d3..f78e8d55aa 100644 --- a/mod/dfrn_confirm.php +++ b/mod/dfrn_confirm.php @@ -42,10 +42,10 @@ use Friendica\Model\Contact; use Friendica\Model\Group; use Friendica\Model\Notify\Type; use Friendica\Model\User; +use Friendica\Network\HTTPRequest; use Friendica\Protocol\Activity; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; @@ -224,7 +224,7 @@ function dfrn_confirm_post(App $a, $handsfree = null) * */ - $res = Network::post($dfrn_confirm, $params, [], 120)->getBody(); + $res = HTTPRequest::post($dfrn_confirm, $params, [], 120)->getBody(); Logger::log(' Confirm: received data: ' . $res, Logger::DATA); diff --git a/mod/dfrn_poll.php b/mod/dfrn_poll.php index 8d50761db1..faa55a1082 100644 --- a/mod/dfrn_poll.php +++ b/mod/dfrn_poll.php @@ -21,13 +21,13 @@ use Friendica\App; use Friendica\Core\Logger; -use Friendica\Core\System; use Friendica\Core\Session; +use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPRequest; use Friendica\Protocol\DFRN; use Friendica\Protocol\OStatus; -use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; @@ -115,7 +115,7 @@ function dfrn_poll_init(App $a) ); if (DBA::isResult($r)) { - $s = Network::fetchUrl($r[0]['poll'] . '?dfrn_id=' . $my_id . '&type=profile-check'); + $s = HTTPRequest::fetchUrl($r[0]['poll'] . '?dfrn_id=' . $my_id . '&type=profile-check'); Logger::log("dfrn_poll: old profile returns " . $s, Logger::DATA); @@ -499,20 +499,20 @@ function dfrn_poll_content(App $a) // URL reply if ($dfrn_version < 2.2) { - $s = Network::fetchUrl($r[0]['poll'] - . '?dfrn_id=' . $encrypted_id - . '&type=profile-check' - . '&dfrn_version=' . DFRN_PROTOCOL_VERSION - . '&challenge=' . $challenge - . '&sec=' . $sec + $s = HTTPRequest::fetchUrl($r[0]['poll'] + . '?dfrn_id=' . $encrypted_id + . '&type=profile-check' + . '&dfrn_version=' . DFRN_PROTOCOL_VERSION + . '&challenge=' . $challenge + . '&sec=' . $sec ); } else { - $s = Network::post($r[0]['poll'], [ - 'dfrn_id' => $encrypted_id, - 'type' => 'profile-check', + $s = HTTPRequest::post($r[0]['poll'], [ + 'dfrn_id' => $encrypted_id, + 'type' => 'profile-check', 'dfrn_version' => DFRN_PROTOCOL_VERSION, - 'challenge' => $challenge, - 'sec' => $sec + 'challenge' => $challenge, + 'sec' => $sec ])->getBody(); } diff --git a/mod/dfrn_request.php b/mod/dfrn_request.php index f8e4c90236..bdc407b0b1 100644 --- a/mod/dfrn_request.php +++ b/mod/dfrn_request.php @@ -29,8 +29,8 @@ use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\Core\Renderer; use Friendica\Core\Search; -use Friendica\Core\System; use Friendica\Core\Session; +use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; @@ -39,6 +39,7 @@ use Friendica\Model\Notify\Type; use Friendica\Model\Profile; use Friendica\Model\User; use Friendica\Module\Security\Login; +use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Protocol\Activity; use Friendica\Util\DateTimeFormat; @@ -203,7 +204,7 @@ function dfrn_request_post(App $a) } if (!empty($dfrn_request) && strlen($confirm_key)) { - Network::fetchUrl($dfrn_request . '?confirm_key=' . $confirm_key); + HTTPRequest::fetchUrl($dfrn_request . '?confirm_key=' . $confirm_key); } // (ignore reply, nothing we can do it failed) diff --git a/mod/match.php b/mod/match.php index 747e0b2f03..4ec47c4cc6 100644 --- a/mod/match.php +++ b/mod/match.php @@ -27,7 +27,7 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Profile; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; use Friendica\Util\Proxy as ProxyUtils; /** @@ -76,7 +76,7 @@ function match_content(App $a) $host = DI::baseUrl(); } - $msearch_json = Network::post($host . '/msearch', $params)->getBody(); + $msearch_json = HTTPRequest::post($host . '/msearch', $params)->getBody(); $msearch = json_decode($msearch_json); diff --git a/mod/oexchange.php b/mod/oexchange.php index 97367c3ea5..5238893320 100644 --- a/mod/oexchange.php +++ b/mod/oexchange.php @@ -23,7 +23,7 @@ use Friendica\App; use Friendica\Core\Renderer; use Friendica\DI; use Friendica\Module\Security\Login; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; use Friendica\Util\Strings; function oexchange_init(App $a) { @@ -58,7 +58,7 @@ function oexchange_content(App $a) { $tags = ((!empty($_REQUEST['tags'])) ? '&tags=' . urlencode(Strings::escapeTags(trim($_REQUEST['tags']))) : ''); - $s = Network::fetchUrl(DI::baseUrl() . '/parse_url?url=' . $url . $title . $description . $tags); + $s = HTTPRequest::fetchUrl(DI::baseUrl() . '/parse_url?url=' . $url . $title . $description . $tags); if (!strlen($s)) { return; diff --git a/mod/ostatus_subscribe.php b/mod/ostatus_subscribe.php index 751afcc731..6b6c94987d 100644 --- a/mod/ostatus_subscribe.php +++ b/mod/ostatus_subscribe.php @@ -23,7 +23,7 @@ use Friendica\App; use Friendica\Core\Protocol; use Friendica\DI; use Friendica\Model\Contact; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; function ostatus_subscribe_content(App $a) { @@ -55,7 +55,7 @@ function ostatus_subscribe_content(App $a) $api = $contact['baseurl'] . '/api/'; // Fetching friends - $curlResult = Network::curl($api . 'statuses/friends.json?screen_name=' . $contact['nick']); + $curlResult = HTTPRequest::curl($api . 'statuses/friends.json?screen_name=' . $contact['nick']); if (!$curlResult->isSuccess()) { DI::pConfig()->delete($uid, 'ostatus', 'legacy_contact'); diff --git a/mod/parse_url.php b/mod/parse_url.php index 67610140b6..49e41246c9 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -28,7 +28,7 @@ use Friendica\Content\PageInfo; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\System; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; use Friendica\Util\ParseUrl; use Friendica\Util\Strings; @@ -85,7 +85,7 @@ function parse_url_content(App $a) // Check if the URL is an image, video or audio file. If so format // the URL with the corresponding BBCode media tag // Fetch the header of the URL - $curlResponse = Network::curl($url, false, ['novalidate' => true, 'nobody' => true]); + $curlResponse = HTTPRequest::curl($url, false, ['novalidate' => true, 'nobody' => true]); if ($curlResponse->isSuccess()) { // Convert the header fields into an array diff --git a/mod/pubsubhubbub.php b/mod/pubsubhubbub.php index 4d33503796..9403d3eb7f 100644 --- a/mod/pubsubhubbub.php +++ b/mod/pubsubhubbub.php @@ -24,7 +24,7 @@ use Friendica\Core\Logger; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\PushSubscriber; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; use Friendica\Util\Strings; function post_var($name) { @@ -126,7 +126,7 @@ function pubsubhubbub_init(App $a) { $hub_callback = rtrim($hub_callback, ' ?&#'); $separator = parse_url($hub_callback, PHP_URL_QUERY) === null ? '?' : '&'; - $fetchResult = Network::fetchUrlFull($hub_callback . $separator . $params); + $fetchResult = HTTPRequest::fetchUrlFull($hub_callback . $separator . $params); $body = $fetchResult->getBody(); $ret = $fetchResult->getReturnCode(); diff --git a/mod/redir.php b/mod/redir.php index d928e66df0..deb97ca1c6 100644 --- a/mod/redir.php +++ b/mod/redir.php @@ -27,7 +27,7 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Profile; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; use Friendica\Util\Strings; function redir_init(App $a) { @@ -171,7 +171,7 @@ function redir_magic($a, $cid, $url) } // Test for magic auth on the target system - $serverret = Network::curl($basepath . '/magic'); + $serverret = HTTPRequest::curl($basepath . '/magic'); if ($serverret->isSuccess()) { $separator = strpos($target_url, '?') ? '&' : '?'; $target_url .= $separator . 'zrl=' . urlencode($visitor) . '&addr=' . urlencode($contact_url); diff --git a/src/Content/OEmbed.php b/src/Content/OEmbed.php index db467a2630..8cfb8ce0a1 100644 --- a/src/Content/OEmbed.php +++ b/src/Content/OEmbed.php @@ -31,6 +31,7 @@ use Friendica\Core\Hook; use Friendica\Core\Renderer; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPRequest; use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; use Friendica\Util\ParseUrl; @@ -95,7 +96,7 @@ class OEmbed if (!in_array($ext, $noexts)) { // try oembed autodiscovery - $html_text = Network::fetchUrl($embedurl, false, 15, 'text/*'); + $html_text = HTTPRequest::fetchUrl($embedurl, false, 15, 'text/*'); if ($html_text) { $dom = @DOMDocument::loadHTML($html_text); if ($dom) { @@ -103,14 +104,14 @@ class OEmbed $entries = $xpath->query("//link[@type='application/json+oembed']"); foreach ($entries as $e) { $href = $e->getAttributeNode('href')->nodeValue; - $json_string = Network::fetchUrl($href . '&maxwidth=' . $a->videowidth); + $json_string = HTTPRequest::fetchUrl($href . '&maxwidth=' . $a->videowidth); break; } $entries = $xpath->query("//link[@type='text/json+oembed']"); foreach ($entries as $e) { $href = $e->getAttributeNode('href')->nodeValue; - $json_string = Network::fetchUrl($href . '&maxwidth=' . $a->videowidth); + $json_string = HTTPRequest::fetchUrl($href . '&maxwidth=' . $a->videowidth); break; } } diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index ce34d58ac5..f03ea6104b 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -38,12 +38,11 @@ use Friendica\Model\Contact; use Friendica\Model\Event; use Friendica\Model\Photo; use Friendica\Model\Tag; -use Friendica\Network\Probe; +use Friendica\Network\HTTPRequest; use Friendica\Object\Image; use Friendica\Protocol\Activity; use Friendica\Util\Images; use Friendica\Util\Map; -use Friendica\Util\Network; use Friendica\Util\ParseUrl; use Friendica\Util\Proxy as ProxyUtils; use Friendica\Util\Strings; @@ -487,7 +486,7 @@ class BBCode continue; } - $curlResult = Network::curl($mtch[1], true); + $curlResult = HTTPRequest::curl($mtch[1], true); if (!$curlResult->isSuccess()) { continue; } @@ -1108,7 +1107,7 @@ class BBCode $text = "[url=" . $match[2] . ']' . $match[2] . "[/url]"; // if its not a picture then look if its a page that contains a picture link - $body = Network::fetchUrl($match[1]); + $body = HTTPRequest::fetchUrl($match[1]); $doc = new DOMDocument(); @$doc->loadHTML($body); @@ -1187,7 +1186,7 @@ class BBCode } // if its not a picture then look if its a page that contains a picture link - $body = Network::fetchUrl($match[1]); + $body = HTTPRequest::fetchUrl($match[1]); $doc = new DOMDocument(); @$doc->loadHTML($body); diff --git a/src/Core/Installer.php b/src/Core/Installer.php index 37b51d2ed9..7b6291ff39 100644 --- a/src/Core/Installer.php +++ b/src/Core/Installer.php @@ -27,8 +27,8 @@ use Friendica\Core\Config\Cache; use Friendica\Database\Database; use Friendica\Database\DBStructure; use Friendica\DI; +use Friendica\Network\HTTPRequest; use Friendica\Util\Images; -use Friendica\Util\Network; use Friendica\Util\Strings; /** @@ -548,11 +548,11 @@ class Installer $help = ""; $error_msg = ""; if (function_exists('curl_init')) { - $fetchResult = Network::fetchUrlFull($baseurl . "/install/testrewrite"); + $fetchResult = HTTPRequest::fetchUrlFull($baseurl . "/install/testrewrite"); $url = Strings::normaliseLink($baseurl . "/install/testrewrite"); if ($fetchResult->getReturnCode() != 204) { - $fetchResult = Network::fetchUrlFull($url); + $fetchResult = HTTPRequest::fetchUrlFull($url); } if ($fetchResult->getReturnCode() != 204) { diff --git a/src/Core/Protocol.php b/src/Core/Protocol.php index e510f1868c..84b589bf29 100644 --- a/src/Core/Protocol.php +++ b/src/Core/Protocol.php @@ -21,7 +21,7 @@ namespace Friendica\Core; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; /** * Manage compatibility with federated networks @@ -123,7 +123,7 @@ class Protocol if (preg_match('=https?://(.*)/user/(.*)=ism', $profile_url, $matches)) { $statusnet_host = $matches[1]; $statusnet_user = $matches[2]; - $UserData = Network::fetchUrl('http://' . $statusnet_host . '/api/users/show.json?user_id=' . $statusnet_user); + $UserData = HTTPRequest::fetchUrl('http://' . $statusnet_host . '/api/users/show.json?user_id=' . $statusnet_user); $user = json_decode($UserData); if ($user) { $matches[2] = $user->screen_name; diff --git a/src/Core/Search.php b/src/Core/Search.php index edc88ffd7d..26af05e746 100644 --- a/src/Core/Search.php +++ b/src/Core/Search.php @@ -26,6 +26,7 @@ use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\GContact; use Friendica\Network\HTTPException; +use Friendica\Network\HTTPRequest; use Friendica\Object\Search\ContactResult; use Friendica\Object\Search\ResultList; use Friendica\Util\Network; @@ -123,7 +124,7 @@ class Search $searchUrl .= '&page=' . $page; } - $resultJson = Network::fetchUrl($searchUrl, false, 0, 'application/json'); + $resultJson = HTTPRequest::fetchUrl($searchUrl, false, 0, 'application/json'); $results = json_decode($resultJson, true); @@ -284,7 +285,7 @@ class Search $return = GContact::searchByName($search, $mode); } else { $p = $page > 1 ? 'p=' . $page : ''; - $curlResult = Network::curl(self::getGlobalDirectory() . '/search/people?' . $p . '&q=' . urlencode($search), false, ['accept_content' => 'application/json']); + $curlResult = HTTPRequest::curl(self::getGlobalDirectory() . '/search/people?' . $p . '&q=' . urlencode($search), false, ['accept_content' => 'application/json']); if ($curlResult->isSuccess()) { $searchResult = json_decode($curlResult->getBody(), true); if (!empty($searchResult['profiles'])) { diff --git a/src/Core/Worker.php b/src/Core/Worker.php index fe3d17ad7f..a5c4226c47 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -25,8 +25,8 @@ use Friendica\Core; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Process; +use Friendica\Network\HTTPRequest; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Network; /** * Contains the class for the worker background job processing @@ -997,7 +997,7 @@ class Worker } $url = DI::baseUrl() . '/worker'; - Network::fetchUrl($url, false, 1); + HTTPRequest::fetchUrl($url, false, 1); } /** diff --git a/src/Model/APContact.php b/src/Model/APContact.php index 5966b8c25e..9fc72ac5a4 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -24,15 +24,13 @@ namespace Friendica\Model; use Friendica\Content\Text\HTML; use Friendica\Core\Logger; use Friendica\Database\DBA; -use Friendica\DI; use Friendica\Network\Probe; use Friendica\Protocol\ActivityNamespace; use Friendica\Protocol\ActivityPub; use Friendica\Util\Crypto; -use Friendica\Util\Network; -use Friendica\Util\JsonLD; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Strings; +use Friendica\Util\JsonLD; +use Friendica\Util\Network; class APContact { diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 6a3c7da74c..a7cf837bdd 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -26,11 +26,12 @@ use DOMXPath; use Exception; use Friendica\Core\Logger; use Friendica\Core\Protocol; -use Friendica\Core\System; use Friendica\Core\Search; +use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\PortableContact; @@ -537,7 +538,7 @@ class GContact $done[] = DI::baseUrl() . '/poco'; if (strlen(DI::config()->get('system', 'directory'))) { - $x = Network::fetchUrl(Search::getGlobalDirectory() . '/pubsites'); + $x = HTTPRequest::fetchUrl(Search::getGlobalDirectory() . '/pubsites'); if (!empty($x)) { $j = json_decode($x); if (!empty($j->entries)) { @@ -845,7 +846,7 @@ class GContact return false; } - $curlResult = Network::curl($gserver['noscrape'] . '/' . $data['nick']); + $curlResult = HTTPRequest::curl($gserver['noscrape'] . '/' . $data['nick']); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { $noscrape = json_decode($curlResult->getBody(), true); @@ -927,7 +928,7 @@ class GContact private static function updateFromFeed(array $data) { // Search for the newest entry in the feed - $curlResult = Network::curl($data['poll']); + $curlResult = HTTPRequest::curl($data['poll']); if (!$curlResult->isSuccess()) { $fields = ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()]; DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]); @@ -1205,7 +1206,7 @@ class GContact $url = $server . '/main/statistics'; - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if (!$curlResult->isSuccess()) { return false; } diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 8cad1aad08..80ef201a8a 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -23,20 +23,21 @@ namespace Friendica\Model; use DOMDocument; use DOMXPath; +use Friendica\Core\Logger; use Friendica\Core\Protocol; +use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Module\Register; use Friendica\Network\CurlResult; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; +use Friendica\Protocol\Diaspora; +use Friendica\Protocol\PortableContact; use Friendica\Util\DateTimeFormat; +use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; -use Friendica\Core\Logger; -use Friendica\Core\System; -use Friendica\Protocol\PortableContact; -use Friendica\Protocol\Diaspora; /** * This class handles GServer related functions @@ -309,7 +310,7 @@ class GServer // When a nodeinfo is present, we don't need to dig further $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); + $curlResult = HTTPRequest::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); if ($curlResult->isTimeout()) { self::setFailure($url); return false; @@ -342,7 +343,7 @@ class GServer $basedata = ['detection-method' => self::DETECT_MANUAL]; } - $curlResult = Network::curl($baseurl, false, ['timeout' => $xrd_timeout]); + $curlResult = HTTPRequest::curl($baseurl, false, ['timeout' => $xrd_timeout]); if ($curlResult->isSuccess()) { $basedata = self::analyseRootHeader($curlResult, $basedata); $basedata = self::analyseRootBody($curlResult, $basedata, $baseurl); @@ -498,7 +499,7 @@ class GServer { Logger::info('Discover relay data', ['server' => $server_url]); - $curlResult = Network::curl($server_url . '/.well-known/x-social-relay'); + $curlResult = HTTPRequest::curl($server_url . '/.well-known/x-social-relay'); if (!$curlResult->isSuccess()) { return; } @@ -579,7 +580,7 @@ class GServer */ private static function fetchStatistics(string $url) { - $curlResult = Network::curl($url . '/statistics.json'); + $curlResult = HTTPRequest::curl($url . '/statistics.json'); if (!$curlResult->isSuccess()) { return []; } @@ -689,7 +690,8 @@ class GServer */ private static function parseNodeinfo1(string $nodeinfo_url) { - $curlResult = Network::curl($nodeinfo_url); + $curlResult = HTTPRequest::curl($nodeinfo_url); + if (!$curlResult->isSuccess()) { return []; } @@ -765,7 +767,7 @@ class GServer */ private static function parseNodeinfo2(string $nodeinfo_url) { - $curlResult = Network::curl($nodeinfo_url); + $curlResult = HTTPRequest::curl($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; } @@ -842,7 +844,7 @@ class GServer */ private static function fetchSiteinfo(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/siteinfo.json'); + $curlResult = HTTPRequest::curl($url . '/siteinfo.json'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -911,7 +913,7 @@ class GServer private static function validHostMeta(string $url) { $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = Network::curl($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); + $curlResult = HTTPRequest::curl($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); if (!$curlResult->isSuccess()) { return false; } @@ -1007,7 +1009,7 @@ class GServer { $serverdata['poco'] = ''; - $curlResult = Network::curl($url. '/poco'); + $curlResult = HTTPRequest::curl($url . '/poco'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1037,7 +1039,7 @@ class GServer */ public static function checkMastodonDirectory(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/api/v1/directory?limit=1'); + $curlResult = HTTPRequest::curl($url . '/api/v1/directory?limit=1'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1064,7 +1066,8 @@ class GServer */ private static function detectNextcloud(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/status.php'); + $curlResult = HTTPRequest::curl($url . '/status.php'); + if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1097,7 +1100,8 @@ class GServer */ private static function detectMastodonAlikes(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/api/v1/instance'); + $curlResult = HTTPRequest::curl($url . '/api/v1/instance'); + if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1162,7 +1166,7 @@ class GServer */ private static function detectHubzilla(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/api/statusnet/config.json'); + $curlResult = HTTPRequest::curl($url . '/api/statusnet/config.json'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1260,7 +1264,7 @@ class GServer private static function detectGNUSocial(string $url, array $serverdata) { // Test for GNU Social - $curlResult = Network::curl($url . '/api/gnusocial/version.json'); + $curlResult = HTTPRequest::curl($url . '/api/gnusocial/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { $serverdata['platform'] = 'gnusocial'; @@ -1278,7 +1282,7 @@ class GServer } // Test for Statusnet - $curlResult = Network::curl($url . '/api/statusnet/version.json'); + $curlResult = HTTPRequest::curl($url . '/api/statusnet/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { @@ -1314,9 +1318,9 @@ class GServer */ private static function detectFriendica(string $url, array $serverdata) { - $curlResult = Network::curl($url . '/friendica/json'); + $curlResult = HTTPRequest::curl($url . '/friendica/json'); if (!$curlResult->isSuccess()) { - $curlResult = Network::curl($url . '/friendika/json'); + $curlResult = HTTPRequest::curl($url . '/friendika/json'); $friendika = true; $platform = 'Friendika'; } else { @@ -1631,7 +1635,7 @@ class GServer $protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus']; foreach ($protocols as $protocol) { $query = '{nodes(protocol:"' . $protocol . '"){host}}'; - $curlResult = Network::fetchUrl('https://the-federation.info/graphql?query=' . urlencode($query)); + $curlResult = HTTPRequest::fetchUrl('https://the-federation.info/graphql?query=' . urlencode($query)); if (!empty($curlResult)) { $data = json_decode($curlResult, true); if (!empty($data['data']['nodes'])) { @@ -1649,7 +1653,8 @@ class GServer if (!empty($accesstoken)) { $api = 'https://instances.social/api/1.0/instances/list?count=0'; $header = ['Authorization: Bearer '.$accesstoken]; - $curlResult = Network::curl($api, false, ['headers' => $header]); + $curlResult = HTTPRequest::curl($api, false, ['headers' => $header]); + if ($curlResult->isSuccess()) { $servers = json_decode($curlResult->getBody(), true); diff --git a/src/Model/Photo.php b/src/Model/Photo.php index 9d8b5611f7..125718bf50 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -28,10 +28,10 @@ use Friendica\Database\DBA; use Friendica\Database\DBStructure; use Friendica\DI; use Friendica\Model\Storage\SystemResource; +use Friendica\Network\HTTPRequest; use Friendica\Object\Image; use Friendica\Util\DateTimeFormat; use Friendica\Util\Images; -use Friendica\Util\Network; use Friendica\Util\Security; use Friendica\Util\Strings; @@ -421,7 +421,7 @@ class Photo $filename = basename($image_url); if (!empty($image_url)) { - $ret = Network::curl($image_url, true); + $ret = HTTPRequest::curl($image_url, true); $img_str = $ret->getBody(); $type = $ret->getContentType(); } else { diff --git a/src/Model/Profile.php b/src/Model/Profile.php index 2fcbde0779..c8fd9d029c 100644 --- a/src/Model/Profile.php +++ b/src/Model/Profile.php @@ -33,6 +33,7 @@ use Friendica\Core\Session; use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Network\HTTPRequest; use Friendica\Protocol\Activity; use Friendica\Protocol\Diaspora; use Friendica\Util\DateTimeFormat; @@ -737,7 +738,7 @@ class Profile $magic_path = $basepath . '/magic' . '?owa=1&dest=' . $dest . '&' . $addr_request; // We have to check if the remote server does understand /magic without invoking something - $serverret = Network::curl($basepath . '/magic'); + $serverret = HTTPRequest::curl($basepath . '/magic'); if ($serverret->isSuccess()) { Logger::log('Doing magic auth for visitor ' . $my_url . ' to ' . $magic_path, Logger::DEBUG); System::externalRedirect($magic_path); diff --git a/src/Model/User.php b/src/Model/User.php index b4ada344e1..fda105687d 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -34,6 +34,7 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\TwoFactor\AppSpecificPassword; use Friendica\Network\HTTPException\InternalServerErrorException; +use Friendica\Network\HTTPRequest; use Friendica\Object\Image; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; @@ -823,7 +824,7 @@ class User $photo_failure = false; $filename = basename($photo); - $curlResult = Network::curl($photo, true); + $curlResult = HTTPRequest::curl($photo, true); if ($curlResult->isSuccess()) { $img_str = $curlResult->getBody(); $type = $curlResult->getContentType(); diff --git a/src/Module/Admin/Summary.php b/src/Module/Admin/Summary.php index c19b7f7f87..6dcef2ea43 100644 --- a/src/Module/Admin/Summary.php +++ b/src/Module/Admin/Summary.php @@ -31,12 +31,10 @@ use Friendica\Database\DBStructure; use Friendica\DI; use Friendica\Model\Register; use Friendica\Module\BaseAdmin; -use Friendica\Module\Update\Profile; use Friendica\Network\HTTPException\InternalServerErrorException; -use Friendica\Render\FriendicaSmarty; +use Friendica\Network\HTTPRequest; use Friendica\Util\ConfigFileLoader; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Network; class Summary extends BaseAdmin { @@ -249,7 +247,7 @@ class Summary extends BaseAdmin private static function checkSelfHostMeta() { // Fetch the host-meta to check if this really is a vital server - return Network::curl(DI::baseUrl()->get() . '/.well-known/host-meta')->isSuccess(); + return HTTPRequest::curl(DI::baseUrl()->get() . '/.well-known/host-meta')->isSuccess(); } } diff --git a/src/Module/Debug/Feed.php b/src/Module/Debug/Feed.php index e969de9cc3..6214b49dd5 100644 --- a/src/Module/Debug/Feed.php +++ b/src/Module/Debug/Feed.php @@ -25,8 +25,8 @@ use Friendica\BaseModule; use Friendica\Core\Renderer; use Friendica\DI; use Friendica\Model; +use Friendica\Network\HTTPRequest; use Friendica\Protocol; -use Friendica\Util\Network; /** * Tests a given feed of a contact @@ -49,7 +49,7 @@ class Feed extends BaseModule $contact = Model\Contact::getByURLForUser($url, local_user(), false); - $xml = Network::fetchUrl($contact['poll']); + $xml = HTTPRequest::fetchUrl($contact['poll']); $import_result = Protocol\Feed::import($xml); diff --git a/src/Module/Magic.php b/src/Module/Magic.php index f27ffeac58..b65159585b 100644 --- a/src/Module/Magic.php +++ b/src/Module/Magic.php @@ -27,8 +27,8 @@ use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Network\HTTPRequest; use Friendica\Util\HTTPSignature; -use Friendica\Util\Network; use Friendica\Util\Strings; /** @@ -101,7 +101,7 @@ class Magic extends BaseModule ); // Try to get an authentication token from the other instance. - $curlResult = Network::curl($basepath . '/owa', false, ['headers' => $headers]); + $curlResult = HTTPRequest::curl($basepath . '/owa', false, ['headers' => $headers]); if ($curlResult->isSuccess()) { $j = json_decode($curlResult->getBody(), true); diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php new file mode 100644 index 0000000000..6986a93599 --- /dev/null +++ b/src/Network/HTTPRequest.php @@ -0,0 +1,364 @@ +. + * + */ + +namespace Friendica\Network; + +use Friendica\Core\Logger; +use Friendica\Core\System; +use Friendica\DI; +use Friendica\Util\Network; + +/** + * Performs HTTP requests to a given URL + */ +class HTTPRequest +{ + /** + * fetches an URL. + * + * @param string $url URL to fetch + * @param bool $binary default false + * TRUE if asked to return binary results (file download) + * @param array $opts (optional parameters) assoziative array with: + * 'accept_content' => supply Accept: header with 'accept_content' as the value + * 'timeout' => int Timeout in seconds, default system config value or 60 seconds + * 'http_auth' => username:password + * 'novalidate' => do not validate SSL certs, default is to validate using our CA list + * 'nobody' => only return the header + * 'cookiejar' => path to cookie jar file + * 'header' => header array + * @param int $redirects The recursion counter for internal use - default 0 + * + * @return CurlResult + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + public static function curl(string $url, bool $binary = false, array $opts = [], int &$redirects = 0) + { + $stamp1 = microtime(true); + + $a = DI::app(); + + if (strlen($url) > 1000) { + Logger::log('URL is longer than 1000 characters. Callstack: ' . System::callstack(20), Logger::DEBUG); + return CurlResult::createErrorCurl(substr($url, 0, 200)); + } + + $parts2 = []; + $parts = parse_url($url); + $path_parts = explode('/', $parts['path'] ?? ''); + foreach ($path_parts as $part) { + if (strlen($part) <> mb_strlen($part)) { + $parts2[] = rawurlencode($part); + } else { + $parts2[] = $part; + } + } + $parts['path'] = implode('/', $parts2); + $url = Network::unparseURL($parts); + + if (Network::isUrlBlocked($url)) { + Logger::log('domain of ' . $url . ' is blocked', Logger::DATA); + return CurlResult::createErrorCurl($url); + } + + $ch = @curl_init($url); + + if (($redirects > 8) || (!$ch)) { + return CurlResult::createErrorCurl($url); + } + + @curl_setopt($ch, CURLOPT_HEADER, true); + + if (!empty($opts['cookiejar'])) { + curl_setopt($ch, CURLOPT_COOKIEJAR, $opts["cookiejar"]); + curl_setopt($ch, CURLOPT_COOKIEFILE, $opts["cookiejar"]); + } + + // These settings aren't needed. We're following the location already. + // @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); + // @curl_setopt($ch, CURLOPT_MAXREDIRS, 5); + + if (!empty($opts['accept_content'])) { + curl_setopt( + $ch, + CURLOPT_HTTPHEADER, + ['Accept: ' . $opts['accept_content']] + ); + } + + if (!empty($opts['header'])) { + curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['header']); + } + + @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + @curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); + + $range = intval(DI::config()->get('system', 'curl_range_bytes', 0)); + + if ($range > 0) { + @curl_setopt($ch, CURLOPT_RANGE, '0-' . $range); + } + + // Without this setting it seems as if some webservers send compressed content + // This seems to confuse curl so that it shows this uncompressed. + /// @todo We could possibly set this value to "gzip" or something similar + curl_setopt($ch, CURLOPT_ENCODING, ''); + + if (!empty($opts['headers'])) { + @curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['headers']); + } + + if (!empty($opts['nobody'])) { + @curl_setopt($ch, CURLOPT_NOBODY, $opts['nobody']); + } + + if (!empty($opts['timeout'])) { + @curl_setopt($ch, CURLOPT_TIMEOUT, $opts['timeout']); + } else { + $curl_time = DI::config()->get('system', 'curl_timeout', 60); + @curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time)); + } + + // by default we will allow self-signed certs + // but you can override this + + $check_cert = DI::config()->get('system', 'verifyssl'); + @curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false)); + + if ($check_cert) { + @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); + } + + $proxy = DI::config()->get('system', 'proxy'); + + if (strlen($proxy)) { + @curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1); + @curl_setopt($ch, CURLOPT_PROXY, $proxy); + $proxyuser = @DI::config()->get('system', 'proxyuser'); + + if (strlen($proxyuser)) { + @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser); + } + } + + if (DI::config()->get('system', 'ipv4_resolve', false)) { + curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + } + + if ($binary) { + @curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1); + } + + // don't let curl abort the entire application + // if it throws any errors. + + $s = @curl_exec($ch); + $curl_info = @curl_getinfo($ch); + + // Special treatment for HTTP Code 416 + // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/416 + if (($curl_info['http_code'] == 416) && ($range > 0)) { + @curl_setopt($ch, CURLOPT_RANGE, ''); + $s = @curl_exec($ch); + $curl_info = @curl_getinfo($ch); + } + + $curlResponse = new CurlResult($url, $s, $curl_info, curl_errno($ch), curl_error($ch)); + + if ($curlResponse->isRedirectUrl()) { + $redirects++; + Logger::log('curl: redirect ' . $url . ' to ' . $curlResponse->getRedirectUrl()); + @curl_close($ch); + return self::curl($curlResponse->getRedirectUrl(), $binary, $opts, $redirects); + } + + @curl_close($ch); + + DI::profiler()->saveTimestamp($stamp1, 'network', System::callstack()); + + return $curlResponse; + } + + /** + * Send POST request to $url + * + * @param string $url URL to post + * @param mixed $params array of POST variables + * @param array $headers HTTP headers + * @param int $redirects Recursion counter for internal use - default = 0 + * @param int $timeout The timeout in seconds, default system config value or 60 seconds + * + * @return CurlResult The content + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + public static function post(string $url, $params, array $headers = [], int $timeout = 0, int &$redirects = 0) + { + $stamp1 = microtime(true); + + if (Network::isUrlBlocked($url)) { + Logger::log('post_url: domain of ' . $url . ' is blocked', Logger::DATA); + return CurlResult::createErrorCurl($url); + } + + $a = DI::app(); + $ch = curl_init($url); + + if (($redirects > 8) || (!$ch)) { + return CurlResult::createErrorCurl($url); + } + + Logger::log('post_url: start ' . $url, Logger::DATA); + + curl_setopt($ch, CURLOPT_HEADER, true); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_POST, 1); + curl_setopt($ch, CURLOPT_POSTFIELDS, $params); + curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); + + if (DI::config()->get('system', 'ipv4_resolve', false)) { + curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); + } + + if (intval($timeout)) { + curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); + } else { + $curl_time = DI::config()->get('system', 'curl_timeout', 60); + curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time)); + } + + if (!empty($headers)) { + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); + } + + $check_cert = DI::config()->get('system', 'verifyssl'); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false)); + + if ($check_cert) { + @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); + } + + $proxy = DI::config()->get('system', 'proxy'); + + if (strlen($proxy)) { + curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1); + curl_setopt($ch, CURLOPT_PROXY, $proxy); + $proxyuser = DI::config()->get('system', 'proxyuser'); + if (strlen($proxyuser)) { + curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser); + } + } + + // don't let curl abort the entire application + // if it throws any errors. + + $s = @curl_exec($ch); + + $curl_info = curl_getinfo($ch); + + $curlResponse = new CurlResult($url, $s, $curl_info, curl_errno($ch), curl_error($ch)); + + if ($curlResponse->isRedirectUrl()) { + $redirects++; + Logger::log('post_url: redirect ' . $url . ' to ' . $curlResponse->getRedirectUrl()); + curl_close($ch); + return self::post($curlResponse->getRedirectUrl(), $params, $headers, $redirects, $timeout); + } + + curl_close($ch); + + DI::profiler()->saveTimestamp($stamp1, 'network', System::callstack()); + + // Very old versions of Lighttpd don't like the "Expect" header, so we remove it when needed + if ($curlResponse->getReturnCode() == 417) { + $redirects++; + + if (empty($headers)) { + $headers = ['Expect:']; + } else { + if (!in_array('Expect:', $headers)) { + array_push($headers, 'Expect:'); + } + } + Logger::info('Server responds with 417, applying workaround', ['url' => $url]); + return self::post($url, $params, $headers, $redirects, $timeout); + } + + Logger::log('post_url: end ' . $url, Logger::DATA); + + return $curlResponse; + } + + /** + * Curl wrapper + * + * If binary flag is true, return binary results. + * Set the cookiejar argument to a string (e.g. "/tmp/friendica-cookies.txt") + * to preserve cookies from one request to the next. + * + * @param string $url URL to fetch + * @param bool $binary default false + * TRUE if asked to return binary results (file download) + * @param int $timeout Timeout in seconds, default system config value or 60 seconds + * @param string $accept_content supply Accept: header with 'accept_content' as the value + * @param string $cookiejar Path to cookie jar file + * @param int $redirects The recursion counter for internal use - default 0 + * + * @return string The fetched content + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + public static function fetchUrl(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) + { + $ret = self::fetchUrlFull($url, $binary, $timeout, $accept_content, $cookiejar, $redirects); + + return $ret->getBody(); + } + + /** + * Curl wrapper with array of return values. + * + * Inner workings and parameters are the same as @ref fetchUrl but returns an array with + * all the information collected during the fetch. + * + * @param string $url URL to fetch + * @param bool $binary default false + * TRUE if asked to return binary results (file download) + * @param int $timeout Timeout in seconds, default system config value or 60 seconds + * @param string $accept_content supply Accept: header with 'accept_content' as the value + * @param string $cookiejar Path to cookie jar file + * @param int $redirects The recursion counter for internal use - default 0 + * + * @return CurlResult With all relevant information, 'body' contains the actual fetched content. + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + public static function fetchUrlFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) + { + return self::curl( + $url, + $binary, + [ + 'timeout' => $timeout, + 'accept_content' => $accept_content, + 'cookiejar' => $cookiejar + ], + $redirects + ); + } +} diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 920dac47e6..dadd794fe6 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -166,7 +166,7 @@ class Probe Logger::info('Probing', ['host' => $host, 'ssl_url' => $ssl_url, 'url' => $url, 'callstack' => System::callstack(20)]); $xrd = null; - $curlResult = Network::curl($ssl_url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); + $curlResult = HTTPRequest::curl($ssl_url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); $ssl_connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0); if ($curlResult->isSuccess()) { $xml = $curlResult->getBody(); @@ -183,7 +183,7 @@ class Probe } if (!is_object($xrd) && !empty($url)) { - $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); + $curlResult = HTTPRequest::curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); $connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0); if ($curlResult->isTimeout()) { Logger::info('Probing timeout', ['url' => $url]); @@ -427,7 +427,7 @@ class Probe */ private static function getHideStatus($url) { - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if (!$curlResult->isSuccess()) { return false; } @@ -841,7 +841,7 @@ class Probe public static function pollZot($url, $data) { - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if ($curlResult->isTimeout()) { return $data; } @@ -938,7 +938,7 @@ class Probe { $xrd_timeout = DI::config()->get('system', 'xrd_timeout', 20); - $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => $type]); + $curlResult = HTTPRequest::curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => $type]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1007,7 +1007,7 @@ class Probe */ private static function pollNoscrape($noscrape_url, $data) { - $curlResult = Network::curl($noscrape_url); + $curlResult = HTTPRequest::curl($noscrape_url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1265,7 +1265,7 @@ class Probe */ private static function pollHcard($hcard_url, $data, $dfrn = false) { - $curlResult = Network::curl($hcard_url); + $curlResult = HTTPRequest::curl($hcard_url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1519,7 +1519,7 @@ class Probe $pubkey = substr($pubkey, 5); } } elseif (Strings::normaliseLink($pubkey) == 'http://') { - $curlResult = Network::curl($pubkey); + $curlResult = HTTPRequest::curl($pubkey); if ($curlResult->isTimeout()) { self::$istimeout = true; return $short ? false : []; @@ -1552,7 +1552,7 @@ class Probe } // Fetch all additional data from the feed - $curlResult = Network::curl($data["poll"]); + $curlResult = HTTPRequest::curl($data["poll"]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1604,7 +1604,7 @@ class Probe */ private static function pumpioProfileData($profile_link) { - $curlResult = Network::curl($profile_link); + $curlResult = HTTPRequest::curl($profile_link); if (!$curlResult->isSuccess()) { return []; } @@ -1835,7 +1835,7 @@ class Probe */ private static function feed($url, $probe = true) { - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index 2f8c2f419e..24c6250a41 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -21,12 +21,12 @@ namespace Friendica\Protocol; -use Friendica\Util\JsonLD; -use Friendica\Util\Network; use Friendica\Core\Protocol; use Friendica\Model\APContact; use Friendica\Model\User; +use Friendica\Network\HTTPRequest; use Friendica\Util\HTTPSignature; +use Friendica\Util\JsonLD; /** * ActivityPub Protocol class @@ -93,7 +93,7 @@ class ActivityPub return HTTPSignature::fetch($url, $uid); } - $curlResult = Network::curl($url, false, ['accept_content' => 'application/activity+json, application/ld+json']); + $curlResult = HTTPRequest::curl($url, false, ['accept_content' => 'application/activity+json, application/ld+json']); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { return false; } diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index b7c3204b79..21adc58cc0 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -43,6 +43,7 @@ use Friendica\Model\Post\Category; use Friendica\Model\Profile; use Friendica\Model\Tag; use Friendica\Model\User; +use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; @@ -1194,7 +1195,7 @@ class DFRN Logger::log('dfrn_deliver: ' . $url); - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if ($curlResult->isTimeout()) { return -2; // timed out @@ -1343,7 +1344,7 @@ class DFRN Logger::debug('dfrn_deliver', ['post' => $postvars]); - $postResult = Network::post($contact['notify'], $postvars); + $postResult = HTTPRequest::post($contact['notify'], $postvars); $xml = $postResult->getBody(); @@ -1440,7 +1441,7 @@ class DFRN $content_type = ($public_batch ? "application/magic-envelope+xml" : "application/json"); - $postResult = Network::post($dest_url, $envelope, ["Content-Type: ".$content_type]); + $postResult = HTTPRequest::post($dest_url, $envelope, ["Content-Type: " . $content_type]); $xml = $postResult->getBody(); $curl_stat = $postResult->getReturnCode(); diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index bd99b361e3..46d2bc1d4a 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -41,6 +41,7 @@ use Friendica\Model\Mail; use Friendica\Model\Post; use Friendica\Model\Tag; use Friendica\Model\User; +use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; @@ -1379,7 +1380,7 @@ class Diaspora Logger::log("Fetch post from ".$source_url, Logger::DEBUG); - $envelope = Network::fetchUrl($source_url); + $envelope = HTTPRequest::fetchUrl($source_url); if ($envelope) { Logger::log("Envelope was fetched.", Logger::DEBUG); $x = self::verifyMagicEnvelope($envelope); @@ -3260,7 +3261,7 @@ class Diaspora if (!intval(DI::config()->get("system", "diaspora_test"))) { $content_type = (($public_batch) ? "application/magic-envelope+xml" : "application/json"); - $postResult = Network::post($dest_url."/", $envelope, ["Content-Type: ".$content_type]); + $postResult = HTTPRequest::post($dest_url . "/", $envelope, ["Content-Type: " . $content_type]); $return_code = $postResult->getReturnCode(); } else { Logger::log("test_mode"); diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index 9a52476b56..779c99358e 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -39,10 +39,10 @@ use Friendica\Model\Item; use Friendica\Model\ItemURI; use Friendica\Model\Tag; use Friendica\Model\User; +use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Util\DateTimeFormat; use Friendica\Util\Images; -use Friendica\Util\Network; use Friendica\Util\Proxy as ProxyUtils; use Friendica\Util\Strings; use Friendica\Util\XML; @@ -756,7 +756,7 @@ class OStatus self::$conv_list[$conversation] = true; - $curlResult = Network::curl($conversation, false, ['accept_content' => 'application/atom+xml, text/html']); + $curlResult = HTTPRequest::curl($conversation, false, ['accept_content' => 'application/atom+xml, text/html']); if (!$curlResult->isSuccess()) { return; @@ -785,7 +785,7 @@ class OStatus } } if ($file != '') { - $conversation_atom = Network::curl($attribute['href']); + $conversation_atom = HTTPRequest::curl($attribute['href']); if ($conversation_atom->isSuccess()) { $xml = $conversation_atom->getBody(); @@ -902,7 +902,7 @@ class OStatus return; } - $curlResult = Network::curl($self); + $curlResult = HTTPRequest::curl($self); if (!$curlResult->isSuccess()) { return; @@ -949,7 +949,7 @@ class OStatus } $stored = false; - $curlResult = Network::curl($related, false, ['accept_content' => 'application/atom+xml, text/html']); + $curlResult = HTTPRequest::curl($related, false, ['accept_content' => 'application/atom+xml, text/html']); if (!$curlResult->isSuccess()) { return; @@ -980,7 +980,7 @@ class OStatus } } if ($atom_file != '') { - $curlResult = Network::curl($atom_file); + $curlResult = HTTPRequest::curl($atom_file); if ($curlResult->isSuccess()) { Logger::log('Fetched XML for URI ' . $related_uri, Logger::DEBUG); @@ -992,7 +992,7 @@ class OStatus // Workaround for older GNU Social servers if (($xml == '') && strstr($related, '/notice/')) { - $curlResult = Network::curl(str_replace('/notice/', '/api/statuses/show/', $related).'.atom'); + $curlResult = HTTPRequest::curl(str_replace('/notice/', '/api/statuses/show/', $related) . '.atom'); if ($curlResult->isSuccess()) { Logger::log('GNU Social workaround to fetch XML for URI ' . $related_uri, Logger::DEBUG); @@ -1003,7 +1003,7 @@ class OStatus // Even more worse workaround for GNU Social ;-) if ($xml == '') { $related_guess = self::convertHref($related_uri); - $curlResult = Network::curl(str_replace('/notice/', '/api/statuses/show/', $related_guess).'.atom'); + $curlResult = HTTPRequest::curl(str_replace('/notice/', '/api/statuses/show/', $related_guess) . '.atom'); if ($curlResult->isSuccess()) { Logger::log('GNU Social workaround 2 to fetch XML for URI ' . $related_uri, Logger::DEBUG); diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index d66676cac5..f2ad51070d 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -30,8 +30,8 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\GContact; use Friendica\Model\GServer; +use Friendica\Network\HTTPRequest; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Network; use Friendica\Util\Strings; /** @@ -103,7 +103,7 @@ class PortableContact Logger::log('load: ' . $url, Logger::DEBUG); - $fetchresult = Network::fetchUrlFull($url); + $fetchresult = HTTPRequest::fetchUrlFull($url); $s = $fetchresult->getBody(); Logger::log('load: returns ' . $s, Logger::DATA); @@ -251,7 +251,7 @@ class PortableContact */ private static function fetchServerlist($poco) { - $curlResult = Network::curl($poco . "/@server"); + $curlResult = HTTPRequest::curl($poco . "/@server"); if (!$curlResult->isSuccess()) { return; @@ -291,7 +291,7 @@ class PortableContact Logger::info("Fetch all users from the server " . $server["url"]); - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { $data = json_decode($curlResult->getBody(), true); @@ -314,7 +314,7 @@ class PortableContact $success = false; - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { Logger::info("Fetch all global contacts from the server " . $server["nurl"]); @@ -372,7 +372,7 @@ class PortableContact // Fetch all contacts from a given user from the other server $url = $server['poco'] . '/' . $username . '/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,contactType,generation'; - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if ($curlResult->isSuccess()) { $data = json_decode($curlResult->getBody(), true); diff --git a/src/Protocol/Salmon.php b/src/Protocol/Salmon.php index 7708459102..d01ea2ce12 100644 --- a/src/Protocol/Salmon.php +++ b/src/Protocol/Salmon.php @@ -22,9 +22,9 @@ namespace Friendica\Protocol; use Friendica\Core\Logger; +use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Util\Crypto; -use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; @@ -72,7 +72,7 @@ class Salmon $ret[$x] = substr($ret[$x], 5); } } elseif (Strings::normaliseLink($ret[$x]) == 'http://') { - $ret[$x] = Network::fetchUrl($ret[$x]); + $ret[$x] = HTTPRequest::fetchUrl($ret[$x]); } } } @@ -155,7 +155,7 @@ class Salmon $salmon = XML::fromArray($xmldata, $xml, false, $namespaces); // slap them - $postResult = Network::post($url, $salmon, [ + $postResult = HTTPRequest::post($url, $salmon, [ 'Content-type: application/magic-envelope+xml', 'Content-length: ' . strlen($salmon) ]); @@ -180,7 +180,7 @@ class Salmon $salmon = XML::fromArray($xmldata, $xml, false, $namespaces); // slap them - $postResult = Network::post($url, $salmon, [ + $postResult = HTTPRequest::post($url, $salmon, [ 'Content-type: application/magic-envelope+xml', 'Content-length: ' . strlen($salmon) ]); @@ -203,7 +203,7 @@ class Salmon $salmon = XML::fromArray($xmldata, $xml, false, $namespaces); // slap them - $postResult = Network::post($url, $salmon, [ + $postResult = HTTPRequest::post($url, $salmon, [ 'Content-type: application/magic-envelope+xml', 'Content-length: ' . strlen($salmon)]); $return_code = $postResult->getReturnCode(); diff --git a/src/Util/ExAuth.php b/src/Util/ExAuth.php index de13ee82f5..710c009796 100644 --- a/src/Util/ExAuth.php +++ b/src/Util/ExAuth.php @@ -37,6 +37,7 @@ namespace Friendica\Util; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\User; +use Friendica\Network\HTTPRequest; class ExAuth { @@ -181,7 +182,7 @@ class ExAuth $url = ($ssl ? 'https' : 'http') . '://' . $host . '/noscrape/' . $user; - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if (!$curlResult->isSuccess()) { return false; diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index d151516bea..5d57c2281f 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -21,11 +21,12 @@ namespace Friendica\Util; -use Friendica\Database\DBA; use Friendica\Core\Logger; +use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Model\User; use Friendica\Model\APContact; +use Friendica\Model\User; +use Friendica\Network\HTTPRequest; /** * Implements HTTP Signatures per draft-cavage-http-signatures-07. @@ -297,7 +298,7 @@ class HTTPSignature $headers[] = 'Content-Type: application/activity+json'; - $postResult = Network::post($target, $content, $headers); + $postResult = HTTPRequest::post($target, $content, $headers); $return_code = $postResult->getReturnCode(); Logger::log('Transmit to ' . $target . ' returned ' . $return_code, Logger::DEBUG); @@ -442,7 +443,7 @@ class HTTPSignature $curl_opts = $opts; $curl_opts['header'] = $headers; - $curlResult = Network::curl($request, false, $curl_opts); + $curlResult = HTTPRequest::curl($request, false, $curl_opts); $return_code = $curlResult->getReturnCode(); Logger::log('Fetched for user ' . $uid . ' from ' . $request . ' returned ' . $return_code, Logger::DEBUG); diff --git a/src/Util/Images.php b/src/Util/Images.php index 35f0cfc042..9e3be4f4f9 100644 --- a/src/Util/Images.php +++ b/src/Util/Images.php @@ -24,6 +24,7 @@ namespace Friendica\Util; use Friendica\Core\Logger; use Friendica\Core\System; use Friendica\DI; +use Friendica\Network\HTTPRequest; /** * Image utilities @@ -184,7 +185,7 @@ class Images return $data; } - $img_str = Network::fetchUrl($url, true, 4); + $img_str = HTTPRequest::fetchUrl($url, true, 4); if (!$img_str) { return []; diff --git a/src/Util/Network.php b/src/Util/Network.php index ddec359907..888dc20a69 100644 --- a/src/Util/Network.php +++ b/src/Util/Network.php @@ -27,340 +27,9 @@ use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\System; use Friendica\DI; -use Friendica\Network\CurlResult; class Network { - /** - * Curl wrapper - * - * If binary flag is true, return binary results. - * Set the cookiejar argument to a string (e.g. "/tmp/friendica-cookies.txt") - * to preserve cookies from one request to the next. - * - * @param string $url URL to fetch - * @param bool $binary default false - * TRUE if asked to return binary results (file download) - * @param int $timeout Timeout in seconds, default system config value or 60 seconds - * @param string $accept_content supply Accept: header with 'accept_content' as the value - * @param string $cookiejar Path to cookie jar file - * @param int $redirects The recursion counter for internal use - default 0 - * - * @return string The fetched content - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function fetchUrl(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) - { - $ret = self::fetchUrlFull($url, $binary, $timeout, $accept_content, $cookiejar, $redirects); - - return $ret->getBody(); - } - - /** - * Curl wrapper with array of return values. - * - * Inner workings and parameters are the same as @ref fetchUrl but returns an array with - * all the information collected during the fetch. - * - * @param string $url URL to fetch - * @param bool $binary default false - * TRUE if asked to return binary results (file download) - * @param int $timeout Timeout in seconds, default system config value or 60 seconds - * @param string $accept_content supply Accept: header with 'accept_content' as the value - * @param string $cookiejar Path to cookie jar file - * @param int $redirects The recursion counter for internal use - default 0 - * - * @return CurlResult With all relevant information, 'body' contains the actual fetched content. - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function fetchUrlFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) - { - return self::curl( - $url, - $binary, - [ - 'timeout' => $timeout, - 'accept_content' => $accept_content, - 'cookiejar' => $cookiejar - ], - $redirects - ); - } - - /** - * fetches an URL. - * - * @param string $url URL to fetch - * @param bool $binary default false - * TRUE if asked to return binary results (file download) - * @param array $opts (optional parameters) assoziative array with: - * 'accept_content' => supply Accept: header with 'accept_content' as the value - * 'timeout' => int Timeout in seconds, default system config value or 60 seconds - * 'http_auth' => username:password - * 'novalidate' => do not validate SSL certs, default is to validate using our CA list - * 'nobody' => only return the header - * 'cookiejar' => path to cookie jar file - * 'header' => header array - * @param int $redirects The recursion counter for internal use - default 0 - * - * @return CurlResult - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function curl(string $url, bool $binary = false, array $opts = [], int &$redirects = 0) - { - $stamp1 = microtime(true); - - $a = DI::app(); - - if (strlen($url) > 1000) { - Logger::log('URL is longer than 1000 characters. Callstack: ' . System::callstack(20), Logger::DEBUG); - return CurlResult::createErrorCurl(substr($url, 0, 200)); - } - - $parts2 = []; - $parts = parse_url($url); - $path_parts = explode('/', $parts['path'] ?? ''); - foreach ($path_parts as $part) { - if (strlen($part) <> mb_strlen($part)) { - $parts2[] = rawurlencode($part); - } else { - $parts2[] = $part; - } - } - $parts['path'] = implode('/', $parts2); - $url = self::unparseURL($parts); - - if (self::isUrlBlocked($url)) { - Logger::log('domain of ' . $url . ' is blocked', Logger::DATA); - return CurlResult::createErrorCurl($url); - } - - $ch = @curl_init($url); - - if (($redirects > 8) || (!$ch)) { - return CurlResult::createErrorCurl($url); - } - - @curl_setopt($ch, CURLOPT_HEADER, true); - - if (!empty($opts['cookiejar'])) { - curl_setopt($ch, CURLOPT_COOKIEJAR, $opts["cookiejar"]); - curl_setopt($ch, CURLOPT_COOKIEFILE, $opts["cookiejar"]); - } - - // These settings aren't needed. We're following the location already. - // @curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true); - // @curl_setopt($ch, CURLOPT_MAXREDIRS, 5); - - if (!empty($opts['accept_content'])) { - curl_setopt( - $ch, - CURLOPT_HTTPHEADER, - ['Accept: ' . $opts['accept_content']] - ); - } - - if (!empty($opts['header'])) { - curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['header']); - } - - @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - @curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); - - $range = intval(DI::config()->get('system', 'curl_range_bytes', 0)); - - if ($range > 0) { - @curl_setopt($ch, CURLOPT_RANGE, '0-' . $range); - } - - // Without this setting it seems as if some webservers send compressed content - // This seems to confuse curl so that it shows this uncompressed. - /// @todo We could possibly set this value to "gzip" or something similar - curl_setopt($ch, CURLOPT_ENCODING, ''); - - if (!empty($opts['headers'])) { - @curl_setopt($ch, CURLOPT_HTTPHEADER, $opts['headers']); - } - - if (!empty($opts['nobody'])) { - @curl_setopt($ch, CURLOPT_NOBODY, $opts['nobody']); - } - - if (!empty($opts['timeout'])) { - @curl_setopt($ch, CURLOPT_TIMEOUT, $opts['timeout']); - } else { - $curl_time = DI::config()->get('system', 'curl_timeout', 60); - @curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time)); - } - - // by default we will allow self-signed certs - // but you can override this - - $check_cert = DI::config()->get('system', 'verifyssl'); - @curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false)); - - if ($check_cert) { - @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); - } - - $proxy = DI::config()->get('system', 'proxy'); - - if (strlen($proxy)) { - @curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1); - @curl_setopt($ch, CURLOPT_PROXY, $proxy); - $proxyuser = @DI::config()->get('system', 'proxyuser'); - - if (strlen($proxyuser)) { - @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser); - } - } - - if (DI::config()->get('system', 'ipv4_resolve', false)) { - curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); - } - - if ($binary) { - @curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1); - } - - // don't let curl abort the entire application - // if it throws any errors. - - $s = @curl_exec($ch); - $curl_info = @curl_getinfo($ch); - - // Special treatment for HTTP Code 416 - // See https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/416 - if (($curl_info['http_code'] == 416) && ($range > 0)) { - @curl_setopt($ch, CURLOPT_RANGE, ''); - $s = @curl_exec($ch); - $curl_info = @curl_getinfo($ch); - } - - $curlResponse = new CurlResult($url, $s, $curl_info, curl_errno($ch), curl_error($ch)); - - if ($curlResponse->isRedirectUrl()) { - $redirects++; - Logger::log('curl: redirect ' . $url . ' to ' . $curlResponse->getRedirectUrl()); - @curl_close($ch); - return self::curl($curlResponse->getRedirectUrl(), $binary, $opts, $redirects); - } - - @curl_close($ch); - - DI::profiler()->saveTimestamp($stamp1, 'network', System::callstack()); - - return $curlResponse; - } - - /** - * Send POST request to $url - * - * @param string $url URL to post - * @param mixed $params array of POST variables - * @param array $headers HTTP headers - * @param int $redirects Recursion counter for internal use - default = 0 - * @param int $timeout The timeout in seconds, default system config value or 60 seconds - * - * @return CurlResult The content - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function post(string $url, $params, array $headers = [], int $timeout = 0, int &$redirects = 0) - { - $stamp1 = microtime(true); - - if (self::isUrlBlocked($url)) { - Logger::log('post_url: domain of ' . $url . ' is blocked', Logger::DATA); - return CurlResult::createErrorCurl($url); - } - - $a = DI::app(); - $ch = curl_init($url); - - if (($redirects > 8) || (!$ch)) { - return CurlResult::createErrorCurl($url); - } - - Logger::log('post_url: start ' . $url, Logger::DATA); - - curl_setopt($ch, CURLOPT_HEADER, true); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_POST, 1); - curl_setopt($ch, CURLOPT_POSTFIELDS, $params); - curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); - - if (DI::config()->get('system', 'ipv4_resolve', false)) { - curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); - } - - if (intval($timeout)) { - curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); - } else { - $curl_time = DI::config()->get('system', 'curl_timeout', 60); - curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time)); - } - - if (!empty($headers)) { - curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); - } - - $check_cert = DI::config()->get('system', 'verifyssl'); - curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false)); - - if ($check_cert) { - @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); - } - - $proxy = DI::config()->get('system', 'proxy'); - - if (strlen($proxy)) { - curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1); - curl_setopt($ch, CURLOPT_PROXY, $proxy); - $proxyuser = DI::config()->get('system', 'proxyuser'); - if (strlen($proxyuser)) { - curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser); - } - } - - // don't let curl abort the entire application - // if it throws any errors. - - $s = @curl_exec($ch); - - $curl_info = curl_getinfo($ch); - - $curlResponse = new CurlResult($url, $s, $curl_info, curl_errno($ch), curl_error($ch)); - - if ($curlResponse->isRedirectUrl()) { - $redirects++; - Logger::log('post_url: redirect ' . $url . ' to ' . $curlResponse->getRedirectUrl()); - curl_close($ch); - return self::post($curlResponse->getRedirectUrl(), $params, $headers, $redirects, $timeout); - } - - curl_close($ch); - - DI::profiler()->saveTimestamp($stamp1, 'network', System::callstack()); - - // Very old versions of Lighttpd don't like the "Expect" header, so we remove it when needed - if ($curlResponse->getReturnCode() == 417) { - $redirects++; - - if (empty($headers)) { - $headers = ['Expect:']; - } else { - if (!in_array('Expect:', $headers)) { - array_push($headers, 'Expect:'); - } - } - Logger::info('Server responds with 417, applying workaround', ['url' => $url]); - return self::post($url, $params, $headers, $redirects, $timeout); - } - - Logger::log('post_url: end ' . $url, Logger::DATA); - - return $curlResponse; - } /** * Return raw post data from a post request diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index b6d172a3a1..577ffd4c11 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -27,6 +27,7 @@ use Friendica\Content\OEmbed; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Database\DBA; +use Friendica\Network\HTTPRequest; /** * Get information about a given URL @@ -159,7 +160,7 @@ class ParseUrl return $siteinfo; } - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if (!$curlResult->isSuccess()) { return $siteinfo; } diff --git a/src/Worker/CheckVersion.php b/src/Worker/CheckVersion.php index 9572342c3e..f0369daab3 100644 --- a/src/Worker/CheckVersion.php +++ b/src/Worker/CheckVersion.php @@ -24,7 +24,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; /** * Check the git repository VERSION file and save the version to the DB @@ -55,7 +55,7 @@ class CheckVersion Logger::log("Checking VERSION from: ".$checked_url, Logger::DEBUG); // fetch the VERSION file - $gitversion = DBA::escape(trim(Network::fetchUrl($checked_url))); + $gitversion = DBA::escape(trim(HTTPRequest::fetchUrl($checked_url))); Logger::log("Upstream VERSION is: ".$gitversion, Logger::DEBUG); DI::config()->set('system', 'git_friendica_version', $gitversion); diff --git a/src/Worker/CronJobs.php b/src/Worker/CronJobs.php index 319a369d1f..23434beb14 100644 --- a/src/Worker/CronJobs.php +++ b/src/Worker/CronJobs.php @@ -30,12 +30,10 @@ use Friendica\Database\PostUpdate; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\GContact; -use Friendica\Model\GServer; use Friendica\Model\Nodeinfo; use Friendica\Model\Photo; use Friendica\Model\User; -use Friendica\Network\Probe; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; use Friendica\Util\Proxy as ProxyUtils; use Friendica\Util\Strings; @@ -63,7 +61,7 @@ class CronJobs // Now trying to register $url = 'http://the-federation.info/register/' . DI::baseUrl()->getHostname(); Logger::debug('Check registering url', ['url' => $url]); - $ret = Network::fetchUrl($url); + $ret = HTTPRequest::fetchUrl($url); Logger::debug('Check registering answer', ['answer' => $ret]); Logger::info('cron_end'); break; diff --git a/src/Worker/Directory.php b/src/Worker/Directory.php index 6c6d26f26c..0dea9841f0 100644 --- a/src/Worker/Directory.php +++ b/src/Worker/Directory.php @@ -26,7 +26,7 @@ use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; /** * Sends updated profile data to the directory @@ -54,7 +54,7 @@ class Directory Logger::log('Updating directory: ' . $arr['url'], Logger::DEBUG); if (strlen($arr['url'])) { - Network::fetchUrl($dir . '?url=' . bin2hex($arr['url'])); + HTTPRequest::fetchUrl($dir . '?url=' . bin2hex($arr['url'])); } return; diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index fa8f748334..13d1b8ee7a 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -28,13 +28,13 @@ use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Item; use Friendica\Model\User; +use Friendica\Network\HTTPRequest; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Email; use Friendica\Protocol\Feed; use Friendica\Protocol\PortableContact; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Network; use Friendica\Util\Strings; use Friendica\Util\XML; @@ -291,7 +291,7 @@ class OnePoll . '&type=data&last_update=' . $last_update . '&perm=' . $perm; - $curlResult = Network::curl($url); + $curlResult = HTTPRequest::curl($url); if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) { // set the last-update so we don't keep polling @@ -405,7 +405,7 @@ class OnePoll $postvars['dfrn_version'] = DFRN_PROTOCOL_VERSION; $postvars['perm'] = 'rw'; - return Network::post($contact['poll'], $postvars)->getBody(); + return HTTPRequest::post($contact['poll'], $postvars)->getBody(); } /** @@ -444,7 +444,7 @@ class OnePoll } $cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-'); - $curlResult = Network::curl($contact['poll'], false, ['cookiejar' => $cookiejar]); + $curlResult = HTTPRequest::curl($contact['poll'], false, ['cookiejar' => $cookiejar]); unlink($cookiejar); if ($curlResult->isTimeout()) { @@ -756,7 +756,7 @@ class OnePoll DBA::update('contact', ['hub-verify' => $verify_token], ['id' => $contact['id']]); } - $postResult = Network::post($url, $params); + $postResult = HTTPRequest::post($url, $params); Logger::log('subscribe_to_hub: returns: ' . $postResult->getReturnCode(), Logger::DEBUG); diff --git a/src/Worker/PubSubPublish.php b/src/Worker/PubSubPublish.php index 2eb94eeb72..4fa3b3e894 100644 --- a/src/Worker/PubSubPublish.php +++ b/src/Worker/PubSubPublish.php @@ -25,8 +25,8 @@ use Friendica\Core\Logger; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\PushSubscriber; +use Friendica\Network\HTTPRequest; use Friendica\Protocol\OStatus; -use Friendica\Util\Network; class PubSubPublish { @@ -68,7 +68,7 @@ class PubSubPublish Logger::log('POST ' . print_r($headers, true) . "\n" . $params, Logger::DATA); - $postResult = Network::post($subscriber['callback_url'], $params, $headers); + $postResult = HTTPRequest::post($subscriber['callback_url'], $params, $headers); $ret = $postResult->getReturnCode(); if ($ret >= 200 && $ret <= 299) { diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php index afe54e5fb1..1dcf0c8db6 100644 --- a/src/Worker/SearchDirectory.php +++ b/src/Worker/SearchDirectory.php @@ -30,7 +30,7 @@ use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\GContact; use Friendica\Model\GServer; -use Friendica\Util\Network; +use Friendica\Network\HTTPRequest; use Friendica\Util\Strings; class SearchDirectory @@ -52,7 +52,7 @@ class SearchDirectory } } - $x = Network::fetchUrl(Search::getGlobalDirectory() . '/lsearch?p=1&n=500&search=' . urlencode($search)); + $x = HTTPRequest::fetchUrl(Search::getGlobalDirectory() . '/lsearch?p=1&n=500&search=' . urlencode($search)); $j = json_decode($x); if (!empty($j->results)) { From 9d00e4f1bc8f6ec95bd4ab450676ab039b2ee2f9 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:11:01 +0100 Subject: [PATCH 02/17] Introduce HTPPRequest DI call and constructor --- src/DI.php | 12 +++++++ src/Network/HTTPRequest.php | 65 ++++++++++++++++++++++++------------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/src/DI.php b/src/DI.php index 9ed0c5b24d..5986ca9615 100644 --- a/src/DI.php +++ b/src/DI.php @@ -323,6 +323,18 @@ abstract class DI return self::$dice->create(Model\Storage\IStorage::class); } + // + // "Network" namespace + // + + /** + * @return Network\HTTPRequest + */ + public static function httpRequest() + { + return self::$dice->create(Network\HTTPRequest::class); + } + // // "Repository" namespace // diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 6986a93599..7d7d59a6de 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -21,23 +21,44 @@ namespace Friendica\Network; +use Friendica\App; +use Friendica\Core\Config\IConfig; use Friendica\Core\Logger; use Friendica\Core\System; use Friendica\DI; use Friendica\Util\Network; +use Friendica\Util\Profiler; +use Psr\Log\LoggerInterface; /** * Performs HTTP requests to a given URL */ class HTTPRequest { + /** @var LoggerInterface */ + private $logger; + /** @var Profiler */ + private $profiler; + /** @var IConfig */ + private $config; + /** @var string */ + private $userAgent; + + public function __construct(LoggerInterface $logger, Profiler $profiler, IConfig $config, App $a) + { + $this->logger = $logger; + $this->profiler = $profiler; + $this->config = $config; + $this->userAgent = $a->getUserAgent(); + } + /** * fetches an URL. * - * @param string $url URL to fetch - * @param bool $binary default false + * @param string $url URL to fetch + * @param bool $binary default false * TRUE if asked to return binary results (file download) - * @param array $opts (optional parameters) assoziative array with: + * @param array $opts (optional parameters) assoziative array with: * 'accept_content' => supply Accept: header with 'accept_content' as the value * 'timeout' => int Timeout in seconds, default system config value or 60 seconds * 'http_auth' => username:password @@ -45,7 +66,7 @@ class HTTPRequest * 'nobody' => only return the header * 'cookiejar' => path to cookie jar file * 'header' => header array - * @param int $redirects The recursion counter for internal use - default 0 + * @param int $redirects The recursion counter for internal use - default 0 * * @return CurlResult * @throws \Friendica\Network\HTTPException\InternalServerErrorException @@ -54,8 +75,6 @@ class HTTPRequest { $stamp1 = microtime(true); - $a = DI::app(); - if (strlen($url) > 1000) { Logger::log('URL is longer than 1000 characters. Callstack: ' . System::callstack(20), Logger::DEBUG); return CurlResult::createErrorCurl(substr($url, 0, 200)); @@ -200,11 +219,11 @@ class HTTPRequest /** * Send POST request to $url * - * @param string $url URL to post - * @param mixed $params array of POST variables - * @param array $headers HTTP headers - * @param int $redirects Recursion counter for internal use - default = 0 - * @param int $timeout The timeout in seconds, default system config value or 60 seconds + * @param string $url URL to post + * @param mixed $params array of POST variables + * @param array $headers HTTP headers + * @param int $redirects Recursion counter for internal use - default = 0 + * @param int $timeout The timeout in seconds, default system config value or 60 seconds * * @return CurlResult The content * @throws \Friendica\Network\HTTPException\InternalServerErrorException @@ -313,13 +332,13 @@ class HTTPRequest * Set the cookiejar argument to a string (e.g. "/tmp/friendica-cookies.txt") * to preserve cookies from one request to the next. * - * @param string $url URL to fetch - * @param bool $binary default false + * @param string $url URL to fetch + * @param bool $binary default false * TRUE if asked to return binary results (file download) - * @param int $timeout Timeout in seconds, default system config value or 60 seconds - * @param string $accept_content supply Accept: header with 'accept_content' as the value - * @param string $cookiejar Path to cookie jar file - * @param int $redirects The recursion counter for internal use - default 0 + * @param int $timeout Timeout in seconds, default system config value or 60 seconds + * @param string $accept_content supply Accept: header with 'accept_content' as the value + * @param string $cookiejar Path to cookie jar file + * @param int $redirects The recursion counter for internal use - default 0 * * @return string The fetched content * @throws \Friendica\Network\HTTPException\InternalServerErrorException @@ -337,13 +356,13 @@ class HTTPRequest * Inner workings and parameters are the same as @ref fetchUrl but returns an array with * all the information collected during the fetch. * - * @param string $url URL to fetch - * @param bool $binary default false + * @param string $url URL to fetch + * @param bool $binary default false * TRUE if asked to return binary results (file download) - * @param int $timeout Timeout in seconds, default system config value or 60 seconds - * @param string $accept_content supply Accept: header with 'accept_content' as the value - * @param string $cookiejar Path to cookie jar file - * @param int $redirects The recursion counter for internal use - default 0 + * @param int $timeout Timeout in seconds, default system config value or 60 seconds + * @param string $accept_content supply Accept: header with 'accept_content' as the value + * @param string $cookiejar Path to cookie jar file + * @param int $redirects The recursion counter for internal use - default 0 * * @return CurlResult With all relevant information, 'body' contains the actual fetched content. * @throws \Friendica\Network\HTTPException\InternalServerErrorException From 2973ed6448f56dd807df3ec0d20d095226d14b65 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:15:46 +0100 Subject: [PATCH 03/17] Make "HTTPRequest::curl" dynamic --- mod/ostatus_subscribe.php | 3 +-- mod/parse_url.php | 3 +-- mod/redir.php | 3 +-- src/Core/Search.php | 2 +- src/Model/GContact.php | 7 +++---- src/Model/GServer.php | 36 ++++++++++++++++---------------- src/Model/Photo.php | 3 +-- src/Model/Profile.php | 3 +-- src/Module/Admin/Summary.php | 3 +-- src/Module/Magic.php | 3 +-- src/Network/HTTPRequest.php | 28 ++++++++++++------------- src/Network/Probe.php | 22 +++++++++---------- src/Protocol/ActivityPub.php | 3 +-- src/Protocol/DFRN.php | 2 +- src/Protocol/OStatus.php | 15 +++++++------ src/Protocol/PortableContact.php | 8 +++---- src/Util/ExAuth.php | 3 +-- src/Util/HTTPSignature.php | 2 +- src/Util/ParseUrl.php | 3 +-- src/Worker/OnePoll.php | 4 ++-- 20 files changed, 72 insertions(+), 84 deletions(-) diff --git a/mod/ostatus_subscribe.php b/mod/ostatus_subscribe.php index 6b6c94987d..5a3a625cec 100644 --- a/mod/ostatus_subscribe.php +++ b/mod/ostatus_subscribe.php @@ -23,7 +23,6 @@ use Friendica\App; use Friendica\Core\Protocol; use Friendica\DI; use Friendica\Model\Contact; -use Friendica\Network\HTTPRequest; function ostatus_subscribe_content(App $a) { @@ -55,7 +54,7 @@ function ostatus_subscribe_content(App $a) $api = $contact['baseurl'] . '/api/'; // Fetching friends - $curlResult = HTTPRequest::curl($api . 'statuses/friends.json?screen_name=' . $contact['nick']); + $curlResult = DI::httpRequest()->curl($api . 'statuses/friends.json?screen_name=' . $contact['nick']); if (!$curlResult->isSuccess()) { DI::pConfig()->delete($uid, 'ostatus', 'legacy_contact'); diff --git a/mod/parse_url.php b/mod/parse_url.php index 49e41246c9..0e80d971a6 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -28,7 +28,6 @@ use Friendica\Content\PageInfo; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\System; -use Friendica\Network\HTTPRequest; use Friendica\Util\ParseUrl; use Friendica\Util\Strings; @@ -85,7 +84,7 @@ function parse_url_content(App $a) // Check if the URL is an image, video or audio file. If so format // the URL with the corresponding BBCode media tag // Fetch the header of the URL - $curlResponse = HTTPRequest::curl($url, false, ['novalidate' => true, 'nobody' => true]); + $curlResponse = DI::httpRequest()->curl($url, false, ['novalidate' => true, 'nobody' => true]); if ($curlResponse->isSuccess()) { // Convert the header fields into an array diff --git a/mod/redir.php b/mod/redir.php index deb97ca1c6..4069518cdd 100644 --- a/mod/redir.php +++ b/mod/redir.php @@ -27,7 +27,6 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Profile; -use Friendica\Network\HTTPRequest; use Friendica\Util\Strings; function redir_init(App $a) { @@ -171,7 +170,7 @@ function redir_magic($a, $cid, $url) } // Test for magic auth on the target system - $serverret = HTTPRequest::curl($basepath . '/magic'); + $serverret = DI::httpRequest()->curl($basepath . '/magic'); if ($serverret->isSuccess()) { $separator = strpos($target_url, '?') ? '&' : '?'; $target_url .= $separator . 'zrl=' . urlencode($visitor) . '&addr=' . urlencode($contact_url); diff --git a/src/Core/Search.php b/src/Core/Search.php index 26af05e746..aafa4024a8 100644 --- a/src/Core/Search.php +++ b/src/Core/Search.php @@ -285,7 +285,7 @@ class Search $return = GContact::searchByName($search, $mode); } else { $p = $page > 1 ? 'p=' . $page : ''; - $curlResult = HTTPRequest::curl(self::getGlobalDirectory() . '/search/people?' . $p . '&q=' . urlencode($search), false, ['accept_content' => 'application/json']); + $curlResult = DI::httpRequest()->curl(self::getGlobalDirectory() . '/search/people?' . $p . '&q=' . urlencode($search), false, ['accept_content' => 'application/json']); if ($curlResult->isSuccess()) { $searchResult = json_decode($curlResult->getBody(), true); if (!empty($searchResult['profiles'])) { diff --git a/src/Model/GContact.php b/src/Model/GContact.php index a7cf837bdd..669bc60c5d 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -36,7 +36,6 @@ use Friendica\Network\Probe; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\PortableContact; use Friendica\Util\DateTimeFormat; -use Friendica\Util\Network; use Friendica\Util\Strings; /** @@ -846,7 +845,7 @@ class GContact return false; } - $curlResult = HTTPRequest::curl($gserver['noscrape'] . '/' . $data['nick']); + $curlResult = DI::httpRequest()->curl($gserver['noscrape'] . '/' . $data['nick']); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { $noscrape = json_decode($curlResult->getBody(), true); @@ -928,7 +927,7 @@ class GContact private static function updateFromFeed(array $data) { // Search for the newest entry in the feed - $curlResult = HTTPRequest::curl($data['poll']); + $curlResult = DI::httpRequest()->curl($data['poll']); if (!$curlResult->isSuccess()) { $fields = ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()]; DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]); @@ -1206,7 +1205,7 @@ class GContact $url = $server . '/main/statistics'; - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if (!$curlResult->isSuccess()) { return false; } diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 80ef201a8a..c868e19d40 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -310,7 +310,7 @@ class GServer // When a nodeinfo is present, we don't need to dig further $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = HTTPRequest::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpRequest()->curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); if ($curlResult->isTimeout()) { self::setFailure($url); return false; @@ -343,7 +343,7 @@ class GServer $basedata = ['detection-method' => self::DETECT_MANUAL]; } - $curlResult = HTTPRequest::curl($baseurl, false, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpRequest()->curl($baseurl, false, ['timeout' => $xrd_timeout]); if ($curlResult->isSuccess()) { $basedata = self::analyseRootHeader($curlResult, $basedata); $basedata = self::analyseRootBody($curlResult, $basedata, $baseurl); @@ -499,7 +499,7 @@ class GServer { Logger::info('Discover relay data', ['server' => $server_url]); - $curlResult = HTTPRequest::curl($server_url . '/.well-known/x-social-relay'); + $curlResult = DI::httpRequest()->curl($server_url . '/.well-known/x-social-relay'); if (!$curlResult->isSuccess()) { return; } @@ -580,7 +580,7 @@ class GServer */ private static function fetchStatistics(string $url) { - $curlResult = HTTPRequest::curl($url . '/statistics.json'); + $curlResult = DI::httpRequest()->curl($url . '/statistics.json'); if (!$curlResult->isSuccess()) { return []; } @@ -690,7 +690,7 @@ class GServer */ private static function parseNodeinfo1(string $nodeinfo_url) { - $curlResult = HTTPRequest::curl($nodeinfo_url); + $curlResult = DI::httpRequest()->curl($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; @@ -767,7 +767,7 @@ class GServer */ private static function parseNodeinfo2(string $nodeinfo_url) { - $curlResult = HTTPRequest::curl($nodeinfo_url); + $curlResult = DI::httpRequest()->curl($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; } @@ -844,7 +844,7 @@ class GServer */ private static function fetchSiteinfo(string $url, array $serverdata) { - $curlResult = HTTPRequest::curl($url . '/siteinfo.json'); + $curlResult = DI::httpRequest()->curl($url . '/siteinfo.json'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -913,7 +913,7 @@ class GServer private static function validHostMeta(string $url) { $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = HTTPRequest::curl($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpRequest()->curl($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); if (!$curlResult->isSuccess()) { return false; } @@ -1009,7 +1009,7 @@ class GServer { $serverdata['poco'] = ''; - $curlResult = HTTPRequest::curl($url . '/poco'); + $curlResult = DI::httpRequest()->curl($url . '/poco'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1039,7 +1039,7 @@ class GServer */ public static function checkMastodonDirectory(string $url, array $serverdata) { - $curlResult = HTTPRequest::curl($url . '/api/v1/directory?limit=1'); + $curlResult = DI::httpRequest()->curl($url . '/api/v1/directory?limit=1'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1066,7 +1066,7 @@ class GServer */ private static function detectNextcloud(string $url, array $serverdata) { - $curlResult = HTTPRequest::curl($url . '/status.php'); + $curlResult = DI::httpRequest()->curl($url . '/status.php'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -1100,7 +1100,7 @@ class GServer */ private static function detectMastodonAlikes(string $url, array $serverdata) { - $curlResult = HTTPRequest::curl($url . '/api/v1/instance'); + $curlResult = DI::httpRequest()->curl($url . '/api/v1/instance'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -1166,7 +1166,7 @@ class GServer */ private static function detectHubzilla(string $url, array $serverdata) { - $curlResult = HTTPRequest::curl($url . '/api/statusnet/config.json'); + $curlResult = DI::httpRequest()->curl($url . '/api/statusnet/config.json'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1264,7 +1264,7 @@ class GServer private static function detectGNUSocial(string $url, array $serverdata) { // Test for GNU Social - $curlResult = HTTPRequest::curl($url . '/api/gnusocial/version.json'); + $curlResult = DI::httpRequest()->curl($url . '/api/gnusocial/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { $serverdata['platform'] = 'gnusocial'; @@ -1282,7 +1282,7 @@ class GServer } // Test for Statusnet - $curlResult = HTTPRequest::curl($url . '/api/statusnet/version.json'); + $curlResult = DI::httpRequest()->curl($url . '/api/statusnet/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { @@ -1318,9 +1318,9 @@ class GServer */ private static function detectFriendica(string $url, array $serverdata) { - $curlResult = HTTPRequest::curl($url . '/friendica/json'); + $curlResult = DI::httpRequest()->curl($url . '/friendica/json'); if (!$curlResult->isSuccess()) { - $curlResult = HTTPRequest::curl($url . '/friendika/json'); + $curlResult = DI::httpRequest()->curl($url . '/friendika/json'); $friendika = true; $platform = 'Friendika'; } else { @@ -1653,7 +1653,7 @@ class GServer if (!empty($accesstoken)) { $api = 'https://instances.social/api/1.0/instances/list?count=0'; $header = ['Authorization: Bearer '.$accesstoken]; - $curlResult = HTTPRequest::curl($api, false, ['headers' => $header]); + $curlResult = DI::httpRequest()->curl($api, false, ['headers' => $header]); if ($curlResult->isSuccess()) { $servers = json_decode($curlResult->getBody(), true); diff --git a/src/Model/Photo.php b/src/Model/Photo.php index 125718bf50..a10711db2a 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -28,7 +28,6 @@ use Friendica\Database\DBA; use Friendica\Database\DBStructure; use Friendica\DI; use Friendica\Model\Storage\SystemResource; -use Friendica\Network\HTTPRequest; use Friendica\Object\Image; use Friendica\Util\DateTimeFormat; use Friendica\Util\Images; @@ -421,7 +420,7 @@ class Photo $filename = basename($image_url); if (!empty($image_url)) { - $ret = HTTPRequest::curl($image_url, true); + $ret = DI::httpRequest()->curl($image_url, true); $img_str = $ret->getBody(); $type = $ret->getContentType(); } else { diff --git a/src/Model/Profile.php b/src/Model/Profile.php index c8fd9d029c..d32940ae67 100644 --- a/src/Model/Profile.php +++ b/src/Model/Profile.php @@ -33,7 +33,6 @@ use Friendica\Core\Session; use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Network\HTTPRequest; use Friendica\Protocol\Activity; use Friendica\Protocol\Diaspora; use Friendica\Util\DateTimeFormat; @@ -738,7 +737,7 @@ class Profile $magic_path = $basepath . '/magic' . '?owa=1&dest=' . $dest . '&' . $addr_request; // We have to check if the remote server does understand /magic without invoking something - $serverret = HTTPRequest::curl($basepath . '/magic'); + $serverret = DI::httpRequest()->curl($basepath . '/magic'); if ($serverret->isSuccess()) { Logger::log('Doing magic auth for visitor ' . $my_url . ' to ' . $magic_path, Logger::DEBUG); System::externalRedirect($magic_path); diff --git a/src/Module/Admin/Summary.php b/src/Module/Admin/Summary.php index 6dcef2ea43..ad84bb6ab6 100644 --- a/src/Module/Admin/Summary.php +++ b/src/Module/Admin/Summary.php @@ -32,7 +32,6 @@ use Friendica\DI; use Friendica\Model\Register; use Friendica\Module\BaseAdmin; use Friendica\Network\HTTPException\InternalServerErrorException; -use Friendica\Network\HTTPRequest; use Friendica\Util\ConfigFileLoader; use Friendica\Util\DateTimeFormat; @@ -247,7 +246,7 @@ class Summary extends BaseAdmin private static function checkSelfHostMeta() { // Fetch the host-meta to check if this really is a vital server - return HTTPRequest::curl(DI::baseUrl()->get() . '/.well-known/host-meta')->isSuccess(); + return DI::httpRequest()->curl(DI::baseUrl()->get() . '/.well-known/host-meta')->isSuccess(); } } diff --git a/src/Module/Magic.php b/src/Module/Magic.php index b65159585b..a025617d8d 100644 --- a/src/Module/Magic.php +++ b/src/Module/Magic.php @@ -27,7 +27,6 @@ use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; -use Friendica\Network\HTTPRequest; use Friendica\Util\HTTPSignature; use Friendica\Util\Strings; @@ -101,7 +100,7 @@ class Magic extends BaseModule ); // Try to get an authentication token from the other instance. - $curlResult = HTTPRequest::curl($basepath . '/owa', false, ['headers' => $headers]); + $curlResult = DI::httpRequest()->curl($basepath . '/owa', false, ['headers' => $headers]); if ($curlResult->isSuccess()) { $j = json_decode($curlResult->getBody(), true); diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 7d7d59a6de..73e5cd8e0b 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -71,12 +71,12 @@ class HTTPRequest * @return CurlResult * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function curl(string $url, bool $binary = false, array $opts = [], int &$redirects = 0) + public function curl(string $url, bool $binary = false, array $opts = [], int &$redirects = 0) { $stamp1 = microtime(true); if (strlen($url) > 1000) { - Logger::log('URL is longer than 1000 characters. Callstack: ' . System::callstack(20), Logger::DEBUG); + $this->logger->debug('URL is longer than 1000 characters.', ['url' => $url, 'callstack' => System::callstack(20)]); return CurlResult::createErrorCurl(substr($url, 0, 200)); } @@ -94,7 +94,7 @@ class HTTPRequest $url = Network::unparseURL($parts); if (Network::isUrlBlocked($url)) { - Logger::log('domain of ' . $url . ' is blocked', Logger::DATA); + $this->logger->info('Domain is blocked.', ['url' => $url]); return CurlResult::createErrorCurl($url); } @@ -128,9 +128,9 @@ class HTTPRequest } @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - @curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); + @curl_setopt($ch, CURLOPT_USERAGENT, $this->userAgent); - $range = intval(DI::config()->get('system', 'curl_range_bytes', 0)); + $range = intval($this->config->get('system', 'curl_range_bytes', 0)); if ($range > 0) { @curl_setopt($ch, CURLOPT_RANGE, '0-' . $range); @@ -152,33 +152,33 @@ class HTTPRequest if (!empty($opts['timeout'])) { @curl_setopt($ch, CURLOPT_TIMEOUT, $opts['timeout']); } else { - $curl_time = DI::config()->get('system', 'curl_timeout', 60); + $curl_time = $this->config->get('system', 'curl_timeout', 60); @curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time)); } // by default we will allow self-signed certs // but you can override this - $check_cert = DI::config()->get('system', 'verifyssl'); + $check_cert = $this->config->get('system', 'verifyssl'); @curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false)); if ($check_cert) { @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); } - $proxy = DI::config()->get('system', 'proxy'); + $proxy = $this->config->get('system', 'proxy'); - if (strlen($proxy)) { + if (!empty($proxy)) { @curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1); @curl_setopt($ch, CURLOPT_PROXY, $proxy); - $proxyuser = @DI::config()->get('system', 'proxyuser'); + $proxyuser = $this->config->get('system', 'proxyuser'); - if (strlen($proxyuser)) { + if (!empty($proxyuser)) { @curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser); } } - if (DI::config()->get('system', 'ipv4_resolve', false)) { + if ($this->config->get('system', 'ipv4_resolve', false)) { curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); } @@ -204,14 +204,14 @@ class HTTPRequest if ($curlResponse->isRedirectUrl()) { $redirects++; - Logger::log('curl: redirect ' . $url . ' to ' . $curlResponse->getRedirectUrl()); + $this->logger->notice('Curl redirect.', ['url' => $url, 'to' => $curlResponse->getRedirectUrl()]); @curl_close($ch); return self::curl($curlResponse->getRedirectUrl(), $binary, $opts, $redirects); } @curl_close($ch); - DI::profiler()->saveTimestamp($stamp1, 'network', System::callstack()); + $this->profiler->saveTimestamp($stamp1, 'network', System::callstack()); return $curlResponse; } diff --git a/src/Network/Probe.php b/src/Network/Probe.php index dadd794fe6..01dc284081 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -166,7 +166,7 @@ class Probe Logger::info('Probing', ['host' => $host, 'ssl_url' => $ssl_url, 'url' => $url, 'callstack' => System::callstack(20)]); $xrd = null; - $curlResult = HTTPRequest::curl($ssl_url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); + $curlResult = DI::httpRequest()->curl($ssl_url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); $ssl_connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0); if ($curlResult->isSuccess()) { $xml = $curlResult->getBody(); @@ -183,7 +183,7 @@ class Probe } if (!is_object($xrd) && !empty($url)) { - $curlResult = HTTPRequest::curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); + $curlResult = DI::httpRequest()->curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); $connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0); if ($curlResult->isTimeout()) { Logger::info('Probing timeout', ['url' => $url]); @@ -427,7 +427,7 @@ class Probe */ private static function getHideStatus($url) { - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if (!$curlResult->isSuccess()) { return false; } @@ -841,7 +841,7 @@ class Probe public static function pollZot($url, $data) { - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if ($curlResult->isTimeout()) { return $data; } @@ -938,7 +938,7 @@ class Probe { $xrd_timeout = DI::config()->get('system', 'xrd_timeout', 20); - $curlResult = HTTPRequest::curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => $type]); + $curlResult = DI::httpRequest()->curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => $type]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1007,7 +1007,7 @@ class Probe */ private static function pollNoscrape($noscrape_url, $data) { - $curlResult = HTTPRequest::curl($noscrape_url); + $curlResult = DI::httpRequest()->curl($noscrape_url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1265,7 +1265,7 @@ class Probe */ private static function pollHcard($hcard_url, $data, $dfrn = false) { - $curlResult = HTTPRequest::curl($hcard_url); + $curlResult = DI::httpRequest()->curl($hcard_url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1519,7 +1519,7 @@ class Probe $pubkey = substr($pubkey, 5); } } elseif (Strings::normaliseLink($pubkey) == 'http://') { - $curlResult = HTTPRequest::curl($pubkey); + $curlResult = DI::httpRequest()->curl($pubkey); if ($curlResult->isTimeout()) { self::$istimeout = true; return $short ? false : []; @@ -1552,7 +1552,7 @@ class Probe } // Fetch all additional data from the feed - $curlResult = HTTPRequest::curl($data["poll"]); + $curlResult = DI::httpRequest()->curl($data["poll"]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1604,7 +1604,7 @@ class Probe */ private static function pumpioProfileData($profile_link) { - $curlResult = HTTPRequest::curl($profile_link); + $curlResult = DI::httpRequest()->curl($profile_link); if (!$curlResult->isSuccess()) { return []; } @@ -1835,7 +1835,7 @@ class Probe */ private static function feed($url, $probe = true) { - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index 24c6250a41..f417080114 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -24,7 +24,6 @@ namespace Friendica\Protocol; use Friendica\Core\Protocol; use Friendica\Model\APContact; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; use Friendica\Util\HTTPSignature; use Friendica\Util\JsonLD; @@ -93,7 +92,7 @@ class ActivityPub return HTTPSignature::fetch($url, $uid); } - $curlResult = HTTPRequest::curl($url, false, ['accept_content' => 'application/activity+json, application/ld+json']); + $curlResult = DI::httpRequest()->curl($url, false, ['accept_content' => 'application/activity+json, application/ld+json']); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { return false; } diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 21adc58cc0..ef7eb0f481 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -1195,7 +1195,7 @@ class DFRN Logger::log('dfrn_deliver: ' . $url); - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if ($curlResult->isTimeout()) { return -2; // timed out diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index 779c99358e..a9ee2277e3 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -39,7 +39,6 @@ use Friendica\Model\Item; use Friendica\Model\ItemURI; use Friendica\Model\Tag; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Util\DateTimeFormat; use Friendica\Util\Images; @@ -756,7 +755,7 @@ class OStatus self::$conv_list[$conversation] = true; - $curlResult = HTTPRequest::curl($conversation, false, ['accept_content' => 'application/atom+xml, text/html']); + $curlResult = DI::httpRequest()->curl($conversation, false, ['accept_content' => 'application/atom+xml, text/html']); if (!$curlResult->isSuccess()) { return; @@ -785,7 +784,7 @@ class OStatus } } if ($file != '') { - $conversation_atom = HTTPRequest::curl($attribute['href']); + $conversation_atom = DI::httpRequest()->curl($attribute['href']); if ($conversation_atom->isSuccess()) { $xml = $conversation_atom->getBody(); @@ -902,7 +901,7 @@ class OStatus return; } - $curlResult = HTTPRequest::curl($self); + $curlResult = DI::httpRequest()->curl($self); if (!$curlResult->isSuccess()) { return; @@ -949,7 +948,7 @@ class OStatus } $stored = false; - $curlResult = HTTPRequest::curl($related, false, ['accept_content' => 'application/atom+xml, text/html']); + $curlResult = DI::httpRequest()->curl($related, false, ['accept_content' => 'application/atom+xml, text/html']); if (!$curlResult->isSuccess()) { return; @@ -980,7 +979,7 @@ class OStatus } } if ($atom_file != '') { - $curlResult = HTTPRequest::curl($atom_file); + $curlResult = DI::httpRequest()->curl($atom_file); if ($curlResult->isSuccess()) { Logger::log('Fetched XML for URI ' . $related_uri, Logger::DEBUG); @@ -992,7 +991,7 @@ class OStatus // Workaround for older GNU Social servers if (($xml == '') && strstr($related, '/notice/')) { - $curlResult = HTTPRequest::curl(str_replace('/notice/', '/api/statuses/show/', $related) . '.atom'); + $curlResult = DI::httpRequest()->curl(str_replace('/notice/', '/api/statuses/show/', $related) . '.atom'); if ($curlResult->isSuccess()) { Logger::log('GNU Social workaround to fetch XML for URI ' . $related_uri, Logger::DEBUG); @@ -1003,7 +1002,7 @@ class OStatus // Even more worse workaround for GNU Social ;-) if ($xml == '') { $related_guess = self::convertHref($related_uri); - $curlResult = HTTPRequest::curl(str_replace('/notice/', '/api/statuses/show/', $related_guess) . '.atom'); + $curlResult = DI::httpRequest()->curl(str_replace('/notice/', '/api/statuses/show/', $related_guess) . '.atom'); if ($curlResult->isSuccess()) { Logger::log('GNU Social workaround 2 to fetch XML for URI ' . $related_uri, Logger::DEBUG); diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index f2ad51070d..0284c9b8aa 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -251,7 +251,7 @@ class PortableContact */ private static function fetchServerlist($poco) { - $curlResult = HTTPRequest::curl($poco . "/@server"); + $curlResult = DI::httpRequest()->curl($poco . "/@server"); if (!$curlResult->isSuccess()) { return; @@ -291,7 +291,7 @@ class PortableContact Logger::info("Fetch all users from the server " . $server["url"]); - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { $data = json_decode($curlResult->getBody(), true); @@ -314,7 +314,7 @@ class PortableContact $success = false; - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { Logger::info("Fetch all global contacts from the server " . $server["nurl"]); @@ -372,7 +372,7 @@ class PortableContact // Fetch all contacts from a given user from the other server $url = $server['poco'] . '/' . $username . '/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,contactType,generation'; - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if ($curlResult->isSuccess()) { $data = json_decode($curlResult->getBody(), true); diff --git a/src/Util/ExAuth.php b/src/Util/ExAuth.php index 710c009796..5933860827 100644 --- a/src/Util/ExAuth.php +++ b/src/Util/ExAuth.php @@ -37,7 +37,6 @@ namespace Friendica\Util; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; class ExAuth { @@ -182,7 +181,7 @@ class ExAuth $url = ($ssl ? 'https' : 'http') . '://' . $host . '/noscrape/' . $user; - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if (!$curlResult->isSuccess()) { return false; diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index 5d57c2281f..8c1d4f986e 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -443,7 +443,7 @@ class HTTPSignature $curl_opts = $opts; $curl_opts['header'] = $headers; - $curlResult = HTTPRequest::curl($request, false, $curl_opts); + $curlResult = DI::httpRequest()->curl($request, false, $curl_opts); $return_code = $curlResult->getReturnCode(); Logger::log('Fetched for user ' . $uid . ' from ' . $request . ' returned ' . $return_code, Logger::DEBUG); diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index 577ffd4c11..cf38ffd7be 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -27,7 +27,6 @@ use Friendica\Content\OEmbed; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Database\DBA; -use Friendica\Network\HTTPRequest; /** * Get information about a given URL @@ -160,7 +159,7 @@ class ParseUrl return $siteinfo; } - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if (!$curlResult->isSuccess()) { return $siteinfo; } diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index 13d1b8ee7a..0dc67d80f5 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -291,7 +291,7 @@ class OnePoll . '&type=data&last_update=' . $last_update . '&perm=' . $perm; - $curlResult = HTTPRequest::curl($url); + $curlResult = DI::httpRequest()->curl($url); if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) { // set the last-update so we don't keep polling @@ -444,7 +444,7 @@ class OnePoll } $cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-'); - $curlResult = HTTPRequest::curl($contact['poll'], false, ['cookiejar' => $cookiejar]); + $curlResult = DI::httpRequest()->curl($contact['poll'], false, ['cookiejar' => $cookiejar]); unlink($cookiejar); if ($curlResult->isTimeout()) { From 8793096c16ea97cf73fb4f04199e4aab8574dcd1 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:18:28 +0100 Subject: [PATCH 04/17] Make "HTTPRequest::post" dynamic --- mod/dfrn_confirm.php | 3 +-- mod/dfrn_poll.php | 2 +- mod/match.php | 3 +-- src/Network/HTTPRequest.php | 26 ++++++++++++-------------- src/Protocol/DFRN.php | 5 ++--- src/Protocol/Diaspora.php | 2 +- src/Protocol/Salmon.php | 6 +++--- src/Util/HTTPSignature.php | 3 +-- src/Worker/OnePoll.php | 5 ++--- src/Worker/PubSubPublish.php | 3 +-- 10 files changed, 25 insertions(+), 33 deletions(-) diff --git a/mod/dfrn_confirm.php b/mod/dfrn_confirm.php index f78e8d55aa..f7460b79f3 100644 --- a/mod/dfrn_confirm.php +++ b/mod/dfrn_confirm.php @@ -42,7 +42,6 @@ use Friendica\Model\Contact; use Friendica\Model\Group; use Friendica\Model\Notify\Type; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; use Friendica\Protocol\Activity; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; @@ -224,7 +223,7 @@ function dfrn_confirm_post(App $a, $handsfree = null) * */ - $res = HTTPRequest::post($dfrn_confirm, $params, [], 120)->getBody(); + $res = DI::httpRequest()->post($dfrn_confirm, $params, [], 120)->getBody(); Logger::log(' Confirm: received data: ' . $res, Logger::DATA); diff --git a/mod/dfrn_poll.php b/mod/dfrn_poll.php index faa55a1082..00732982ed 100644 --- a/mod/dfrn_poll.php +++ b/mod/dfrn_poll.php @@ -507,7 +507,7 @@ function dfrn_poll_content(App $a) . '&sec=' . $sec ); } else { - $s = HTTPRequest::post($r[0]['poll'], [ + $s = DI::httpRequest()->post($r[0]['poll'], [ 'dfrn_id' => $encrypted_id, 'type' => 'profile-check', 'dfrn_version' => DFRN_PROTOCOL_VERSION, diff --git a/mod/match.php b/mod/match.php index 4ec47c4cc6..064107764a 100644 --- a/mod/match.php +++ b/mod/match.php @@ -27,7 +27,6 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Profile; -use Friendica\Network\HTTPRequest; use Friendica\Util\Proxy as ProxyUtils; /** @@ -76,7 +75,7 @@ function match_content(App $a) $host = DI::baseUrl(); } - $msearch_json = HTTPRequest::post($host . '/msearch', $params)->getBody(); + $msearch_json = DI::httpRequest()->post($host . '/msearch', $params)->getBody(); $msearch = json_decode($msearch_json); diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 73e5cd8e0b..131fac8a78 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -25,7 +25,6 @@ use Friendica\App; use Friendica\Core\Config\IConfig; use Friendica\Core\Logger; use Friendica\Core\System; -use Friendica\DI; use Friendica\Util\Network; use Friendica\Util\Profiler; use Psr\Log\LoggerInterface; @@ -228,23 +227,22 @@ class HTTPRequest * @return CurlResult The content * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function post(string $url, $params, array $headers = [], int $timeout = 0, int &$redirects = 0) + public function post(string $url, $params, array $headers = [], int $timeout = 0, int &$redirects = 0) { $stamp1 = microtime(true); if (Network::isUrlBlocked($url)) { - Logger::log('post_url: domain of ' . $url . ' is blocked', Logger::DATA); + $this->logger->info('Domain is blocked.'. ['url' => $url]); return CurlResult::createErrorCurl($url); } - $a = DI::app(); $ch = curl_init($url); if (($redirects > 8) || (!$ch)) { return CurlResult::createErrorCurl($url); } - Logger::log('post_url: start ' . $url, Logger::DATA); + $this->logger->debug('Post_url: start.', ['url' => $url]); curl_setopt($ch, CURLOPT_HEADER, true); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); @@ -252,14 +250,14 @@ class HTTPRequest curl_setopt($ch, CURLOPT_POSTFIELDS, $params); curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); - if (DI::config()->get('system', 'ipv4_resolve', false)) { + if ($this->config->get('system', 'ipv4_resolve', false)) { curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); } if (intval($timeout)) { curl_setopt($ch, CURLOPT_TIMEOUT, $timeout); } else { - $curl_time = DI::config()->get('system', 'curl_timeout', 60); + $curl_time = $this->config->get('system', 'curl_timeout', 60); curl_setopt($ch, CURLOPT_TIMEOUT, intval($curl_time)); } @@ -267,20 +265,20 @@ class HTTPRequest curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); } - $check_cert = DI::config()->get('system', 'verifyssl'); + $check_cert = $this->config->get('system', 'verifyssl'); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false)); if ($check_cert) { @curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 2); } - $proxy = DI::config()->get('system', 'proxy'); + $proxy = $this->config->get('system', 'proxy'); - if (strlen($proxy)) { + if (!empty($proxy)) { curl_setopt($ch, CURLOPT_HTTPPROXYTUNNEL, 1); curl_setopt($ch, CURLOPT_PROXY, $proxy); - $proxyuser = DI::config()->get('system', 'proxyuser'); - if (strlen($proxyuser)) { + $proxyuser = $this->config->get('system', 'proxyuser'); + if (!empty($proxyuser)) { curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyuser); } } @@ -296,14 +294,14 @@ class HTTPRequest if ($curlResponse->isRedirectUrl()) { $redirects++; - Logger::log('post_url: redirect ' . $url . ' to ' . $curlResponse->getRedirectUrl()); + $this->logger->info('Post redirect.', ['url' => $url, 'to' => $curlResponse->getRedirectUrl()]); curl_close($ch); return self::post($curlResponse->getRedirectUrl(), $params, $headers, $redirects, $timeout); } curl_close($ch); - DI::profiler()->saveTimestamp($stamp1, 'network', System::callstack()); + $this->profiler->saveTimestamp($stamp1, 'network', System::callstack()); // Very old versions of Lighttpd don't like the "Expect" header, so we remove it when needed if ($curlResponse->getReturnCode() == 417) { diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index ef7eb0f481..7c87ee0b25 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -43,7 +43,6 @@ use Friendica\Model\Post\Category; use Friendica\Model\Profile; use Friendica\Model\Tag; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; @@ -1344,7 +1343,7 @@ class DFRN Logger::debug('dfrn_deliver', ['post' => $postvars]); - $postResult = HTTPRequest::post($contact['notify'], $postvars); + $postResult = DI::httpRequest()->post($contact['notify'], $postvars); $xml = $postResult->getBody(); @@ -1441,7 +1440,7 @@ class DFRN $content_type = ($public_batch ? "application/magic-envelope+xml" : "application/json"); - $postResult = HTTPRequest::post($dest_url, $envelope, ["Content-Type: " . $content_type]); + $postResult = DI::httpRequest()->post($dest_url, $envelope, ["Content-Type: " . $content_type]); $xml = $postResult->getBody(); $curl_stat = $postResult->getReturnCode(); diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 46d2bc1d4a..ed369a3047 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -3261,7 +3261,7 @@ class Diaspora if (!intval(DI::config()->get("system", "diaspora_test"))) { $content_type = (($public_batch) ? "application/magic-envelope+xml" : "application/json"); - $postResult = HTTPRequest::post($dest_url . "/", $envelope, ["Content-Type: " . $content_type]); + $postResult = DI::httpRequest()->post($dest_url . "/", $envelope, ["Content-Type: " . $content_type]); $return_code = $postResult->getReturnCode(); } else { Logger::log("test_mode"); diff --git a/src/Protocol/Salmon.php b/src/Protocol/Salmon.php index d01ea2ce12..35707b6356 100644 --- a/src/Protocol/Salmon.php +++ b/src/Protocol/Salmon.php @@ -155,7 +155,7 @@ class Salmon $salmon = XML::fromArray($xmldata, $xml, false, $namespaces); // slap them - $postResult = HTTPRequest::post($url, $salmon, [ + $postResult = DI::httpRequest()->post($url, $salmon, [ 'Content-type: application/magic-envelope+xml', 'Content-length: ' . strlen($salmon) ]); @@ -180,7 +180,7 @@ class Salmon $salmon = XML::fromArray($xmldata, $xml, false, $namespaces); // slap them - $postResult = HTTPRequest::post($url, $salmon, [ + $postResult = DI::httpRequest()->post($url, $salmon, [ 'Content-type: application/magic-envelope+xml', 'Content-length: ' . strlen($salmon) ]); @@ -203,7 +203,7 @@ class Salmon $salmon = XML::fromArray($xmldata, $xml, false, $namespaces); // slap them - $postResult = HTTPRequest::post($url, $salmon, [ + $postResult = DI::httpRequest()->post($url, $salmon, [ 'Content-type: application/magic-envelope+xml', 'Content-length: ' . strlen($salmon)]); $return_code = $postResult->getReturnCode(); diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index 8c1d4f986e..84a11b8ec0 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -26,7 +26,6 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\APContact; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; /** * Implements HTTP Signatures per draft-cavage-http-signatures-07. @@ -298,7 +297,7 @@ class HTTPSignature $headers[] = 'Content-Type: application/activity+json'; - $postResult = HTTPRequest::post($target, $content, $headers); + $postResult = DI::httpRequest()->post($target, $content, $headers); $return_code = $postResult->getReturnCode(); Logger::log('Transmit to ' . $target . ' returned ' . $return_code, Logger::DEBUG); diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index 0dc67d80f5..ed6bfacb35 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -28,7 +28,6 @@ use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\Item; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; use Friendica\Protocol\Activity; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\Email; @@ -405,7 +404,7 @@ class OnePoll $postvars['dfrn_version'] = DFRN_PROTOCOL_VERSION; $postvars['perm'] = 'rw'; - return HTTPRequest::post($contact['poll'], $postvars)->getBody(); + return DI::httpRequest()->post($contact['poll'], $postvars)->getBody(); } /** @@ -756,7 +755,7 @@ class OnePoll DBA::update('contact', ['hub-verify' => $verify_token], ['id' => $contact['id']]); } - $postResult = HTTPRequest::post($url, $params); + $postResult = DI::httpRequest()->post($url, $params); Logger::log('subscribe_to_hub: returns: ' . $postResult->getReturnCode(), Logger::DEBUG); diff --git a/src/Worker/PubSubPublish.php b/src/Worker/PubSubPublish.php index 4fa3b3e894..eab68b4304 100644 --- a/src/Worker/PubSubPublish.php +++ b/src/Worker/PubSubPublish.php @@ -25,7 +25,6 @@ use Friendica\Core\Logger; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\PushSubscriber; -use Friendica\Network\HTTPRequest; use Friendica\Protocol\OStatus; class PubSubPublish @@ -68,7 +67,7 @@ class PubSubPublish Logger::log('POST ' . print_r($headers, true) . "\n" . $params, Logger::DATA); - $postResult = HTTPRequest::post($subscriber['callback_url'], $params, $headers); + $postResult = DI::httpRequest()->post($subscriber['callback_url'], $params, $headers); $ret = $postResult->getReturnCode(); if ($ret >= 200 && $ret <= 299) { From 3b4cf87c95a6a2b5ca46b2fbcb65c4e9e0e94dbc Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:19:59 +0100 Subject: [PATCH 05/17] Make "HTTPRequest::fetchUrlFull" dynamic --- mod/pubsubhubbub.php | 2 +- src/Core/Installer.php | 4 ++-- src/Network/HTTPRequest.php | 4 ++-- src/Protocol/PortableContact.php | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mod/pubsubhubbub.php b/mod/pubsubhubbub.php index 9403d3eb7f..96f26838ed 100644 --- a/mod/pubsubhubbub.php +++ b/mod/pubsubhubbub.php @@ -126,7 +126,7 @@ function pubsubhubbub_init(App $a) { $hub_callback = rtrim($hub_callback, ' ?&#'); $separator = parse_url($hub_callback, PHP_URL_QUERY) === null ? '?' : '&'; - $fetchResult = HTTPRequest::fetchUrlFull($hub_callback . $separator . $params); + $fetchResult = DI::httpRequest()->fetchUrlFull($hub_callback . $separator . $params); $body = $fetchResult->getBody(); $ret = $fetchResult->getReturnCode(); diff --git a/src/Core/Installer.php b/src/Core/Installer.php index 7b6291ff39..af7c7aa496 100644 --- a/src/Core/Installer.php +++ b/src/Core/Installer.php @@ -548,11 +548,11 @@ class Installer $help = ""; $error_msg = ""; if (function_exists('curl_init')) { - $fetchResult = HTTPRequest::fetchUrlFull($baseurl . "/install/testrewrite"); + $fetchResult = DI::httpRequest()->fetchUrlFull($baseurl . "/install/testrewrite"); $url = Strings::normaliseLink($baseurl . "/install/testrewrite"); if ($fetchResult->getReturnCode() != 204) { - $fetchResult = HTTPRequest::fetchUrlFull($url); + $fetchResult = DI::httpRequest()->fetchUrlFull($url); } if ($fetchResult->getReturnCode() != 204) { diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 131fac8a78..14395c37f3 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -365,9 +365,9 @@ class HTTPRequest * @return CurlResult With all relevant information, 'body' contains the actual fetched content. * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function fetchUrlFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) + public function fetchUrlFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) { - return self::curl( + return $this->curl( $url, $binary, [ diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 0284c9b8aa..f118ffa949 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -103,7 +103,7 @@ class PortableContact Logger::log('load: ' . $url, Logger::DEBUG); - $fetchresult = HTTPRequest::fetchUrlFull($url); + $fetchresult = DI::httpRequest()->fetchUrlFull($url); $s = $fetchresult->getBody(); Logger::log('load: returns ' . $s, Logger::DATA); From 1aa07f87a44b50e93f25006c4007450834abda6f Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:20:28 +0100 Subject: [PATCH 06/17] Make "HTTPRequest::fetchUrl" dynamic --- mod/dfrn_poll.php | 5 ++--- mod/dfrn_request.php | 3 +-- mod/oexchange.php | 3 +-- src/Content/OEmbed.php | 7 +++---- src/Content/Text/BBCode.php | 7 +++---- src/Core/Protocol.php | 4 +--- src/Core/Search.php | 3 +-- src/Core/Worker.php | 3 +-- src/Model/GContact.php | 3 +-- src/Model/GServer.php | 3 +-- src/Model/User.php | 3 +-- src/Module/Debug/Feed.php | 3 +-- src/Network/HTTPRequest.php | 4 ++-- src/Protocol/Diaspora.php | 3 +-- src/Protocol/Salmon.php | 3 +-- src/Util/Images.php | 3 +-- src/Worker/CheckVersion.php | 3 +-- src/Worker/CronJobs.php | 3 +-- src/Worker/Directory.php | 3 +-- src/Worker/SearchDirectory.php | 3 +-- 20 files changed, 26 insertions(+), 46 deletions(-) diff --git a/mod/dfrn_poll.php b/mod/dfrn_poll.php index 00732982ed..7f7fbe498f 100644 --- a/mod/dfrn_poll.php +++ b/mod/dfrn_poll.php @@ -25,7 +25,6 @@ use Friendica\Core\Session; use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Network\HTTPRequest; use Friendica\Protocol\DFRN; use Friendica\Protocol\OStatus; use Friendica\Util\Strings; @@ -115,7 +114,7 @@ function dfrn_poll_init(App $a) ); if (DBA::isResult($r)) { - $s = HTTPRequest::fetchUrl($r[0]['poll'] . '?dfrn_id=' . $my_id . '&type=profile-check'); + $s = DI::httpRequest()->fetchUrl($r[0]['poll'] . '?dfrn_id=' . $my_id . '&type=profile-check'); Logger::log("dfrn_poll: old profile returns " . $s, Logger::DATA); @@ -499,7 +498,7 @@ function dfrn_poll_content(App $a) // URL reply if ($dfrn_version < 2.2) { - $s = HTTPRequest::fetchUrl($r[0]['poll'] + $s = DI::httpRequest()->fetchUrl($r[0]['poll'] . '?dfrn_id=' . $encrypted_id . '&type=profile-check' . '&dfrn_version=' . DFRN_PROTOCOL_VERSION diff --git a/mod/dfrn_request.php b/mod/dfrn_request.php index bdc407b0b1..8c8557650e 100644 --- a/mod/dfrn_request.php +++ b/mod/dfrn_request.php @@ -39,7 +39,6 @@ use Friendica\Model\Notify\Type; use Friendica\Model\Profile; use Friendica\Model\User; use Friendica\Module\Security\Login; -use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Protocol\Activity; use Friendica\Util\DateTimeFormat; @@ -204,7 +203,7 @@ function dfrn_request_post(App $a) } if (!empty($dfrn_request) && strlen($confirm_key)) { - HTTPRequest::fetchUrl($dfrn_request . '?confirm_key=' . $confirm_key); + DI::httpRequest()->fetchUrl($dfrn_request . '?confirm_key=' . $confirm_key); } // (ignore reply, nothing we can do it failed) diff --git a/mod/oexchange.php b/mod/oexchange.php index 5238893320..4746651c16 100644 --- a/mod/oexchange.php +++ b/mod/oexchange.php @@ -23,7 +23,6 @@ use Friendica\App; use Friendica\Core\Renderer; use Friendica\DI; use Friendica\Module\Security\Login; -use Friendica\Network\HTTPRequest; use Friendica\Util\Strings; function oexchange_init(App $a) { @@ -58,7 +57,7 @@ function oexchange_content(App $a) { $tags = ((!empty($_REQUEST['tags'])) ? '&tags=' . urlencode(Strings::escapeTags(trim($_REQUEST['tags']))) : ''); - $s = HTTPRequest::fetchUrl(DI::baseUrl() . '/parse_url?url=' . $url . $title . $description . $tags); + $s = DI::httpRequest()->fetchUrl(DI::baseUrl() . '/parse_url?url=' . $url . $title . $description . $tags); if (!strlen($s)) { return; diff --git a/src/Content/OEmbed.php b/src/Content/OEmbed.php index 8cfb8ce0a1..592d25c10d 100644 --- a/src/Content/OEmbed.php +++ b/src/Content/OEmbed.php @@ -31,7 +31,6 @@ use Friendica\Core\Hook; use Friendica\Core\Renderer; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Network\HTTPRequest; use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; use Friendica\Util\ParseUrl; @@ -96,7 +95,7 @@ class OEmbed if (!in_array($ext, $noexts)) { // try oembed autodiscovery - $html_text = HTTPRequest::fetchUrl($embedurl, false, 15, 'text/*'); + $html_text = DI::httpRequest()->fetchUrl($embedurl, false, 15, 'text/*'); if ($html_text) { $dom = @DOMDocument::loadHTML($html_text); if ($dom) { @@ -104,14 +103,14 @@ class OEmbed $entries = $xpath->query("//link[@type='application/json+oembed']"); foreach ($entries as $e) { $href = $e->getAttributeNode('href')->nodeValue; - $json_string = HTTPRequest::fetchUrl($href . '&maxwidth=' . $a->videowidth); + $json_string = DI::httpRequest()->fetchUrl($href . '&maxwidth=' . $a->videowidth); break; } $entries = $xpath->query("//link[@type='text/json+oembed']"); foreach ($entries as $e) { $href = $e->getAttributeNode('href')->nodeValue; - $json_string = HTTPRequest::fetchUrl($href . '&maxwidth=' . $a->videowidth); + $json_string = DI::httpRequest()->fetchUrl($href . '&maxwidth=' . $a->videowidth); break; } } diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index f03ea6104b..9d9679c494 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -38,7 +38,6 @@ use Friendica\Model\Contact; use Friendica\Model\Event; use Friendica\Model\Photo; use Friendica\Model\Tag; -use Friendica\Network\HTTPRequest; use Friendica\Object\Image; use Friendica\Protocol\Activity; use Friendica\Util\Images; @@ -486,7 +485,7 @@ class BBCode continue; } - $curlResult = HTTPRequest::curl($mtch[1], true); + $curlResult = DI::httpRequest()->curl($mtch[1], true); if (!$curlResult->isSuccess()) { continue; } @@ -1107,7 +1106,7 @@ class BBCode $text = "[url=" . $match[2] . ']' . $match[2] . "[/url]"; // if its not a picture then look if its a page that contains a picture link - $body = HTTPRequest::fetchUrl($match[1]); + $body = DI::httpRequest()->fetchUrl($match[1]); $doc = new DOMDocument(); @$doc->loadHTML($body); @@ -1186,7 +1185,7 @@ class BBCode } // if its not a picture then look if its a page that contains a picture link - $body = HTTPRequest::fetchUrl($match[1]); + $body = DI::httpRequest()->fetchUrl($match[1]); $doc = new DOMDocument(); @$doc->loadHTML($body); diff --git a/src/Core/Protocol.php b/src/Core/Protocol.php index 84b589bf29..e6133240c4 100644 --- a/src/Core/Protocol.php +++ b/src/Core/Protocol.php @@ -21,8 +21,6 @@ namespace Friendica\Core; -use Friendica\Network\HTTPRequest; - /** * Manage compatibility with federated networks */ @@ -123,7 +121,7 @@ class Protocol if (preg_match('=https?://(.*)/user/(.*)=ism', $profile_url, $matches)) { $statusnet_host = $matches[1]; $statusnet_user = $matches[2]; - $UserData = HTTPRequest::fetchUrl('http://' . $statusnet_host . '/api/users/show.json?user_id=' . $statusnet_user); + $UserData = DI::httpRequest()->fetchUrl('http://' . $statusnet_host . '/api/users/show.json?user_id=' . $statusnet_user); $user = json_decode($UserData); if ($user) { $matches[2] = $user->screen_name; diff --git a/src/Core/Search.php b/src/Core/Search.php index aafa4024a8..768a0bf275 100644 --- a/src/Core/Search.php +++ b/src/Core/Search.php @@ -26,7 +26,6 @@ use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\GContact; use Friendica\Network\HTTPException; -use Friendica\Network\HTTPRequest; use Friendica\Object\Search\ContactResult; use Friendica\Object\Search\ResultList; use Friendica\Util\Network; @@ -124,7 +123,7 @@ class Search $searchUrl .= '&page=' . $page; } - $resultJson = HTTPRequest::fetchUrl($searchUrl, false, 0, 'application/json'); + $resultJson = DI::httpRequest()->fetchUrl($searchUrl, false, 0, 'application/json'); $results = json_decode($resultJson, true); diff --git a/src/Core/Worker.php b/src/Core/Worker.php index a5c4226c47..83a24c38fe 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -25,7 +25,6 @@ use Friendica\Core; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Process; -use Friendica\Network\HTTPRequest; use Friendica\Util\DateTimeFormat; /** @@ -997,7 +996,7 @@ class Worker } $url = DI::baseUrl() . '/worker'; - HTTPRequest::fetchUrl($url, false, 1); + DI::httpRequest()->fetchUrl($url, false, 1); } /** diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 669bc60c5d..dabc907f89 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -31,7 +31,6 @@ use Friendica\Core\System; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\PortableContact; @@ -537,7 +536,7 @@ class GContact $done[] = DI::baseUrl() . '/poco'; if (strlen(DI::config()->get('system', 'directory'))) { - $x = HTTPRequest::fetchUrl(Search::getGlobalDirectory() . '/pubsites'); + $x = DI::httpRequest()->fetchUrl(Search::getGlobalDirectory() . '/pubsites'); if (!empty($x)) { $j = json_decode($x); if (!empty($j->entries)) { diff --git a/src/Model/GServer.php b/src/Model/GServer.php index c868e19d40..76082fe1ff 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -31,7 +31,6 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Module\Register; use Friendica\Network\CurlResult; -use Friendica\Network\HTTPRequest; use Friendica\Protocol\Diaspora; use Friendica\Protocol\PortableContact; use Friendica\Util\DateTimeFormat; @@ -1635,7 +1634,7 @@ class GServer $protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus']; foreach ($protocols as $protocol) { $query = '{nodes(protocol:"' . $protocol . '"){host}}'; - $curlResult = HTTPRequest::fetchUrl('https://the-federation.info/graphql?query=' . urlencode($query)); + $curlResult = DI::httpRequest()->fetchUrl('https://the-federation.info/graphql?query=' . urlencode($query)); if (!empty($curlResult)) { $data = json_decode($curlResult, true); if (!empty($data['data']['nodes'])) { diff --git a/src/Model/User.php b/src/Model/User.php index fda105687d..9980c48d5a 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -34,7 +34,6 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\TwoFactor\AppSpecificPassword; use Friendica\Network\HTTPException\InternalServerErrorException; -use Friendica\Network\HTTPRequest; use Friendica\Object\Image; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; @@ -824,7 +823,7 @@ class User $photo_failure = false; $filename = basename($photo); - $curlResult = HTTPRequest::curl($photo, true); + $curlResult = DI::httpRequest()->curl($photo, true); if ($curlResult->isSuccess()) { $img_str = $curlResult->getBody(); $type = $curlResult->getContentType(); diff --git a/src/Module/Debug/Feed.php b/src/Module/Debug/Feed.php index 6214b49dd5..deeb8d7ec3 100644 --- a/src/Module/Debug/Feed.php +++ b/src/Module/Debug/Feed.php @@ -25,7 +25,6 @@ use Friendica\BaseModule; use Friendica\Core\Renderer; use Friendica\DI; use Friendica\Model; -use Friendica\Network\HTTPRequest; use Friendica\Protocol; /** @@ -49,7 +48,7 @@ class Feed extends BaseModule $contact = Model\Contact::getByURLForUser($url, local_user(), false); - $xml = HTTPRequest::fetchUrl($contact['poll']); + $xml = DI::httpRequest()->fetchUrl($contact['poll']); $import_result = Protocol\Feed::import($xml); diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 14395c37f3..eaef6966d7 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -341,9 +341,9 @@ class HTTPRequest * @return string The fetched content * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function fetchUrl(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) + public function fetchUrl(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) { - $ret = self::fetchUrlFull($url, $binary, $timeout, $accept_content, $cookiejar, $redirects); + $ret = $this->fetchUrlFull($url, $binary, $timeout, $accept_content, $cookiejar, $redirects); return $ret->getBody(); } diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index ed369a3047..5e1f096773 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -41,7 +41,6 @@ use Friendica\Model\Mail; use Friendica\Model\Post; use Friendica\Model\Tag; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Util\Crypto; use Friendica\Util\DateTimeFormat; @@ -1380,7 +1379,7 @@ class Diaspora Logger::log("Fetch post from ".$source_url, Logger::DEBUG); - $envelope = HTTPRequest::fetchUrl($source_url); + $envelope = DI::httpRequest()->fetchUrl($source_url); if ($envelope) { Logger::log("Envelope was fetched.", Logger::DEBUG); $x = self::verifyMagicEnvelope($envelope); diff --git a/src/Protocol/Salmon.php b/src/Protocol/Salmon.php index 35707b6356..921c060f88 100644 --- a/src/Protocol/Salmon.php +++ b/src/Protocol/Salmon.php @@ -22,7 +22,6 @@ namespace Friendica\Protocol; use Friendica\Core\Logger; -use Friendica\Network\HTTPRequest; use Friendica\Network\Probe; use Friendica\Util\Crypto; use Friendica\Util\Strings; @@ -72,7 +71,7 @@ class Salmon $ret[$x] = substr($ret[$x], 5); } } elseif (Strings::normaliseLink($ret[$x]) == 'http://') { - $ret[$x] = HTTPRequest::fetchUrl($ret[$x]); + $ret[$x] = DI::httpRequest()->fetchUrl($ret[$x]); } } } diff --git a/src/Util/Images.php b/src/Util/Images.php index 9e3be4f4f9..2d161a5c43 100644 --- a/src/Util/Images.php +++ b/src/Util/Images.php @@ -24,7 +24,6 @@ namespace Friendica\Util; use Friendica\Core\Logger; use Friendica\Core\System; use Friendica\DI; -use Friendica\Network\HTTPRequest; /** * Image utilities @@ -185,7 +184,7 @@ class Images return $data; } - $img_str = HTTPRequest::fetchUrl($url, true, 4); + $img_str = DI::httpRequest()->fetchUrl($url, true, 4); if (!$img_str) { return []; diff --git a/src/Worker/CheckVersion.php b/src/Worker/CheckVersion.php index f0369daab3..260d6b16f2 100644 --- a/src/Worker/CheckVersion.php +++ b/src/Worker/CheckVersion.php @@ -24,7 +24,6 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Network\HTTPRequest; /** * Check the git repository VERSION file and save the version to the DB @@ -55,7 +54,7 @@ class CheckVersion Logger::log("Checking VERSION from: ".$checked_url, Logger::DEBUG); // fetch the VERSION file - $gitversion = DBA::escape(trim(HTTPRequest::fetchUrl($checked_url))); + $gitversion = DBA::escape(trim(DI::httpRequest()->fetchUrl($checked_url))); Logger::log("Upstream VERSION is: ".$gitversion, Logger::DEBUG); DI::config()->set('system', 'git_friendica_version', $gitversion); diff --git a/src/Worker/CronJobs.php b/src/Worker/CronJobs.php index 23434beb14..1e45058569 100644 --- a/src/Worker/CronJobs.php +++ b/src/Worker/CronJobs.php @@ -33,7 +33,6 @@ use Friendica\Model\GContact; use Friendica\Model\Nodeinfo; use Friendica\Model\Photo; use Friendica\Model\User; -use Friendica\Network\HTTPRequest; use Friendica\Util\Proxy as ProxyUtils; use Friendica\Util\Strings; @@ -61,7 +60,7 @@ class CronJobs // Now trying to register $url = 'http://the-federation.info/register/' . DI::baseUrl()->getHostname(); Logger::debug('Check registering url', ['url' => $url]); - $ret = HTTPRequest::fetchUrl($url); + $ret = DI::httpRequest()->fetchUrl($url); Logger::debug('Check registering answer', ['answer' => $ret]); Logger::info('cron_end'); break; diff --git a/src/Worker/Directory.php b/src/Worker/Directory.php index 0dea9841f0..2cab09f339 100644 --- a/src/Worker/Directory.php +++ b/src/Worker/Directory.php @@ -26,7 +26,6 @@ use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; -use Friendica\Network\HTTPRequest; /** * Sends updated profile data to the directory @@ -54,7 +53,7 @@ class Directory Logger::log('Updating directory: ' . $arr['url'], Logger::DEBUG); if (strlen($arr['url'])) { - HTTPRequest::fetchUrl($dir . '?url=' . bin2hex($arr['url'])); + DI::httpRequest()->fetchUrl($dir . '?url=' . bin2hex($arr['url'])); } return; diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php index 1dcf0c8db6..c3c344d93c 100644 --- a/src/Worker/SearchDirectory.php +++ b/src/Worker/SearchDirectory.php @@ -30,7 +30,6 @@ use Friendica\DI; use Friendica\Model\Contact; use Friendica\Model\GContact; use Friendica\Model\GServer; -use Friendica\Network\HTTPRequest; use Friendica\Util\Strings; class SearchDirectory @@ -52,7 +51,7 @@ class SearchDirectory } } - $x = HTTPRequest::fetchUrl(Search::getGlobalDirectory() . '/lsearch?p=1&n=500&search=' . urlencode($search)); + $x = DI::httpRequest()->fetchUrl(Search::getGlobalDirectory() . '/lsearch?p=1&n=500&search=' . urlencode($search)); $j = json_decode($x); if (!empty($j->results)) { From cebdcb65991607c3a13ce691f60472e141e5d1af Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:20:50 +0100 Subject: [PATCH 07/17] Fixing post() --- src/Network/HTTPRequest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index eaef6966d7..ac2d70cad2 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -248,7 +248,7 @@ class HTTPRequest curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_POST, 1); curl_setopt($ch, CURLOPT_POSTFIELDS, $params); - curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); + curl_setopt($ch, CURLOPT_USERAGENT, $this->userAgent); if ($this->config->get('system', 'ipv4_resolve', false)) { curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); From 57587efe58e1ef9aed8c218f264696b10b331fea Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:26:38 +0100 Subject: [PATCH 08/17] Move "getUserAgent()" to "HTTPRequest" class --- src/App.php | 16 ---------------- src/Content/Text/BBCode.php | 4 ++-- src/Network/HTTPRequest.php | 29 ++++++++++++++++++++++------- src/Util/Network.php | 4 ++-- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/App.php b/src/App.php index 9b6f6a5a29..65ae3fe2f4 100644 --- a/src/App.php +++ b/src/App.php @@ -240,22 +240,6 @@ class App } } - /** - * Returns the current UserAgent as a String - * - * @return string the UserAgent as a String - * @throws HTTPException\InternalServerErrorException - */ - public function getUserAgent() - { - return - FRIENDICA_PLATFORM . " '" . - FRIENDICA_CODENAME . "' " . - FRIENDICA_VERSION . '-' . - DB_UPDATE_VERSION . '; ' . - $this->baseURL->get(); - } - /** * Returns the current theme name. May be overriden by the mobile theme name. * diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index 9d9679c494..036bc4a270 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1094,7 +1094,7 @@ class BBCode $ch = @curl_init($match[1]); @curl_setopt($ch, CURLOPT_NOBODY, true); @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - @curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); + @curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); @curl_exec($ch); $curl_info = @curl_getinfo($ch); @@ -1168,7 +1168,7 @@ class BBCode $ch = @curl_init($match[1]); @curl_setopt($ch, CURLOPT_NOBODY, true); @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - @curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); + @curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); @curl_exec($ch); $curl_info = @curl_getinfo($ch); diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index ac2d70cad2..1a0048b2a1 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -41,14 +41,14 @@ class HTTPRequest /** @var IConfig */ private $config; /** @var string */ - private $userAgent; + private $baseUrl; - public function __construct(LoggerInterface $logger, Profiler $profiler, IConfig $config, App $a) + public function __construct(LoggerInterface $logger, Profiler $profiler, IConfig $config, App\BaseURL $baseUrl) { - $this->logger = $logger; - $this->profiler = $profiler; - $this->config = $config; - $this->userAgent = $a->getUserAgent(); + $this->logger = $logger; + $this->profiler = $profiler; + $this->config = $config; + $this->baseUrl = $baseUrl->get(); } /** @@ -232,7 +232,7 @@ class HTTPRequest $stamp1 = microtime(true); if (Network::isUrlBlocked($url)) { - $this->logger->info('Domain is blocked.'. ['url' => $url]); + $this->logger->info('Domain is blocked.' . ['url' => $url]); return CurlResult::createErrorCurl($url); } @@ -378,4 +378,19 @@ class HTTPRequest $redirects ); } + + /** + * Returns the current UserAgent as a String + * + * @return string the UserAgent as a String + */ + public function getUserAgent() + { + return + FRIENDICA_PLATFORM . " '" . + FRIENDICA_CODENAME . "' " . + FRIENDICA_VERSION . '-' . + DB_UPDATE_VERSION . '; ' . + $this->baseUrl; + } } diff --git a/src/Util/Network.php b/src/Util/Network.php index 888dc20a69..a8b216b34b 100644 --- a/src/Util/Network.php +++ b/src/Util/Network.php @@ -350,7 +350,7 @@ class Network curl_setopt($ch, CURLOPT_NOBODY, 1); curl_setopt($ch, CURLOPT_TIMEOUT, 10); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); + curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); curl_exec($ch); $curl_info = @curl_getinfo($ch); @@ -394,7 +394,7 @@ class Network curl_setopt($ch, CURLOPT_NOBODY, 0); curl_setopt($ch, CURLOPT_TIMEOUT, 10); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_USERAGENT, $a->getUserAgent()); + curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); $body = curl_exec($ch); curl_close($ch); From 60e18736b0555e336b0f06c9a0849f39cded4414 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:30:24 +0100 Subject: [PATCH 09/17] Move "Network::finalUrl" to "HTTPRequest" class --- src/Network/HTTPRequest.php | 121 +++++++++++++++++++++++++++++++++++ src/Protocol/Feed.php | 3 +- src/Util/Network.php | 123 ------------------------------------ 3 files changed, 123 insertions(+), 124 deletions(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 1a0048b2a1..08ac203f5a 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -21,10 +21,13 @@ namespace Friendica\Network; +use DOMDocument; +use DomXPath; use Friendica\App; use Friendica\Core\Config\IConfig; use Friendica\Core\Logger; use Friendica\Core\System; +use Friendica\DI; use Friendica\Util\Network; use Friendica\Util\Profiler; use Psr\Log\LoggerInterface; @@ -323,6 +326,124 @@ class HTTPRequest return $curlResponse; } + /** + * Returns the original URL of the provided URL + * + * This function strips tracking query params and follows redirections, either + * through HTTP code or meta refresh tags. Stops after 10 redirections. + * + * @todo Remove the $fetchbody parameter that generates an extraneous HEAD request + * + * @see ParseUrl::getSiteinfo + * + * @param string $url A user-submitted URL + * @param int $depth The current redirection recursion level (internal) + * @param bool $fetchbody Wether to fetch the body or not after the HEAD requests + * @return string A canonical URL + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + public static function finalUrl(string $url, int $depth = 1, bool $fetchbody = false) + { + $url = Network::stripTrackingQueryParams($url); + + if ($depth > 10) { + return $url; + } + + $url = trim($url, "'"); + + $stamp1 = microtime(true); + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HEADER, 1); + curl_setopt($ch, CURLOPT_NOBODY, 1); + curl_setopt($ch, CURLOPT_TIMEOUT, 10); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); + + curl_exec($ch); + $curl_info = @curl_getinfo($ch); + $http_code = $curl_info['http_code']; + curl_close($ch); + + DI::profiler()->saveTimestamp($stamp1, "network", System::callstack()); + + if ($http_code == 0) { + return $url; + } + + if (in_array($http_code, ['301', '302'])) { + if (!empty($curl_info['redirect_url'])) { + return self::finalUrl($curl_info['redirect_url'], ++$depth, $fetchbody); + } elseif (!empty($curl_info['location'])) { + return self::finalUrl($curl_info['location'], ++$depth, $fetchbody); + } + } + + // Check for redirects in the meta elements of the body if there are no redirects in the header. + if (!$fetchbody) { + return self::finalUrl($url, ++$depth, true); + } + + // if the file is too large then exit + if ($curl_info["download_content_length"] > 1000000) { + return $url; + } + + // if it isn't a HTML file then exit + if (!empty($curl_info["content_type"]) && !strstr(strtolower($curl_info["content_type"]), "html")) { + return $url; + } + + $stamp1 = microtime(true); + + $ch = curl_init(); + curl_setopt($ch, CURLOPT_URL, $url); + curl_setopt($ch, CURLOPT_HEADER, 0); + curl_setopt($ch, CURLOPT_NOBODY, 0); + curl_setopt($ch, CURLOPT_TIMEOUT, 10); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); + + $body = curl_exec($ch); + curl_close($ch); + + DI::profiler()->saveTimestamp($stamp1, "network", System::callstack()); + + if (trim($body) == "") { + return $url; + } + + // Check for redirect in meta elements + $doc = new DOMDocument(); + @$doc->loadHTML($body); + + $xpath = new DomXPath($doc); + + $list = $xpath->query("//meta[@content]"); + foreach ($list as $node) { + $attr = []; + if ($node->attributes->length) { + foreach ($node->attributes as $attribute) { + $attr[$attribute->name] = $attribute->value; + } + } + + if (@$attr["http-equiv"] == 'refresh') { + $path = $attr["content"]; + $pathinfo = explode(";", $path); + foreach ($pathinfo as $value) { + if (substr(strtolower($value), 0, 4) == "url=") { + return self::finalUrl(substr($value, 4), ++$depth); + } + } + } + } + + return $url; + } + /** * Curl wrapper * diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index a665b7c85a..a609ae2963 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -35,6 +35,7 @@ use Friendica\Model\Contact; use Friendica\Model\Item; use Friendica\Model\Tag; use Friendica\Model\User; +use Friendica\Network\HTTPRequest; use Friendica\Util\DateTimeFormat; use Friendica\Util\Network; use Friendica\Util\ParseUrl; @@ -350,7 +351,7 @@ class Feed $orig_plink = $item["plink"]; - $item["plink"] = Network::finalUrl($item["plink"]); + $item["plink"] = HTTPRequest::finalUrl($item["plink"]); $item["parent-uri"] = $item["uri"]; diff --git a/src/Util/Network.php b/src/Util/Network.php index a8b216b34b..7795b0cd29 100644 --- a/src/Util/Network.php +++ b/src/Util/Network.php @@ -21,11 +21,8 @@ namespace Friendica\Util; -use DOMDocument; -use DomXPath; use Friendica\Core\Hook; use Friendica\Core\Logger; -use Friendica\Core\System; use Friendica\DI; class Network @@ -314,126 +311,6 @@ class Network return self::unparseURL($parts); } - /** - * Returns the original URL of the provided URL - * - * This function strips tracking query params and follows redirections, either - * through HTTP code or meta refresh tags. Stops after 10 redirections. - * - * @todo Remove the $fetchbody parameter that generates an extraneous HEAD request - * - * @see ParseUrl::getSiteinfo - * - * @param string $url A user-submitted URL - * @param int $depth The current redirection recursion level (internal) - * @param bool $fetchbody Wether to fetch the body or not after the HEAD requests - * @return string A canonical URL - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - */ - public static function finalUrl(string $url, int $depth = 1, bool $fetchbody = false) - { - $a = DI::app(); - - $url = self::stripTrackingQueryParams($url); - - if ($depth > 10) { - return $url; - } - - $url = trim($url, "'"); - - $stamp1 = microtime(true); - - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HEADER, 1); - curl_setopt($ch, CURLOPT_NOBODY, 1); - curl_setopt($ch, CURLOPT_TIMEOUT, 10); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); - - curl_exec($ch); - $curl_info = @curl_getinfo($ch); - $http_code = $curl_info['http_code']; - curl_close($ch); - - DI::profiler()->saveTimestamp($stamp1, "network", System::callstack()); - - if ($http_code == 0) { - return $url; - } - - if (in_array($http_code, ['301', '302'])) { - if (!empty($curl_info['redirect_url'])) { - return self::finalUrl($curl_info['redirect_url'], ++$depth, $fetchbody); - } elseif (!empty($curl_info['location'])) { - return self::finalUrl($curl_info['location'], ++$depth, $fetchbody); - } - } - - // Check for redirects in the meta elements of the body if there are no redirects in the header. - if (!$fetchbody) { - return(self::finalUrl($url, ++$depth, true)); - } - - // if the file is too large then exit - if ($curl_info["download_content_length"] > 1000000) { - return $url; - } - - // if it isn't a HTML file then exit - if (!empty($curl_info["content_type"]) && !strstr(strtolower($curl_info["content_type"]), "html")) { - return $url; - } - - $stamp1 = microtime(true); - - $ch = curl_init(); - curl_setopt($ch, CURLOPT_URL, $url); - curl_setopt($ch, CURLOPT_HEADER, 0); - curl_setopt($ch, CURLOPT_NOBODY, 0); - curl_setopt($ch, CURLOPT_TIMEOUT, 10); - curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); - - $body = curl_exec($ch); - curl_close($ch); - - DI::profiler()->saveTimestamp($stamp1, "network", System::callstack()); - - if (trim($body) == "") { - return $url; - } - - // Check for redirect in meta elements - $doc = new DOMDocument(); - @$doc->loadHTML($body); - - $xpath = new DomXPath($doc); - - $list = $xpath->query("//meta[@content]"); - foreach ($list as $node) { - $attr = []; - if ($node->attributes->length) { - foreach ($node->attributes as $attribute) { - $attr[$attribute->name] = $attribute->value; - } - } - - if (@$attr["http-equiv"] == 'refresh') { - $path = $attr["content"]; - $pathinfo = explode(";", $path); - foreach ($pathinfo as $value) { - if (substr(strtolower($value), 0, 4) == "url=") { - return self::finalUrl(substr($value, 4), ++$depth); - } - } - } - } - - return $url; - } - /** * Find the matching part between two url * From e5649d6dbf32c2cad6cfb3b364161270b0fb8f03 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:33:31 +0100 Subject: [PATCH 10/17] Move "HTTPRequest::finalUrl" dynamic --- src/Network/HTTPRequest.php | 34 ++++++++++++++++------------------ src/Protocol/Feed.php | 2 +- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 08ac203f5a..3e5091e076 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -25,9 +25,7 @@ use DOMDocument; use DomXPath; use Friendica\App; use Friendica\Core\Config\IConfig; -use Friendica\Core\Logger; use Friendica\Core\System; -use Friendica\DI; use Friendica\Util\Network; use Friendica\Util\Profiler; use Psr\Log\LoggerInterface; @@ -130,7 +128,7 @@ class HTTPRequest } @curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - @curl_setopt($ch, CURLOPT_USERAGENT, $this->userAgent); + @curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent()); $range = intval($this->config->get('system', 'curl_range_bytes', 0)); @@ -208,7 +206,7 @@ class HTTPRequest $redirects++; $this->logger->notice('Curl redirect.', ['url' => $url, 'to' => $curlResponse->getRedirectUrl()]); @curl_close($ch); - return self::curl($curlResponse->getRedirectUrl(), $binary, $opts, $redirects); + return $this->curl($curlResponse->getRedirectUrl(), $binary, $opts, $redirects); } @curl_close($ch); @@ -251,7 +249,7 @@ class HTTPRequest curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_POST, 1); curl_setopt($ch, CURLOPT_POSTFIELDS, $params); - curl_setopt($ch, CURLOPT_USERAGENT, $this->userAgent); + curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent()); if ($this->config->get('system', 'ipv4_resolve', false)) { curl_setopt($ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); @@ -299,7 +297,7 @@ class HTTPRequest $redirects++; $this->logger->info('Post redirect.', ['url' => $url, 'to' => $curlResponse->getRedirectUrl()]); curl_close($ch); - return self::post($curlResponse->getRedirectUrl(), $params, $headers, $redirects, $timeout); + return $this->post($curlResponse->getRedirectUrl(), $params, $headers, $redirects, $timeout); } curl_close($ch); @@ -317,11 +315,11 @@ class HTTPRequest array_push($headers, 'Expect:'); } } - Logger::info('Server responds with 417, applying workaround', ['url' => $url]); - return self::post($url, $params, $headers, $redirects, $timeout); + $this->logger->info('Server responds with 417, applying workaround', ['url' => $url]); + return $this->post($url, $params, $headers, $redirects, $timeout); } - Logger::log('post_url: end ' . $url, Logger::DATA); + $this->logger->debug('Post_url: End.', ['url' => $url]); return $curlResponse; } @@ -342,7 +340,7 @@ class HTTPRequest * @return string A canonical URL * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function finalUrl(string $url, int $depth = 1, bool $fetchbody = false) + public function finalUrl(string $url, int $depth = 1, bool $fetchbody = false) { $url = Network::stripTrackingQueryParams($url); @@ -360,14 +358,14 @@ class HTTPRequest curl_setopt($ch, CURLOPT_NOBODY, 1); curl_setopt($ch, CURLOPT_TIMEOUT, 10); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); + curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent()); curl_exec($ch); $curl_info = @curl_getinfo($ch); $http_code = $curl_info['http_code']; curl_close($ch); - DI::profiler()->saveTimestamp($stamp1, "network", System::callstack()); + $this->profiler->saveTimestamp($stamp1, "network", System::callstack()); if ($http_code == 0) { return $url; @@ -375,15 +373,15 @@ class HTTPRequest if (in_array($http_code, ['301', '302'])) { if (!empty($curl_info['redirect_url'])) { - return self::finalUrl($curl_info['redirect_url'], ++$depth, $fetchbody); + return $this->finalUrl($curl_info['redirect_url'], ++$depth, $fetchbody); } elseif (!empty($curl_info['location'])) { - return self::finalUrl($curl_info['location'], ++$depth, $fetchbody); + return $this->finalUrl($curl_info['location'], ++$depth, $fetchbody); } } // Check for redirects in the meta elements of the body if there are no redirects in the header. if (!$fetchbody) { - return self::finalUrl($url, ++$depth, true); + return $this->finalUrl($url, ++$depth, true); } // if the file is too large then exit @@ -404,12 +402,12 @@ class HTTPRequest curl_setopt($ch, CURLOPT_NOBODY, 0); curl_setopt($ch, CURLOPT_TIMEOUT, 10); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); - curl_setopt($ch, CURLOPT_USERAGENT, DI::httpRequest()->getUserAgent()); + curl_setopt($ch, CURLOPT_USERAGENT, $this->getUserAgent()); $body = curl_exec($ch); curl_close($ch); - DI::profiler()->saveTimestamp($stamp1, "network", System::callstack()); + $this->profiler->saveTimestamp($stamp1, "network", System::callstack()); if (trim($body) == "") { return $url; @@ -435,7 +433,7 @@ class HTTPRequest $pathinfo = explode(";", $path); foreach ($pathinfo as $value) { if (substr(strtolower($value), 0, 4) == "url=") { - return self::finalUrl(substr($value, 4), ++$depth); + return $this->finalUrl(substr($value, 4), ++$depth); } } } diff --git a/src/Protocol/Feed.php b/src/Protocol/Feed.php index a609ae2963..9aab4f52c6 100644 --- a/src/Protocol/Feed.php +++ b/src/Protocol/Feed.php @@ -351,7 +351,7 @@ class Feed $orig_plink = $item["plink"]; - $item["plink"] = HTTPRequest::finalUrl($item["plink"]); + $item["plink"] = DI::httpRequest()->finalUrl($item["plink"]); $item["parent-uri"] = $item["uri"]; From 7029012f27e9f5199617ae1ec2c3aae10f0d1fc8 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:35:09 +0100 Subject: [PATCH 11/17] Rename "HTTPRequest::curl()" to HTTPRequest::get() --- mod/ostatus_subscribe.php | 2 +- mod/parse_url.php | 3 ++- mod/redir.php | 2 +- src/Content/Text/BBCode.php | 2 +- src/Core/Search.php | 2 +- src/Model/GContact.php | 6 +++--- src/Model/GServer.php | 36 ++++++++++++++++---------------- src/Model/Photo.php | 2 +- src/Model/Profile.php | 2 +- src/Model/User.php | 2 +- src/Module/Admin/Summary.php | 2 +- src/Module/Magic.php | 2 +- src/Network/HTTPRequest.php | 6 +++--- src/Network/Probe.php | 22 +++++++++---------- src/Protocol/ActivityPub.php | 3 ++- src/Protocol/DFRN.php | 2 +- src/Protocol/OStatus.php | 14 ++++++------- src/Protocol/PortableContact.php | 9 ++++---- src/Util/ExAuth.php | 2 +- src/Util/HTTPSignature.php | 2 +- src/Util/ParseUrl.php | 3 ++- src/Worker/OnePoll.php | 4 ++-- 22 files changed, 66 insertions(+), 64 deletions(-) diff --git a/mod/ostatus_subscribe.php b/mod/ostatus_subscribe.php index 5a3a625cec..459e9e2c92 100644 --- a/mod/ostatus_subscribe.php +++ b/mod/ostatus_subscribe.php @@ -54,7 +54,7 @@ function ostatus_subscribe_content(App $a) $api = $contact['baseurl'] . '/api/'; // Fetching friends - $curlResult = DI::httpRequest()->curl($api . 'statuses/friends.json?screen_name=' . $contact['nick']); + $curlResult = DI::httpRequest()->get($api . 'statuses/friends.json?screen_name=' . $contact['nick']); if (!$curlResult->isSuccess()) { DI::pConfig()->delete($uid, 'ostatus', 'legacy_contact'); diff --git a/mod/parse_url.php b/mod/parse_url.php index 0e80d971a6..a1faab6efb 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -28,6 +28,7 @@ use Friendica\Content\PageInfo; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Core\System; +use Friendica\DI; use Friendica\Util\ParseUrl; use Friendica\Util\Strings; @@ -84,7 +85,7 @@ function parse_url_content(App $a) // Check if the URL is an image, video or audio file. If so format // the URL with the corresponding BBCode media tag // Fetch the header of the URL - $curlResponse = DI::httpRequest()->curl($url, false, ['novalidate' => true, 'nobody' => true]); + $curlResponse = DI::httpRequest()->get($url, false, ['novalidate' => true, 'nobody' => true]); if ($curlResponse->isSuccess()) { // Convert the header fields into an array diff --git a/mod/redir.php b/mod/redir.php index 4069518cdd..b2f76738be 100644 --- a/mod/redir.php +++ b/mod/redir.php @@ -170,7 +170,7 @@ function redir_magic($a, $cid, $url) } // Test for magic auth on the target system - $serverret = DI::httpRequest()->curl($basepath . '/magic'); + $serverret = DI::httpRequest()->get($basepath . '/magic'); if ($serverret->isSuccess()) { $separator = strpos($target_url, '?') ? '&' : '?'; $target_url .= $separator . 'zrl=' . urlencode($visitor) . '&addr=' . urlencode($contact_url); diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index 036bc4a270..bf208a1036 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -485,7 +485,7 @@ class BBCode continue; } - $curlResult = DI::httpRequest()->curl($mtch[1], true); + $curlResult = DI::httpRequest()->get($mtch[1], true); if (!$curlResult->isSuccess()) { continue; } diff --git a/src/Core/Search.php b/src/Core/Search.php index 768a0bf275..c5c6ca08c2 100644 --- a/src/Core/Search.php +++ b/src/Core/Search.php @@ -284,7 +284,7 @@ class Search $return = GContact::searchByName($search, $mode); } else { $p = $page > 1 ? 'p=' . $page : ''; - $curlResult = DI::httpRequest()->curl(self::getGlobalDirectory() . '/search/people?' . $p . '&q=' . urlencode($search), false, ['accept_content' => 'application/json']); + $curlResult = DI::httpRequest()->get(self::getGlobalDirectory() . '/search/people?' . $p . '&q=' . urlencode($search), false, ['accept_content' => 'application/json']); if ($curlResult->isSuccess()) { $searchResult = json_decode($curlResult->getBody(), true); if (!empty($searchResult['profiles'])) { diff --git a/src/Model/GContact.php b/src/Model/GContact.php index dabc907f89..109f5d54b3 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -844,7 +844,7 @@ class GContact return false; } - $curlResult = DI::httpRequest()->curl($gserver['noscrape'] . '/' . $data['nick']); + $curlResult = DI::httpRequest()->get($gserver['noscrape'] . '/' . $data['nick']); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { $noscrape = json_decode($curlResult->getBody(), true); @@ -926,7 +926,7 @@ class GContact private static function updateFromFeed(array $data) { // Search for the newest entry in the feed - $curlResult = DI::httpRequest()->curl($data['poll']); + $curlResult = DI::httpRequest()->get($data['poll']); if (!$curlResult->isSuccess()) { $fields = ['failed' => true, 'last_failure' => DateTimeFormat::utcNow()]; DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]); @@ -1204,7 +1204,7 @@ class GContact $url = $server . '/main/statistics'; - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if (!$curlResult->isSuccess()) { return false; } diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 76082fe1ff..ae4332511d 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -309,7 +309,7 @@ class GServer // When a nodeinfo is present, we don't need to dig further $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = DI::httpRequest()->curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpRequest()->get($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); if ($curlResult->isTimeout()) { self::setFailure($url); return false; @@ -342,7 +342,7 @@ class GServer $basedata = ['detection-method' => self::DETECT_MANUAL]; } - $curlResult = DI::httpRequest()->curl($baseurl, false, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpRequest()->get($baseurl, false, ['timeout' => $xrd_timeout]); if ($curlResult->isSuccess()) { $basedata = self::analyseRootHeader($curlResult, $basedata); $basedata = self::analyseRootBody($curlResult, $basedata, $baseurl); @@ -498,7 +498,7 @@ class GServer { Logger::info('Discover relay data', ['server' => $server_url]); - $curlResult = DI::httpRequest()->curl($server_url . '/.well-known/x-social-relay'); + $curlResult = DI::httpRequest()->get($server_url . '/.well-known/x-social-relay'); if (!$curlResult->isSuccess()) { return; } @@ -579,7 +579,7 @@ class GServer */ private static function fetchStatistics(string $url) { - $curlResult = DI::httpRequest()->curl($url . '/statistics.json'); + $curlResult = DI::httpRequest()->get($url . '/statistics.json'); if (!$curlResult->isSuccess()) { return []; } @@ -689,7 +689,7 @@ class GServer */ private static function parseNodeinfo1(string $nodeinfo_url) { - $curlResult = DI::httpRequest()->curl($nodeinfo_url); + $curlResult = DI::httpRequest()->get($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; @@ -766,7 +766,7 @@ class GServer */ private static function parseNodeinfo2(string $nodeinfo_url) { - $curlResult = DI::httpRequest()->curl($nodeinfo_url); + $curlResult = DI::httpRequest()->get($nodeinfo_url); if (!$curlResult->isSuccess()) { return []; } @@ -843,7 +843,7 @@ class GServer */ private static function fetchSiteinfo(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->curl($url . '/siteinfo.json'); + $curlResult = DI::httpRequest()->get($url . '/siteinfo.json'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -912,7 +912,7 @@ class GServer private static function validHostMeta(string $url) { $xrd_timeout = DI::config()->get('system', 'xrd_timeout'); - $curlResult = DI::httpRequest()->curl($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpRequest()->get($url . '/.well-known/host-meta', false, ['timeout' => $xrd_timeout]); if (!$curlResult->isSuccess()) { return false; } @@ -1008,7 +1008,7 @@ class GServer { $serverdata['poco'] = ''; - $curlResult = DI::httpRequest()->curl($url . '/poco'); + $curlResult = DI::httpRequest()->get($url . '/poco'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1038,7 +1038,7 @@ class GServer */ public static function checkMastodonDirectory(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->curl($url . '/api/v1/directory?limit=1'); + $curlResult = DI::httpRequest()->get($url . '/api/v1/directory?limit=1'); if (!$curlResult->isSuccess()) { return $serverdata; } @@ -1065,7 +1065,7 @@ class GServer */ private static function detectNextcloud(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->curl($url . '/status.php'); + $curlResult = DI::httpRequest()->get($url . '/status.php'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -1099,7 +1099,7 @@ class GServer */ private static function detectMastodonAlikes(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->curl($url . '/api/v1/instance'); + $curlResult = DI::httpRequest()->get($url . '/api/v1/instance'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; @@ -1165,7 +1165,7 @@ class GServer */ private static function detectHubzilla(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->curl($url . '/api/statusnet/config.json'); + $curlResult = DI::httpRequest()->get($url . '/api/statusnet/config.json'); if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -1263,7 +1263,7 @@ class GServer private static function detectGNUSocial(string $url, array $serverdata) { // Test for GNU Social - $curlResult = DI::httpRequest()->curl($url . '/api/gnusocial/version.json'); + $curlResult = DI::httpRequest()->get($url . '/api/gnusocial/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { $serverdata['platform'] = 'gnusocial'; @@ -1281,7 +1281,7 @@ class GServer } // Test for Statusnet - $curlResult = DI::httpRequest()->curl($url . '/api/statusnet/version.json'); + $curlResult = DI::httpRequest()->get($url . '/api/statusnet/version.json'); if ($curlResult->isSuccess() && ($curlResult->getBody() != '{"error":"not implemented"}') && ($curlResult->getBody() != '') && (strlen($curlResult->getBody()) < 30)) { @@ -1317,9 +1317,9 @@ class GServer */ private static function detectFriendica(string $url, array $serverdata) { - $curlResult = DI::httpRequest()->curl($url . '/friendica/json'); + $curlResult = DI::httpRequest()->get($url . '/friendica/json'); if (!$curlResult->isSuccess()) { - $curlResult = DI::httpRequest()->curl($url . '/friendika/json'); + $curlResult = DI::httpRequest()->get($url . '/friendika/json'); $friendika = true; $platform = 'Friendika'; } else { @@ -1652,7 +1652,7 @@ class GServer if (!empty($accesstoken)) { $api = 'https://instances.social/api/1.0/instances/list?count=0'; $header = ['Authorization: Bearer '.$accesstoken]; - $curlResult = DI::httpRequest()->curl($api, false, ['headers' => $header]); + $curlResult = DI::httpRequest()->get($api, false, ['headers' => $header]); if ($curlResult->isSuccess()) { $servers = json_decode($curlResult->getBody(), true); diff --git a/src/Model/Photo.php b/src/Model/Photo.php index a10711db2a..7d984a8ce6 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -420,7 +420,7 @@ class Photo $filename = basename($image_url); if (!empty($image_url)) { - $ret = DI::httpRequest()->curl($image_url, true); + $ret = DI::httpRequest()->get($image_url, true); $img_str = $ret->getBody(); $type = $ret->getContentType(); } else { diff --git a/src/Model/Profile.php b/src/Model/Profile.php index d32940ae67..0cff8ba287 100644 --- a/src/Model/Profile.php +++ b/src/Model/Profile.php @@ -737,7 +737,7 @@ class Profile $magic_path = $basepath . '/magic' . '?owa=1&dest=' . $dest . '&' . $addr_request; // We have to check if the remote server does understand /magic without invoking something - $serverret = DI::httpRequest()->curl($basepath . '/magic'); + $serverret = DI::httpRequest()->get($basepath . '/magic'); if ($serverret->isSuccess()) { Logger::log('Doing magic auth for visitor ' . $my_url . ' to ' . $magic_path, Logger::DEBUG); System::externalRedirect($magic_path); diff --git a/src/Model/User.php b/src/Model/User.php index 9980c48d5a..78ae958041 100644 --- a/src/Model/User.php +++ b/src/Model/User.php @@ -823,7 +823,7 @@ class User $photo_failure = false; $filename = basename($photo); - $curlResult = DI::httpRequest()->curl($photo, true); + $curlResult = DI::httpRequest()->get($photo, true); if ($curlResult->isSuccess()) { $img_str = $curlResult->getBody(); $type = $curlResult->getContentType(); diff --git a/src/Module/Admin/Summary.php b/src/Module/Admin/Summary.php index ad84bb6ab6..a130c48393 100644 --- a/src/Module/Admin/Summary.php +++ b/src/Module/Admin/Summary.php @@ -246,7 +246,7 @@ class Summary extends BaseAdmin private static function checkSelfHostMeta() { // Fetch the host-meta to check if this really is a vital server - return DI::httpRequest()->curl(DI::baseUrl()->get() . '/.well-known/host-meta')->isSuccess(); + return DI::httpRequest()->get(DI::baseUrl()->get() . '/.well-known/host-meta')->isSuccess(); } } diff --git a/src/Module/Magic.php b/src/Module/Magic.php index a025617d8d..95b742bb30 100644 --- a/src/Module/Magic.php +++ b/src/Module/Magic.php @@ -100,7 +100,7 @@ class Magic extends BaseModule ); // Try to get an authentication token from the other instance. - $curlResult = DI::httpRequest()->curl($basepath . '/owa', false, ['headers' => $headers]); + $curlResult = DI::httpRequest()->get($basepath . '/owa', false, ['headers' => $headers]); if ($curlResult->isSuccess()) { $j = json_decode($curlResult->getBody(), true); diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index 3e5091e076..f9279fa602 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -71,7 +71,7 @@ class HTTPRequest * @return CurlResult * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public function curl(string $url, bool $binary = false, array $opts = [], int &$redirects = 0) + public function get(string $url, bool $binary = false, array $opts = [], int &$redirects = 0) { $stamp1 = microtime(true); @@ -206,7 +206,7 @@ class HTTPRequest $redirects++; $this->logger->notice('Curl redirect.', ['url' => $url, 'to' => $curlResponse->getRedirectUrl()]); @curl_close($ch); - return $this->curl($curlResponse->getRedirectUrl(), $binary, $opts, $redirects); + return $this->get($curlResponse->getRedirectUrl(), $binary, $opts, $redirects); } @curl_close($ch); @@ -486,7 +486,7 @@ class HTTPRequest */ public function fetchUrlFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) { - return $this->curl( + return $this->get( $url, $binary, [ diff --git a/src/Network/Probe.php b/src/Network/Probe.php index 01dc284081..c41006b128 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -166,7 +166,7 @@ class Probe Logger::info('Probing', ['host' => $host, 'ssl_url' => $ssl_url, 'url' => $url, 'callstack' => System::callstack(20)]); $xrd = null; - $curlResult = DI::httpRequest()->curl($ssl_url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); + $curlResult = DI::httpRequest()->get($ssl_url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); $ssl_connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0); if ($curlResult->isSuccess()) { $xml = $curlResult->getBody(); @@ -183,7 +183,7 @@ class Probe } if (!is_object($xrd) && !empty($url)) { - $curlResult = DI::httpRequest()->curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); + $curlResult = DI::httpRequest()->get($url, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/xrd+xml']); $connection_error = ($curlResult->getErrorNumber() == CURLE_COULDNT_CONNECT) || ($curlResult->getReturnCode() == 0); if ($curlResult->isTimeout()) { Logger::info('Probing timeout', ['url' => $url]); @@ -427,7 +427,7 @@ class Probe */ private static function getHideStatus($url) { - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if (!$curlResult->isSuccess()) { return false; } @@ -841,7 +841,7 @@ class Probe public static function pollZot($url, $data) { - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if ($curlResult->isTimeout()) { return $data; } @@ -938,7 +938,7 @@ class Probe { $xrd_timeout = DI::config()->get('system', 'xrd_timeout', 20); - $curlResult = DI::httpRequest()->curl($url, false, ['timeout' => $xrd_timeout, 'accept_content' => $type]); + $curlResult = DI::httpRequest()->get($url, false, ['timeout' => $xrd_timeout, 'accept_content' => $type]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1007,7 +1007,7 @@ class Probe */ private static function pollNoscrape($noscrape_url, $data) { - $curlResult = DI::httpRequest()->curl($noscrape_url); + $curlResult = DI::httpRequest()->get($noscrape_url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1265,7 +1265,7 @@ class Probe */ private static function pollHcard($hcard_url, $data, $dfrn = false) { - $curlResult = DI::httpRequest()->curl($hcard_url); + $curlResult = DI::httpRequest()->get($hcard_url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1519,7 +1519,7 @@ class Probe $pubkey = substr($pubkey, 5); } } elseif (Strings::normaliseLink($pubkey) == 'http://') { - $curlResult = DI::httpRequest()->curl($pubkey); + $curlResult = DI::httpRequest()->get($pubkey); if ($curlResult->isTimeout()) { self::$istimeout = true; return $short ? false : []; @@ -1552,7 +1552,7 @@ class Probe } // Fetch all additional data from the feed - $curlResult = DI::httpRequest()->curl($data["poll"]); + $curlResult = DI::httpRequest()->get($data["poll"]); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; @@ -1604,7 +1604,7 @@ class Probe */ private static function pumpioProfileData($profile_link) { - $curlResult = DI::httpRequest()->curl($profile_link); + $curlResult = DI::httpRequest()->get($profile_link); if (!$curlResult->isSuccess()) { return []; } @@ -1835,7 +1835,7 @@ class Probe */ private static function feed($url, $probe = true) { - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if ($curlResult->isTimeout()) { self::$istimeout = true; return []; diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index f417080114..6b29eabce5 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -22,6 +22,7 @@ namespace Friendica\Protocol; use Friendica\Core\Protocol; +use Friendica\DI; use Friendica\Model\APContact; use Friendica\Model\User; use Friendica\Util\HTTPSignature; @@ -92,7 +93,7 @@ class ActivityPub return HTTPSignature::fetch($url, $uid); } - $curlResult = DI::httpRequest()->curl($url, false, ['accept_content' => 'application/activity+json, application/ld+json']); + $curlResult = DI::httpRequest()->get($url, false, ['accept_content' => 'application/activity+json, application/ld+json']); if (!$curlResult->isSuccess() || empty($curlResult->getBody())) { return false; } diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php index 7c87ee0b25..8190806a04 100644 --- a/src/Protocol/DFRN.php +++ b/src/Protocol/DFRN.php @@ -1194,7 +1194,7 @@ class DFRN Logger::log('dfrn_deliver: ' . $url); - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if ($curlResult->isTimeout()) { return -2; // timed out diff --git a/src/Protocol/OStatus.php b/src/Protocol/OStatus.php index a9ee2277e3..9c87f367ad 100644 --- a/src/Protocol/OStatus.php +++ b/src/Protocol/OStatus.php @@ -755,7 +755,7 @@ class OStatus self::$conv_list[$conversation] = true; - $curlResult = DI::httpRequest()->curl($conversation, false, ['accept_content' => 'application/atom+xml, text/html']); + $curlResult = DI::httpRequest()->get($conversation, false, ['accept_content' => 'application/atom+xml, text/html']); if (!$curlResult->isSuccess()) { return; @@ -784,7 +784,7 @@ class OStatus } } if ($file != '') { - $conversation_atom = DI::httpRequest()->curl($attribute['href']); + $conversation_atom = DI::httpRequest()->get($attribute['href']); if ($conversation_atom->isSuccess()) { $xml = $conversation_atom->getBody(); @@ -901,7 +901,7 @@ class OStatus return; } - $curlResult = DI::httpRequest()->curl($self); + $curlResult = DI::httpRequest()->get($self); if (!$curlResult->isSuccess()) { return; @@ -948,7 +948,7 @@ class OStatus } $stored = false; - $curlResult = DI::httpRequest()->curl($related, false, ['accept_content' => 'application/atom+xml, text/html']); + $curlResult = DI::httpRequest()->get($related, false, ['accept_content' => 'application/atom+xml, text/html']); if (!$curlResult->isSuccess()) { return; @@ -979,7 +979,7 @@ class OStatus } } if ($atom_file != '') { - $curlResult = DI::httpRequest()->curl($atom_file); + $curlResult = DI::httpRequest()->get($atom_file); if ($curlResult->isSuccess()) { Logger::log('Fetched XML for URI ' . $related_uri, Logger::DEBUG); @@ -991,7 +991,7 @@ class OStatus // Workaround for older GNU Social servers if (($xml == '') && strstr($related, '/notice/')) { - $curlResult = DI::httpRequest()->curl(str_replace('/notice/', '/api/statuses/show/', $related) . '.atom'); + $curlResult = DI::httpRequest()->get(str_replace('/notice/', '/api/statuses/show/', $related) . '.atom'); if ($curlResult->isSuccess()) { Logger::log('GNU Social workaround to fetch XML for URI ' . $related_uri, Logger::DEBUG); @@ -1002,7 +1002,7 @@ class OStatus // Even more worse workaround for GNU Social ;-) if ($xml == '') { $related_guess = self::convertHref($related_uri); - $curlResult = DI::httpRequest()->curl(str_replace('/notice/', '/api/statuses/show/', $related_guess) . '.atom'); + $curlResult = DI::httpRequest()->get(str_replace('/notice/', '/api/statuses/show/', $related_guess) . '.atom'); if ($curlResult->isSuccess()) { Logger::log('GNU Social workaround 2 to fetch XML for URI ' . $related_uri, Logger::DEBUG); diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index f118ffa949..5216e39370 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -30,7 +30,6 @@ use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\GContact; use Friendica\Model\GServer; -use Friendica\Network\HTTPRequest; use Friendica\Util\DateTimeFormat; use Friendica\Util\Strings; @@ -251,7 +250,7 @@ class PortableContact */ private static function fetchServerlist($poco) { - $curlResult = DI::httpRequest()->curl($poco . "/@server"); + $curlResult = DI::httpRequest()->get($poco . "/@server"); if (!$curlResult->isSuccess()) { return; @@ -291,7 +290,7 @@ class PortableContact Logger::info("Fetch all users from the server " . $server["url"]); - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { $data = json_decode($curlResult->getBody(), true); @@ -314,7 +313,7 @@ class PortableContact $success = false; - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if ($curlResult->isSuccess() && !empty($curlResult->getBody())) { Logger::info("Fetch all global contacts from the server " . $server["nurl"]); @@ -372,7 +371,7 @@ class PortableContact // Fetch all contacts from a given user from the other server $url = $server['poco'] . '/' . $username . '/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,contactType,generation'; - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if ($curlResult->isSuccess()) { $data = json_decode($curlResult->getBody(), true); diff --git a/src/Util/ExAuth.php b/src/Util/ExAuth.php index 5933860827..25eb3cc62e 100644 --- a/src/Util/ExAuth.php +++ b/src/Util/ExAuth.php @@ -181,7 +181,7 @@ class ExAuth $url = ($ssl ? 'https' : 'http') . '://' . $host . '/noscrape/' . $user; - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if (!$curlResult->isSuccess()) { return false; diff --git a/src/Util/HTTPSignature.php b/src/Util/HTTPSignature.php index 84a11b8ec0..89da59ba26 100644 --- a/src/Util/HTTPSignature.php +++ b/src/Util/HTTPSignature.php @@ -442,7 +442,7 @@ class HTTPSignature $curl_opts = $opts; $curl_opts['header'] = $headers; - $curlResult = DI::httpRequest()->curl($request, false, $curl_opts); + $curlResult = DI::httpRequest()->get($request, false, $curl_opts); $return_code = $curlResult->getReturnCode(); Logger::log('Fetched for user ' . $uid . ' from ' . $request . ' returned ' . $return_code, Logger::DEBUG); diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index cf38ffd7be..01ad79d4f1 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -27,6 +27,7 @@ use Friendica\Content\OEmbed; use Friendica\Core\Hook; use Friendica\Core\Logger; use Friendica\Database\DBA; +use Friendica\DI; /** * Get information about a given URL @@ -159,7 +160,7 @@ class ParseUrl return $siteinfo; } - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if (!$curlResult->isSuccess()) { return $siteinfo; } diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index ed6bfacb35..fbd1ab4e59 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -290,7 +290,7 @@ class OnePoll . '&type=data&last_update=' . $last_update . '&perm=' . $perm; - $curlResult = DI::httpRequest()->curl($url); + $curlResult = DI::httpRequest()->get($url); if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) { // set the last-update so we don't keep polling @@ -443,7 +443,7 @@ class OnePoll } $cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-'); - $curlResult = DI::httpRequest()->curl($contact['poll'], false, ['cookiejar' => $cookiejar]); + $curlResult = DI::httpRequest()->get($contact['poll'], false, ['cookiejar' => $cookiejar]); unlink($cookiejar); if ($curlResult->isTimeout()) { From 657d08f09f94f53f2fb7515de73c3687ad71c0d2 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:35:40 +0100 Subject: [PATCH 12/17] Rename "fetchUrl" and "fetchUrlFull" to "fetch" and "fetchFull" --- mod/dfrn_poll.php | 14 +++++++------- mod/dfrn_request.php | 2 +- mod/oexchange.php | 2 +- mod/pubsubhubbub.php | 3 +-- src/Content/OEmbed.php | 6 +++--- src/Content/Text/BBCode.php | 4 ++-- src/Core/Installer.php | 5 ++--- src/Core/Search.php | 2 +- src/Core/Worker.php | 2 +- src/Model/GContact.php | 2 +- src/Model/GServer.php | 2 +- src/Module/Debug/Feed.php | 2 +- src/Network/HTTPRequest.php | 6 +++--- src/Protocol/Diaspora.php | 2 +- src/Protocol/PortableContact.php | 2 +- src/Util/Images.php | 2 +- src/Worker/CheckVersion.php | 2 +- src/Worker/CronJobs.php | 2 +- src/Worker/Directory.php | 2 +- src/Worker/SearchDirectory.php | 2 +- 20 files changed, 32 insertions(+), 34 deletions(-) diff --git a/mod/dfrn_poll.php b/mod/dfrn_poll.php index 7f7fbe498f..183f6022e3 100644 --- a/mod/dfrn_poll.php +++ b/mod/dfrn_poll.php @@ -114,7 +114,7 @@ function dfrn_poll_init(App $a) ); if (DBA::isResult($r)) { - $s = DI::httpRequest()->fetchUrl($r[0]['poll'] . '?dfrn_id=' . $my_id . '&type=profile-check'); + $s = DI::httpRequest()->fetch($r[0]['poll'] . '?dfrn_id=' . $my_id . '&type=profile-check'); Logger::log("dfrn_poll: old profile returns " . $s, Logger::DATA); @@ -498,12 +498,12 @@ function dfrn_poll_content(App $a) // URL reply if ($dfrn_version < 2.2) { - $s = DI::httpRequest()->fetchUrl($r[0]['poll'] - . '?dfrn_id=' . $encrypted_id - . '&type=profile-check' - . '&dfrn_version=' . DFRN_PROTOCOL_VERSION - . '&challenge=' . $challenge - . '&sec=' . $sec + $s = DI::httpRequest()->fetch($r[0]['poll'] + . '?dfrn_id=' . $encrypted_id + . '&type=profile-check' + . '&dfrn_version=' . DFRN_PROTOCOL_VERSION + . '&challenge=' . $challenge + . '&sec=' . $sec ); } else { $s = DI::httpRequest()->post($r[0]['poll'], [ diff --git a/mod/dfrn_request.php b/mod/dfrn_request.php index 8c8557650e..cb21a211f9 100644 --- a/mod/dfrn_request.php +++ b/mod/dfrn_request.php @@ -203,7 +203,7 @@ function dfrn_request_post(App $a) } if (!empty($dfrn_request) && strlen($confirm_key)) { - DI::httpRequest()->fetchUrl($dfrn_request . '?confirm_key=' . $confirm_key); + DI::httpRequest()->fetch($dfrn_request . '?confirm_key=' . $confirm_key); } // (ignore reply, nothing we can do it failed) diff --git a/mod/oexchange.php b/mod/oexchange.php index 4746651c16..b8da9df7ef 100644 --- a/mod/oexchange.php +++ b/mod/oexchange.php @@ -57,7 +57,7 @@ function oexchange_content(App $a) { $tags = ((!empty($_REQUEST['tags'])) ? '&tags=' . urlencode(Strings::escapeTags(trim($_REQUEST['tags']))) : ''); - $s = DI::httpRequest()->fetchUrl(DI::baseUrl() . '/parse_url?url=' . $url . $title . $description . $tags); + $s = DI::httpRequest()->fetch(DI::baseUrl() . '/parse_url?url=' . $url . $title . $description . $tags); if (!strlen($s)) { return; diff --git a/mod/pubsubhubbub.php b/mod/pubsubhubbub.php index 96f26838ed..3445436182 100644 --- a/mod/pubsubhubbub.php +++ b/mod/pubsubhubbub.php @@ -24,7 +24,6 @@ use Friendica\Core\Logger; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\PushSubscriber; -use Friendica\Network\HTTPRequest; use Friendica\Util\Strings; function post_var($name) { @@ -126,7 +125,7 @@ function pubsubhubbub_init(App $a) { $hub_callback = rtrim($hub_callback, ' ?&#'); $separator = parse_url($hub_callback, PHP_URL_QUERY) === null ? '?' : '&'; - $fetchResult = DI::httpRequest()->fetchUrlFull($hub_callback . $separator . $params); + $fetchResult = DI::httpRequest()->fetchFull($hub_callback . $separator . $params); $body = $fetchResult->getBody(); $ret = $fetchResult->getReturnCode(); diff --git a/src/Content/OEmbed.php b/src/Content/OEmbed.php index 592d25c10d..30a113f461 100644 --- a/src/Content/OEmbed.php +++ b/src/Content/OEmbed.php @@ -95,7 +95,7 @@ class OEmbed if (!in_array($ext, $noexts)) { // try oembed autodiscovery - $html_text = DI::httpRequest()->fetchUrl($embedurl, false, 15, 'text/*'); + $html_text = DI::httpRequest()->fetch($embedurl, false, 15, 'text/*'); if ($html_text) { $dom = @DOMDocument::loadHTML($html_text); if ($dom) { @@ -103,14 +103,14 @@ class OEmbed $entries = $xpath->query("//link[@type='application/json+oembed']"); foreach ($entries as $e) { $href = $e->getAttributeNode('href')->nodeValue; - $json_string = DI::httpRequest()->fetchUrl($href . '&maxwidth=' . $a->videowidth); + $json_string = DI::httpRequest()->fetch($href . '&maxwidth=' . $a->videowidth); break; } $entries = $xpath->query("//link[@type='text/json+oembed']"); foreach ($entries as $e) { $href = $e->getAttributeNode('href')->nodeValue; - $json_string = DI::httpRequest()->fetchUrl($href . '&maxwidth=' . $a->videowidth); + $json_string = DI::httpRequest()->fetch($href . '&maxwidth=' . $a->videowidth); break; } } diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index bf208a1036..1181c8f47b 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1106,7 +1106,7 @@ class BBCode $text = "[url=" . $match[2] . ']' . $match[2] . "[/url]"; // if its not a picture then look if its a page that contains a picture link - $body = DI::httpRequest()->fetchUrl($match[1]); + $body = DI::httpRequest()->fetch($match[1]); $doc = new DOMDocument(); @$doc->loadHTML($body); @@ -1185,7 +1185,7 @@ class BBCode } // if its not a picture then look if its a page that contains a picture link - $body = DI::httpRequest()->fetchUrl($match[1]); + $body = DI::httpRequest()->fetch($match[1]); $doc = new DOMDocument(); @$doc->loadHTML($body); diff --git a/src/Core/Installer.php b/src/Core/Installer.php index af7c7aa496..28db93d292 100644 --- a/src/Core/Installer.php +++ b/src/Core/Installer.php @@ -27,7 +27,6 @@ use Friendica\Core\Config\Cache; use Friendica\Database\Database; use Friendica\Database\DBStructure; use Friendica\DI; -use Friendica\Network\HTTPRequest; use Friendica\Util\Images; use Friendica\Util\Strings; @@ -548,11 +547,11 @@ class Installer $help = ""; $error_msg = ""; if (function_exists('curl_init')) { - $fetchResult = DI::httpRequest()->fetchUrlFull($baseurl . "/install/testrewrite"); + $fetchResult = DI::httpRequest()->fetchFull($baseurl . "/install/testrewrite"); $url = Strings::normaliseLink($baseurl . "/install/testrewrite"); if ($fetchResult->getReturnCode() != 204) { - $fetchResult = DI::httpRequest()->fetchUrlFull($url); + $fetchResult = DI::httpRequest()->fetchFull($url); } if ($fetchResult->getReturnCode() != 204) { diff --git a/src/Core/Search.php b/src/Core/Search.php index c5c6ca08c2..577b11266e 100644 --- a/src/Core/Search.php +++ b/src/Core/Search.php @@ -123,7 +123,7 @@ class Search $searchUrl .= '&page=' . $page; } - $resultJson = DI::httpRequest()->fetchUrl($searchUrl, false, 0, 'application/json'); + $resultJson = DI::httpRequest()->fetch($searchUrl, false, 0, 'application/json'); $results = json_decode($resultJson, true); diff --git a/src/Core/Worker.php b/src/Core/Worker.php index 83a24c38fe..937dd0a565 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -996,7 +996,7 @@ class Worker } $url = DI::baseUrl() . '/worker'; - DI::httpRequest()->fetchUrl($url, false, 1); + DI::httpRequest()->fetch($url, false, 1); } /** diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 109f5d54b3..ab0a4fdd84 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -536,7 +536,7 @@ class GContact $done[] = DI::baseUrl() . '/poco'; if (strlen(DI::config()->get('system', 'directory'))) { - $x = DI::httpRequest()->fetchUrl(Search::getGlobalDirectory() . '/pubsites'); + $x = DI::httpRequest()->fetch(Search::getGlobalDirectory() . '/pubsites'); if (!empty($x)) { $j = json_decode($x); if (!empty($j->entries)) { diff --git a/src/Model/GServer.php b/src/Model/GServer.php index ae4332511d..7643c9590e 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1634,7 +1634,7 @@ class GServer $protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus']; foreach ($protocols as $protocol) { $query = '{nodes(protocol:"' . $protocol . '"){host}}'; - $curlResult = DI::httpRequest()->fetchUrl('https://the-federation.info/graphql?query=' . urlencode($query)); + $curlResult = DI::httpRequest()->fetch('https://the-federation.info/graphql?query=' . urlencode($query)); if (!empty($curlResult)) { $data = json_decode($curlResult, true); if (!empty($data['data']['nodes'])) { diff --git a/src/Module/Debug/Feed.php b/src/Module/Debug/Feed.php index deeb8d7ec3..1da0457c45 100644 --- a/src/Module/Debug/Feed.php +++ b/src/Module/Debug/Feed.php @@ -48,7 +48,7 @@ class Feed extends BaseModule $contact = Model\Contact::getByURLForUser($url, local_user(), false); - $xml = DI::httpRequest()->fetchUrl($contact['poll']); + $xml = DI::httpRequest()->fetch($contact['poll']); $import_result = Protocol\Feed::import($xml); diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index f9279fa602..c751406c1b 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -460,9 +460,9 @@ class HTTPRequest * @return string The fetched content * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public function fetchUrl(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) + public function fetch(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) { - $ret = $this->fetchUrlFull($url, $binary, $timeout, $accept_content, $cookiejar, $redirects); + $ret = $this->fetchFull($url, $binary, $timeout, $accept_content, $cookiejar, $redirects); return $ret->getBody(); } @@ -484,7 +484,7 @@ class HTTPRequest * @return CurlResult With all relevant information, 'body' contains the actual fetched content. * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public function fetchUrlFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) + public function fetchFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) { return $this->get( $url, diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php index 5e1f096773..98a315ce21 100644 --- a/src/Protocol/Diaspora.php +++ b/src/Protocol/Diaspora.php @@ -1379,7 +1379,7 @@ class Diaspora Logger::log("Fetch post from ".$source_url, Logger::DEBUG); - $envelope = DI::httpRequest()->fetchUrl($source_url); + $envelope = DI::httpRequest()->fetch($source_url); if ($envelope) { Logger::log("Envelope was fetched.", Logger::DEBUG); $x = self::verifyMagicEnvelope($envelope); diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 5216e39370..cfc140d66d 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -102,7 +102,7 @@ class PortableContact Logger::log('load: ' . $url, Logger::DEBUG); - $fetchresult = DI::httpRequest()->fetchUrlFull($url); + $fetchresult = DI::httpRequest()->fetchFull($url); $s = $fetchresult->getBody(); Logger::log('load: returns ' . $s, Logger::DATA); diff --git a/src/Util/Images.php b/src/Util/Images.php index 2d161a5c43..ef171873f2 100644 --- a/src/Util/Images.php +++ b/src/Util/Images.php @@ -184,7 +184,7 @@ class Images return $data; } - $img_str = DI::httpRequest()->fetchUrl($url, true, 4); + $img_str = DI::httpRequest()->fetch($url, true, 4); if (!$img_str) { return []; diff --git a/src/Worker/CheckVersion.php b/src/Worker/CheckVersion.php index 260d6b16f2..be325461b0 100644 --- a/src/Worker/CheckVersion.php +++ b/src/Worker/CheckVersion.php @@ -54,7 +54,7 @@ class CheckVersion Logger::log("Checking VERSION from: ".$checked_url, Logger::DEBUG); // fetch the VERSION file - $gitversion = DBA::escape(trim(DI::httpRequest()->fetchUrl($checked_url))); + $gitversion = DBA::escape(trim(DI::httpRequest()->fetch($checked_url))); Logger::log("Upstream VERSION is: ".$gitversion, Logger::DEBUG); DI::config()->set('system', 'git_friendica_version', $gitversion); diff --git a/src/Worker/CronJobs.php b/src/Worker/CronJobs.php index 1e45058569..4f988b6e14 100644 --- a/src/Worker/CronJobs.php +++ b/src/Worker/CronJobs.php @@ -60,7 +60,7 @@ class CronJobs // Now trying to register $url = 'http://the-federation.info/register/' . DI::baseUrl()->getHostname(); Logger::debug('Check registering url', ['url' => $url]); - $ret = DI::httpRequest()->fetchUrl($url); + $ret = DI::httpRequest()->fetch($url); Logger::debug('Check registering answer', ['answer' => $ret]); Logger::info('cron_end'); break; diff --git a/src/Worker/Directory.php b/src/Worker/Directory.php index 2cab09f339..d71e593dc5 100644 --- a/src/Worker/Directory.php +++ b/src/Worker/Directory.php @@ -53,7 +53,7 @@ class Directory Logger::log('Updating directory: ' . $arr['url'], Logger::DEBUG); if (strlen($arr['url'])) { - DI::httpRequest()->fetchUrl($dir . '?url=' . bin2hex($arr['url'])); + DI::httpRequest()->fetch($dir . '?url=' . bin2hex($arr['url'])); } return; diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php index c3c344d93c..546c369b2c 100644 --- a/src/Worker/SearchDirectory.php +++ b/src/Worker/SearchDirectory.php @@ -51,7 +51,7 @@ class SearchDirectory } } - $x = DI::httpRequest()->fetchUrl(Search::getGlobalDirectory() . '/lsearch?p=1&n=500&search=' . urlencode($search)); + $x = DI::httpRequest()->fetch(Search::getGlobalDirectory() . '/lsearch?p=1&n=500&search=' . urlencode($search)); $j = json_decode($x); if (!empty($j->results)) { From 0a421064a56492e15b5ed647ee19c26960d046f3 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 22:56:16 +0100 Subject: [PATCH 13/17] Introduce interface "IHTTPRequest" (rely on abstractions, not on concrete implementations) --- src/DI.php | 4 +- src/Network/HTTPRequest.php | 75 +++------------------ src/Network/IHTTPRequest.php | 119 +++++++++++++++++++++++++++++++++ static/dependencies.config.php | 4 ++ 4 files changed, 136 insertions(+), 66 deletions(-) create mode 100644 src/Network/IHTTPRequest.php diff --git a/src/DI.php b/src/DI.php index 5986ca9615..39e892adcb 100644 --- a/src/DI.php +++ b/src/DI.php @@ -328,11 +328,11 @@ abstract class DI // /** - * @return Network\HTTPRequest + * @return Network\IHTTPRequest */ public static function httpRequest() { - return self::$dice->create(Network\HTTPRequest::class); + return self::$dice->create(Network\IHTTPRequest::class); } // diff --git a/src/Network/HTTPRequest.php b/src/Network/HTTPRequest.php index c751406c1b..839586880c 100644 --- a/src/Network/HTTPRequest.php +++ b/src/Network/HTTPRequest.php @@ -33,7 +33,7 @@ use Psr\Log\LoggerInterface; /** * Performs HTTP requests to a given URL */ -class HTTPRequest +class HTTPRequest implements IHTTPRequest { /** @var LoggerInterface */ private $logger; @@ -53,22 +53,10 @@ class HTTPRequest } /** - * fetches an URL. + * {@inheritDoc} * - * @param string $url URL to fetch - * @param bool $binary default false - * TRUE if asked to return binary results (file download) - * @param array $opts (optional parameters) assoziative array with: - * 'accept_content' => supply Accept: header with 'accept_content' as the value - * 'timeout' => int Timeout in seconds, default system config value or 60 seconds - * 'http_auth' => username:password - * 'novalidate' => do not validate SSL certs, default is to validate using our CA list - * 'nobody' => only return the header - * 'cookiejar' => path to cookie jar file - * 'header' => header array - * @param int $redirects The recursion counter for internal use - default 0 + * @param int $redirects The recursion counter for internal use - default 0 * - * @return CurlResult * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ public function get(string $url, bool $binary = false, array $opts = [], int &$redirects = 0) @@ -217,15 +205,10 @@ class HTTPRequest } /** - * Send POST request to $url + * {@inheritDoc} * - * @param string $url URL to post - * @param mixed $params array of POST variables - * @param array $headers HTTP headers - * @param int $redirects Recursion counter for internal use - default = 0 - * @param int $timeout The timeout in seconds, default system config value or 60 seconds + * @param int $redirects The recursion counter for internal use - default 0 * - * @return CurlResult The content * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ public function post(string $url, $params, array $headers = [], int $timeout = 0, int &$redirects = 0) @@ -325,20 +308,7 @@ class HTTPRequest } /** - * Returns the original URL of the provided URL - * - * This function strips tracking query params and follows redirections, either - * through HTTP code or meta refresh tags. Stops after 10 redirections. - * - * @todo Remove the $fetchbody parameter that generates an extraneous HEAD request - * - * @see ParseUrl::getSiteinfo - * - * @param string $url A user-submitted URL - * @param int $depth The current redirection recursion level (internal) - * @param bool $fetchbody Wether to fetch the body or not after the HEAD requests - * @return string A canonical URL - * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * {@inheritDoc} */ public function finalUrl(string $url, int $depth = 1, bool $fetchbody = false) { @@ -443,21 +413,10 @@ class HTTPRequest } /** - * Curl wrapper + * {@inheritDoc} * - * If binary flag is true, return binary results. - * Set the cookiejar argument to a string (e.g. "/tmp/friendica-cookies.txt") - * to preserve cookies from one request to the next. + * @param int $redirects The recursion counter for internal use - default 0 * - * @param string $url URL to fetch - * @param bool $binary default false - * TRUE if asked to return binary results (file download) - * @param int $timeout Timeout in seconds, default system config value or 60 seconds - * @param string $accept_content supply Accept: header with 'accept_content' as the value - * @param string $cookiejar Path to cookie jar file - * @param int $redirects The recursion counter for internal use - default 0 - * - * @return string The fetched content * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ public function fetch(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) @@ -468,20 +427,10 @@ class HTTPRequest } /** - * Curl wrapper with array of return values. + * {@inheritDoc} * - * Inner workings and parameters are the same as @ref fetchUrl but returns an array with - * all the information collected during the fetch. + * @param int $redirects The recursion counter for internal use - default 0 * - * @param string $url URL to fetch - * @param bool $binary default false - * TRUE if asked to return binary results (file download) - * @param int $timeout Timeout in seconds, default system config value or 60 seconds - * @param string $accept_content supply Accept: header with 'accept_content' as the value - * @param string $cookiejar Path to cookie jar file - * @param int $redirects The recursion counter for internal use - default 0 - * - * @return CurlResult With all relevant information, 'body' contains the actual fetched content. * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ public function fetchFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = '', int &$redirects = 0) @@ -499,9 +448,7 @@ class HTTPRequest } /** - * Returns the current UserAgent as a String - * - * @return string the UserAgent as a String + * {@inheritDoc} */ public function getUserAgent() { diff --git a/src/Network/IHTTPRequest.php b/src/Network/IHTTPRequest.php new file mode 100644 index 0000000000..3ebcc5dc1b --- /dev/null +++ b/src/Network/IHTTPRequest.php @@ -0,0 +1,119 @@ +. + * + */ + +namespace Friendica\Network; + +/** + * Interface for calling HTTP requests and returning their responses + */ +interface IHTTPRequest +{ + /** + * Fetches the content of an URL + * + * If binary flag is true, return binary results. + * Set the cookiejar argument to a string (e.g. "/tmp/friendica-cookies.txt") + * to preserve cookies from one request to the next. + * + * @param string $url URL to fetch + * @param bool $binary default false + * TRUE if asked to return binary results (file download) + * @param int $timeout Timeout in seconds, default system config value or 60 seconds + * @param string $accept_content supply Accept: header with 'accept_content' as the value + * @param string $cookiejar Path to cookie jar file + * + * @return string The fetched content + */ + public function fetch(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = ''); + + /** + * Fetches the whole response of an URL. + * + * Inner workings and parameters are the same as @ref fetchUrl but returns an array with + * all the information collected during the fetch. + * + * @param string $url URL to fetch + * @param bool $binary default false + * TRUE if asked to return binary results (file download) + * @param int $timeout Timeout in seconds, default system config value or 60 seconds + * @param string $accept_content supply Accept: header with 'accept_content' as the value + * @param string $cookiejar Path to cookie jar file + * + * @return CurlResult With all relevant information, 'body' contains the actual fetched content. + */ + public function fetchFull(string $url, bool $binary = false, int $timeout = 0, string $accept_content = '', string $cookiejar = ''); + + /** + * Send a GET to an URL. + * + * @param string $url URL to fetch + * @param bool $binary default false + * TRUE if asked to return binary results (file download) + * @param array $opts (optional parameters) assoziative array with: + * 'accept_content' => supply Accept: header with 'accept_content' as the value + * 'timeout' => int Timeout in seconds, default system config value or 60 seconds + * 'http_auth' => username:password + * 'novalidate' => do not validate SSL certs, default is to validate using our CA list + * 'nobody' => only return the header + * 'cookiejar' => path to cookie jar file + * 'header' => header array + * + * @return CurlResult + */ + public function get(string $url, bool $binary = false, array $opts = []); + + /** + * Send POST request to an URL + * + * @param string $url URL to post + * @param mixed $params array of POST variables + * @param array $headers HTTP headers + * @param int $timeout The timeout in seconds, default system config value or 60 seconds + * + * @return CurlResult The content + */ + public function post(string $url, $params, array $headers = [], int $timeout = 0); + + /** + * Returns the original URL of the provided URL + * + * This function strips tracking query params and follows redirections, either + * through HTTP code or meta refresh tags. Stops after 10 redirections. + * + * @param string $url A user-submitted URL + * @param int $depth The current redirection recursion level (internal) + * @param bool $fetchbody Wether to fetch the body or not after the HEAD requests + * + * @return string A canonical URL + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @see ParseUrl::getSiteinfo + * + * @todo Remove the $fetchbody parameter that generates an extraneous HEAD request + */ + public function finalUrl(string $url, int $depth = 1, bool $fetchbody = false); + + /** + * Returns the current UserAgent as a String + * + * @return string the UserAgent as a String + */ + public function getUserAgent(); +} diff --git a/static/dependencies.config.php b/static/dependencies.config.php index 84344a60e2..fe8a8caee0 100644 --- a/static/dependencies.config.php +++ b/static/dependencies.config.php @@ -46,6 +46,7 @@ use Friendica\Database\Database; use Friendica\Factory; use Friendica\Model\Storage\IStorage; use Friendica\Model\User\Cookie; +use Friendica\Network; use Friendica\Util; use Psr\Log\LoggerInterface; @@ -219,4 +220,7 @@ return [ ['getBackend', [], Dice::CHAIN_CALL], ], ], + Network\IHTTPRequest::class => [ + 'instanceOf' => Network\HTTPRequest::class, + ] ]; From 1124090dbc5c9ff54b0f683424f16fb3123505d5 Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 23:28:41 +0100 Subject: [PATCH 14/17] Cleanup dependencies --- src/Core/Protocol.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Core/Protocol.php b/src/Core/Protocol.php index e6133240c4..7b9789752e 100644 --- a/src/Core/Protocol.php +++ b/src/Core/Protocol.php @@ -21,6 +21,8 @@ namespace Friendica\Core; +use Friendica\DI; + /** * Manage compatibility with federated networks */ @@ -89,7 +91,6 @@ class Protocol * @param string $profile_url * @param array $matches preg_match return array: [0] => Full match [1] => hostname [2] => username * @return string - * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ public static function matchByProfileUrl($profile_url, &$matches = []) { @@ -121,7 +122,7 @@ class Protocol if (preg_match('=https?://(.*)/user/(.*)=ism', $profile_url, $matches)) { $statusnet_host = $matches[1]; $statusnet_user = $matches[2]; - $UserData = DI::httpRequest()->fetchUrl('http://' . $statusnet_host . '/api/users/show.json?user_id=' . $statusnet_user); + $UserData = DI::httpRequest()->fetch('http://' . $statusnet_host . '/api/users/show.json?user_id=' . $statusnet_user); $user = json_decode($UserData); if ($user) { $matches[2] = $user->screen_name; From c51128ad201b92e070b12b24a346d0e5266ca5ad Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Wed, 4 Mar 2020 23:36:46 +0100 Subject: [PATCH 15/17] Fix tests --- tests/src/Core/InstallerTest.php | 38 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/tests/src/Core/InstallerTest.php b/tests/src/Core/InstallerTest.php index f512bf17b9..6ec2f1bc7c 100644 --- a/tests/src/Core/InstallerTest.php +++ b/tests/src/Core/InstallerTest.php @@ -26,9 +26,9 @@ use Dice\Dice; use Friendica\Core\Config\Cache; use Friendica\DI; use Friendica\Network\CurlResult; +use Friendica\Network\IHTTPRequest; use Friendica\Test\MockedTest; use Friendica\Test\Util\VFSTrait; -use Friendica\Util\Network; use Mockery\MockInterface; class InstallerTest extends MockedTest @@ -39,6 +39,10 @@ class InstallerTest extends MockedTest * @var \Friendica\Core\L10n|MockInterface */ private $l10nMock; + /** + * @var Dice|MockInterface + */ + private $dice; public function setUp() { @@ -49,14 +53,14 @@ class InstallerTest extends MockedTest $this->l10nMock = \Mockery::mock(\Friendica\Core\L10n::class); /** @var Dice|MockInterface $dice */ - $dice = \Mockery::mock(Dice::class)->makePartial(); - $dice = $dice->addRules(include __DIR__ . '/../../../static/dependencies.config.php'); + $this->dice = \Mockery::mock(Dice::class)->makePartial(); + $this->dice = $this->dice->addRules(include __DIR__ . '/../../../static/dependencies.config.php'); - $dice->shouldReceive('create') + $this->dice->shouldReceive('create') ->with(\Friendica\Core\L10n::class) ->andReturn($this->l10nMock); - DI::init($dice); + DI::init($this->dice); } private function mockL10nT(string $text, $times = null) @@ -305,16 +309,22 @@ class InstallerTest extends MockedTest ->andReturn('test Error'); // Mocking the CURL Request - $networkMock = \Mockery::mock('alias:' . Network::class); + $networkMock = \Mockery::mock(IHTTPRequest::class); $networkMock - ->shouldReceive('fetchUrlFull') + ->shouldReceive('fetchFull') ->with('https://test/install/testrewrite') ->andReturn($curlResult); $networkMock - ->shouldReceive('fetchUrlFull') + ->shouldReceive('fetchFull') ->with('http://test/install/testrewrite') ->andReturn($curlResult); + $this->dice->shouldReceive('create') + ->with(IHTTPRequest::class) + ->andReturn($networkMock); + + DI::init($this->dice); + // Mocking that we can use CURL $this->setFunctions(['curl_init' => true]); @@ -346,16 +356,22 @@ class InstallerTest extends MockedTest ->andReturn('204'); // Mocking the CURL Request - $networkMock = \Mockery::mock('alias:' . Network::class); + $networkMock = \Mockery::mock(IHTTPRequest::class); $networkMock - ->shouldReceive('fetchUrlFull') + ->shouldReceive('fetchFull') ->with('https://test/install/testrewrite') ->andReturn($curlResultF); $networkMock - ->shouldReceive('fetchUrlFull') + ->shouldReceive('fetchFull') ->with('http://test/install/testrewrite') ->andReturn($curlResultW); + $this->dice->shouldReceive('create') + ->with(IHTTPRequest::class) + ->andReturn($networkMock); + + DI::init($this->dice); + // Mocking that we can use CURL $this->setFunctions(['curl_init' => true]); From 1998b7811ba7fc03367eb335f8f0e6b2f9f17e0c Mon Sep 17 00:00:00 2001 From: nupplaPhil Date: Sat, 7 Mar 2020 13:39:09 +0100 Subject: [PATCH 16/17] Fix fatal execution path for found Network::get() parameter execption --- src/Model/Item.php | 8 ++++---- src/Protocol/ActivityPub.php | 2 +- src/Protocol/ActivityPub/Processor.php | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Model/Item.php b/src/Model/Item.php index e31097f53c..46f2fb09a7 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -3639,7 +3639,7 @@ class Item * * @return integer item id */ - public static function fetchByLink($uri, $uid = 0) + public static function fetchByLink(string $uri, int $uid = 0) { $item_id = self::searchByLink($uri, $uid); if (!empty($item_id)) { @@ -3692,7 +3692,7 @@ class Item * * @return array item array with data from the original item */ - public static function addShareDataFromOriginal($item) + public static function addShareDataFromOriginal(array $item) { $shared = self::getShareArray($item); if (empty($shared)) { @@ -3714,9 +3714,9 @@ class Item } // Otherwhise try to find (and possibly fetch) the item via the link. This should work for Diaspora and ActivityPub posts - $id = self::fetchByLink($shared['link'], $uid); + $id = self::fetchByLink($shared['link'] ?? '', $uid); if (empty($id)) { - Logger::info('Original item not found', ['url' => $shared['link'], 'callstack' => System::callstack()]); + Logger::info('Original item not found', ['url' => $shared['link'] ?? '', 'callstack' => System::callstack()]); return $item; } diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index 6b29eabce5..3c4f4f2e67 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -87,7 +87,7 @@ class ActivityPub * @return array * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function fetchContent($url, $uid = 0) + public static function fetchContent(string $url, int $uid = 0) { if (!empty($uid)) { return HTTPSignature::fetch($url, $uid); diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php index e4cef17045..04d8a7467f 100644 --- a/src/Protocol/ActivityPub/Processor.php +++ b/src/Protocol/ActivityPub/Processor.php @@ -681,7 +681,7 @@ class Processor * @return string fetched message URL * @throws \Friendica\Network\HTTPException\InternalServerErrorException */ - public static function fetchMissingActivity($url, $child = []) + public static function fetchMissingActivity(string $url, array $child = []) { if (!empty($child['receiver'])) { $uid = ActivityPub\Receiver::getFirstUserFromReceivers($child['receiver']); From ec3ec3b78a2a431b82ff16136dddd5e6900c3e1e Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Tue, 21 Jul 2020 21:39:59 -0400 Subject: [PATCH 17/17] Replace remaining occurrences of Network::curl --- src/Model/GContact.php | 3 ++- src/Model/GServer.php | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Model/GContact.php b/src/Model/GContact.php index ab0a4fdd84..41ca763fc6 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -35,6 +35,7 @@ use Friendica\Network\Probe; use Friendica\Protocol\ActivityPub; use Friendica\Protocol\PortableContact; use Friendica\Util\DateTimeFormat; +use Friendica\Util\Network; use Friendica\Util\Strings; /** @@ -1377,7 +1378,7 @@ class GContact return; } - $curlResult = Network::curl($data['poco']); + $curlResult = DI::httpRequest()->get($data['poco']); if (!$curlResult->isSuccess()) { return; } diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 7643c9590e..0f47146eb7 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -359,7 +359,7 @@ class GServer // When the base path doesn't seem to contain a social network we try the complete path. // Most detectable system have to be installed in the root directory. // We checked the base to avoid false positives. - $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout]); + $curlResult = DI::httpRequest()->get($url, false, ['timeout' => $xrd_timeout]); if ($curlResult->isSuccess()) { $urldata = self::analyseRootHeader($curlResult, $serverdata); $urldata = self::analyseRootBody($curlResult, $urldata, $url);