From 3263a0b98e0ca77dede015a6c107bddbb4e39b2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20H=C3=A4der?= Date: Tue, 16 Jan 2024 19:52:06 +0100 Subject: [PATCH 1/6] Make hard-coded max response configurable: - see discussion started by @schmaker@schmaker.eu at https://schmaker.eu/display/c83e3896-1265-a3d6-0ab5-a78119129626 - this allows servers with lower RAM amount to still run without OOMs (or much lesser) --- src/Network/HTTPClient/Factory/HttpClient.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Network/HTTPClient/Factory/HttpClient.php b/src/Network/HTTPClient/Factory/HttpClient.php index 26ddde62c0..fca5d53130 100644 --- a/src/Network/HTTPClient/Factory/HttpClient.php +++ b/src/Network/HTTPClient/Factory/HttpClient.php @@ -123,7 +123,7 @@ class HttpClient extends BaseFactory $resolver->setMaxRedirects(10); $resolver->setRequestTimeout(10); // if the file is too large then exit - $resolver->setMaxResponseDataSize(1000000); + $resolver->setMaxResponseDataSize($this->config->get('system', 'max_response_data_size', 1000000)); // Designate a temporary file that will store cookies during the session. // Some websites test the browser for cookie support, so this enhances results. $resolver->setCookieJar(System::getTempPath() .'/resolver-cookie-' . Strings::getRandomName(10)); From 931c73d3c1eb85279efc79f141dbae58df802426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20H=C3=A4der?= Date: Tue, 16 Jan 2024 19:58:43 +0100 Subject: [PATCH 2/6] Continued: - added max_response_data_size to default configuration with hard-coded value from code --- static/defaults.config.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/static/defaults.config.php b/static/defaults.config.php index 819b0ad85f..2552e7d35f 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -613,6 +613,10 @@ return [ // xrd_timeout (Integer) // Timeout in seconds for fetching the XRD links and other requests with an expected shorter timeout 'xrd_timeout' => 20, + + // max_response_data_size (Integer) + // Maximum allowed response data size in Bytes, default is hard-coded value from code + 'max_response_data_size' => 1000000, ], 'proxy' => [ // forwarded_for_headers (String) From 075915e3b74686a67a7d4698ec60d54478c497e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20H=C3=A4der?= Date: Tue, 16 Jan 2024 20:11:59 +0100 Subject: [PATCH 3/6] Continued: - moved to new configuration category 'http_client' (thanks to @MrPetovan) --- src/Network/HTTPClient/Factory/HttpClient.php | 2 +- static/defaults.config.php | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Network/HTTPClient/Factory/HttpClient.php b/src/Network/HTTPClient/Factory/HttpClient.php index fca5d53130..eaf6003ccb 100644 --- a/src/Network/HTTPClient/Factory/HttpClient.php +++ b/src/Network/HTTPClient/Factory/HttpClient.php @@ -123,7 +123,7 @@ class HttpClient extends BaseFactory $resolver->setMaxRedirects(10); $resolver->setRequestTimeout(10); // if the file is too large then exit - $resolver->setMaxResponseDataSize($this->config->get('system', 'max_response_data_size', 1000000)); + $resolver->setMaxResponseDataSize($this->config->get('http_client', 'max_response_data_size', 1000000)); // Designate a temporary file that will store cookies during the session. // Some websites test the browser for cookie support, so this enhances results. $resolver->setCookieJar(System::getTempPath() .'/resolver-cookie-' . Strings::getRandomName(10)); diff --git a/static/defaults.config.php b/static/defaults.config.php index 2552e7d35f..a0d6157694 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -613,7 +613,8 @@ return [ // xrd_timeout (Integer) // Timeout in seconds for fetching the XRD links and other requests with an expected shorter timeout 'xrd_timeout' => 20, - + ], + 'http_client' => [ // max_response_data_size (Integer) // Maximum allowed response data size in Bytes, default is hard-coded value from code 'max_response_data_size' => 1000000, From 7d3723592bba4d2f105e2b73100283c8ad7ccd74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20H=C3=A4der?= Date: Tue, 16 Jan 2024 20:32:17 +0100 Subject: [PATCH 4/6] Renamed 'http_client' -> 'performance' --- src/Network/HTTPClient/Factory/HttpClient.php | 2 +- static/defaults.config.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Network/HTTPClient/Factory/HttpClient.php b/src/Network/HTTPClient/Factory/HttpClient.php index eaf6003ccb..e0de17c75a 100644 --- a/src/Network/HTTPClient/Factory/HttpClient.php +++ b/src/Network/HTTPClient/Factory/HttpClient.php @@ -123,7 +123,7 @@ class HttpClient extends BaseFactory $resolver->setMaxRedirects(10); $resolver->setRequestTimeout(10); // if the file is too large then exit - $resolver->setMaxResponseDataSize($this->config->get('http_client', 'max_response_data_size', 1000000)); + $resolver->setMaxResponseDataSize($this->config->get('performance', 'max_response_data_size', 1000000)); // Designate a temporary file that will store cookies during the session. // Some websites test the browser for cookie support, so this enhances results. $resolver->setCookieJar(System::getTempPath() .'/resolver-cookie-' . Strings::getRandomName(10)); diff --git a/static/defaults.config.php b/static/defaults.config.php index a0d6157694..2ce0280873 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -614,7 +614,7 @@ return [ // Timeout in seconds for fetching the XRD links and other requests with an expected shorter timeout 'xrd_timeout' => 20, ], - 'http_client' => [ + 'performance' => [ // max_response_data_size (Integer) // Maximum allowed response data size in Bytes, default is hard-coded value from code 'max_response_data_size' => 1000000, From a7bdcc121664a324e748151be6e336a2dda031d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20H=C3=A4der?= Date: Wed, 17 Jan 2024 01:11:06 +0100 Subject: [PATCH 5/6] Better description (@MrPetovan) --- static/defaults.config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/defaults.config.php b/static/defaults.config.php index 2ce0280873..dc21a8ce0a 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -616,7 +616,7 @@ return [ ], 'performance' => [ // max_response_data_size (Integer) - // Maximum allowed response data size in Bytes, default is hard-coded value from code + // Maximum allowed outgoing HTTP request response data size in Bytes. Does not affect incoming requests to this node. 'max_response_data_size' => 1000000, ], 'proxy' => [ From 550b4ce4204f64a49b8ad04a111ee524dca464c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20H=C3=A4der?= Date: Wed, 17 Jan 2024 01:50:31 +0100 Subject: [PATCH 6/6] Added note --- static/defaults.config.php | 1 + 1 file changed, 1 insertion(+) diff --git a/static/defaults.config.php b/static/defaults.config.php index dc21a8ce0a..7439c7fdf6 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -617,6 +617,7 @@ return [ 'performance' => [ // max_response_data_size (Integer) // Maximum allowed outgoing HTTP request response data size in Bytes. Does not affect incoming requests to this node. + // Warning: Lowering this value can help with some PHP memory exhaustion issues, but can also partially break some federation features e.g. large posts may not be fetched or received from remote servers. 'max_response_data_size' => 1000000, ], 'proxy' => [