Display structured logs in admin

Tries to parse log lines and to display info in a table.
Additional JSON data is parsed and displayed clicking on a row.

File reading and line parsing is handled in iterators, to avoid to keep
too much data in memory.
Search and filter should be trivial to add.
Log file is read backward to display log events newest first.
A "tail" functionality should be easy to implement.

src/Model/Log/ParsedLogIterator.php

@@ -0,0 +1,81 @@
+<?php
+/**
+ * @copyright Copyright (C) 2021, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+namespace Friendica\Model\Log;
+
+use \Friendica\Util\ReversedFileReader;
+use \Friendica\Object\Log\ParsedLog;
+
+
+/**
+ * An iterator which returns `\Friendica\Objec\Log\ParsedLog` instances
+ *
+ * Uses `\Friendica\Util\ReversedFileReader` to fetch log lines
+ * from newest to oldest
+ */
+class ParsedLogIterator implements \Iterator
+{
+	public function __construct(string $filename, int $limit=0)
+	{
+		$this->reader = new ReversedFileReader($filename);
+		$this->_value = null;
+		$this->_limit = $limit;
+	}
+
+	public function next()
+	{
+		$this->reader->next();
+		if ($this->_limit > 0 && $this->reader->key() > $this->_limit) {
+			$this->_value = null;
+			return;
+		}
+		if ($this->reader->valid()) {
+			$line = $this->reader->current();
+			$this->_value = new ParsedLog($this->reader->key(), $line);
+		} else {
+			$this->_value = null;
+		}
+	}
+
+
+	public function rewind()
+	{
+		$this->_value = null;
+		$this->reader->rewind();
+		$this->next();
+	}
+
+	public function key()
+	{
+		return $this->reader->key();
+	}
+
+	public function current()
+	{
+		return $this->_value;
+	}
+
+	public function valid()
+	{
+		return ! is_null($this->_value);
+	}
+
+}
+

src/Module/Admin/Logs/View.php

@@ -21,49 +21,42 @@
 
 namespace Friendica\Module\Admin\Logs;
 
-use Friendica\Core\Renderer;
 use Friendica\DI;
+use Friendica\Core\Renderer;
+use Friendica\Core\Theme;
 use Friendica\Module\BaseAdmin;
-use Friendica\Util\Strings;
+use Friendica\Model\Log\ParsedLogIterator;
 
 class View extends BaseAdmin
 {
+	const LIMIT = 500;
+
 	public static function content(array $parameters = [])
 	{
 		parent::content($parameters);
 
 		$t = Renderer::getMarkupTemplate('admin/logs/view.tpl');
+		DI::page()->registerFooterScript(Theme::getPathForFile('js/module/admin/logs/view.js'));
+
 		$f = DI::config()->get('system', 'logfile');
-		$data = '';
+		$data = null;
+		$error = null;
+
 
 		if (!file_exists($f)) {
-			$data = DI::l10n()->t('Error trying to open <strong>%1$s</strong> log file.\r\n<br/>Check to see if file %1$s exist and is readable.', $f);
+			$error = DI::l10n()->t('Error trying to open <strong>%1$s</strong> log file.\r\n<br/>Check to see if file %1$s exist and is readable.', $f);
 		} else {
-			$fp = fopen($f, 'r');
+			try {
-			if (!$fp) {
+				$data = new ParsedLogIterator($f, self::LIMIT);
-				$data = DI::l10n()->t('Couldn\'t open <strong>%1$s</strong> log file.\r\n<br/>Check to see if file %1$s is readable.', $f);
+			} catch (Exception $e) {
-			} else {
+				$error = DI::l10n()->t('Couldn\'t open <strong>%1$s</strong> log file.\r\n<br/>Check to see if file %1$s is readable.', $f);
-				$fstat = fstat($fp);
-				$size = $fstat['size'];
-				if ($size != 0) {
-					if ($size > 5000000 || $size < 0) {
-						$size = 5000000;
-					}
-					$seek = fseek($fp, 0 - $size, SEEK_END);
-					if ($seek === 0) {
-						$data = Strings::escapeHtml(fread($fp, $size));
-						while (!feof($fp)) {
-							$data .= Strings::escapeHtml(fread($fp, 4096));
-						}
-					}
-				}
-				fclose($fp);
 			}
 		}
 		return Renderer::replaceMacros($t, [
 			'$title' => DI::l10n()->t('Administration'),
 			'$page' => DI::l10n()->t('View Logs'),
 			'$data' => $data,
+			'$error' => $error,
 			'$logname' => DI::config()->get('system', 'logfile')
 		]);
 	}

src/Object/Log/ParsedLog.php

@@ -0,0 +1,114 @@
+<?php
+/**
+ * @copyright Copyright (C) 2021, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+namespace Friendica\Object\Log;
+
+/**
+ * Parse a log line and offer some utility methods
+ */
+class ParsedLog
+{
+	const REGEXP = '/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}[^ ]*) (\w+) \[(\w*)\]: (.*)/';
+
+	public $id = 0;
+	public $date = null;
+	public $context = null;
+	public $level = null;
+	public $message = null;
+	public $data = null;
+	public $source = null;
+
+	/**
+	 * @param string $logline Source log line to parse
+	 */
+	public function __construct(int $id, string $logline)
+	{
+		$this->id = $id;
+		$this->parse($logline);
+		$this->stop = false;
+	}
+
+	private function parse($logline)
+	{
+		list($logline, $jsonsource) = explode(' - ', $logline);
+		$jsondata = null;
+		if (strpos($logline, '{"') > 0) {
+			list($logline, $jsondata) = explode('{"', $logline, 2);
+			$jsondata = '{"' . $jsondata;
+		}
+		preg_match(self::REGEXP, $logline, $matches);
+		$this->date = $matches[1];
+		$this->context = $matches[2];
+		$this->level = $matches[3];
+		$this->message = $matches[4];
+		$this->data = $jsondata;
+		$this->source = $jsonsource;
+		$this->try_fix_json('data');
+	}
+
+	/**
+	 * In log boundary between message and json data is not specified.
+	 * If message  contains '{' the parser thinks there starts the json data.
+	 * This method try to parse the found json and if it fails, search for next '{'
+	 * in json data and retry
+	 */
+	private function try_fix_json(string $key)
+	{
+		if (is_null($this->$key) || $this->$key == "") {
+			return;
+		}
+		try {
+			$d = json_decode($this->$key, true, 512, JSON_THROW_ON_ERROR);
+		} catch (\JsonException $e) {
+			// try to find next { in $str and move string before to 'message'
+
+			$pos = strpos($this->$key, '{', 1);
+
+			$this->message .= substr($this->$key, 0, $pos);
+			$this->$key = substr($this->key, $pos);
+			$this->try_fix_json($key);
+		}
+	}
+
+	/**
+	 * Return decoded `data` as array suitable for template
+	 *
+	 * @return array
+	 */
+	public function get_data() {
+		$data = json_decode($this->data, true);
+		if ($data) {
+			foreach($data as $k => $v) {
+				$v = print_r($v, true);
+				$data[$k] = $v;
+			}
+		}
+		return $data;
+	}
+
+	/**
+	 * Return decoded `source` as array suitable for template
+	 *
+	 * @return array
+	 */
+	public function get_source() {
+		return json_decode($this->source, true);
+	}
+}

src/Util/ReversedFileReader.php

@@ -0,0 +1,101 @@
+<?php
+/**
+ * @copyright Copyright (C) 2021, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Util;
+
+
+/**
+ * An iterator which returns lines from file in reversed order
+ *
+ * original code https://stackoverflow.com/a/10494801
+ */
+class ReversedFileReader implements \Iterator
+{
+    const BUFFER_SIZE = 4096;
+    const SEPARATOR = "\n";
+
+    public function __construct($filename)
+    {
+        $this->_fh = fopen($filename, 'r');
+		if (!$this->_fh) {
+			// this should use a custom exception.
+			throw \Exception("Unable to open $filename");
+		}
+        $this->_filesize = filesize($filename);
+        $this->_pos = -1;
+        $this->_buffer = null;
+        $this->_key = -1;
+        $this->_value = null;
+    }
+
+    public function _read($size)
+    {
+        $this->_pos -= $size;
+        fseek($this->_fh, $this->_pos);
+        return fread($this->_fh, $size);
+    }
+
+    public function _readline()
+    {
+        $buffer =& $this->_buffer;
+        while (true) {
+            if ($this->_pos == 0) {
+                return array_pop($buffer);
+            }
+            if (count($buffer) > 1) {
+                return array_pop($buffer);
+            }
+            $buffer = explode(self::SEPARATOR, $this->_read(self::BUFFER_SIZE) . $buffer[0]);
+        }
+    }
+
+    public function next()
+    {
+        ++$this->_key;
+        $this->_value = $this->_readline();
+    }
+
+    public function rewind()
+    {
+        if ($this->_filesize > 0) {
+            $this->_pos = $this->_filesize;
+            $this->_value = null;
+            $this->_key = -1;
+            $this->_buffer = explode(self::SEPARATOR, $this->_read($this->_filesize % self::BUFFER_SIZE ?: self::BUFFER_SIZE));
+            $this->next();
+        }
+    }
+
+    public function key()
+	{
+		return $this->_key;
+	}
+
+    public function current()
+	{
+		return $this->_value;
+	}
+
+    public function valid()
+	{
+		return ! is_null($this->_value);
+	}
+}

view/js/module/admin/logs/view.js

@@ -0,0 +1,7 @@
+function log_show_details(id) {
+	document
+		.querySelectorAll('[data-id="' + id + '"]')
+		.forEach(elm => {
+			elm.classList.toggle('hidden')
+		});
+}

view/templates/admin/logs/view.tpl

@@ -1,6 +1,46 @@
 <div id="adminpage">
 	<h1>{{$title}} - {{$page}}</h1>
-	
+
 	<h3>{{$logname}}</h3>
-	<div style="width:100%; height:400px; overflow: auto; "><pre>{{$data}}</pre></div>
+	{{if $error }}
+		<div id="admin-error-message-wrapper" class="alert alert-warning">
+			<p>{{$error nofilter}}</p>
+		</div>
+	{{else}}
+		<table>
+			<thead>
+				<tr>
+					<th>Date</th>
+					<th>Level</th>
+					<th>Context</th>
+					<th>Message</th>
+				</tr>
+			</thead>
+			<tbody>
+				{{foreach $data as $row}}
+				<tr id="ev-{{$row->id}}" onClick="log_show_details('ev-{{$row->id}}')">
+					<td>{{$row->date}}</td>
+					<td>{{$row->level}}</td>
+					<td>{{$row->context}}</td>
+					<td>{{$row->message}}</td>
+				</tr>
+				{{foreach $row->get_data() as $k=>$v}}
+					<tr class="hidden" data-id="ev-{{$row->id}}">
+						<th>{{$k}}</th>
+						<td colspan="3">
+							<pre>{{$v nofilter}}</pre>
+						</td>
+					</tr>
+					{{/foreach}}
+					<tr class="hidden" data-id="ev-{{$row->id}}"><th colspan="4">Source</th></tr>
+					{{foreach $row->get_source() as $k=>$v}}
+						<tr class="hidden" data-id="ev-{{$row->id}}">
+							<th>{{$k}}</th>
+							<td colspan="3">{{$v}}</td>
+						</tr>
+					{{/foreach}}
+				{{/foreach}}
+			</tbody>
+		</table>
+	{{/if}}
 </div>