From ca12f85c92c7f61d62c316483808608e78696d10 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 4 May 2018 08:11:49 -0400 Subject: [PATCH 01/11] Fix db error display --- include/dba.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/dba.php b/include/dba.php index 7f98fc80..cb0c09f9 100644 --- a/include/dba.php +++ b/include/dba.php @@ -40,8 +40,8 @@ class dba if ($this->debug) { $mesg = ''; - if ($this->db->mysqli->errno) { - $debug_text .= $this->db->mysqli->error . EOL; + if ($this->db->errno) { + $debug_text .= $this->db->error . EOL; } if ($result === false) { -- 2.43.5 From 5039d4202b05b3055f2f94137743f9dae673e074 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 4 May 2018 08:12:08 -0400 Subject: [PATCH 02/11] Fix Google+ support addon list --- mod/servers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/servers.php b/mod/servers.php index 3a7866fe..017a5a37 100644 --- a/mod/servers.php +++ b/mod/servers.php @@ -68,7 +68,7 @@ function servers_content(&$a) { 'HTTPS' => $site['ssl_state'] == 1, 'Twitter' => $hasAddon(array('buffer', 'twitter')), 'Facebook' => $hasAddon(array('buffer')), - 'Google+' => $hasAddon(array('buffer', 'gpluspost')), + 'Google+' => $hasAddon(array('buffer', 'fromgplus')), 'RSS/Atom' => true, //Built-in. 'Diaspora*' => $hasAddon(array('diaspora')), 'pump.io' => $hasAddon(array('pumpio')), -- 2.43.5 From 741e72ecc79c0d48dd3ac0957327331f7989e5f7 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 4 May 2018 08:15:21 -0400 Subject: [PATCH 03/11] [Composer] Add dependencies - Add pear/Net_Ping - Add asika/simple-console - Fix composer.json formatting --- composer.json | 30 ++++++--- composer.lock | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 10 deletions(-) create mode 100644 composer.lock diff --git a/composer.json b/composer.json index 9dd0b869..3d2b316d 100644 --- a/composer.json +++ b/composer.json @@ -1,11 +1,21 @@ { - "name": "friendica/dir", - "description": "The internet is our social network", - "license": "AGPL3", - "autoload": { - "psr-4": {"Friendica\\Directory\\": "src"} - }, - "require": { - "php": ">=5.3" - } -} \ No newline at end of file + "name": "friendica/dir", + "description": "The internet is our social network", + "license": "AGPL3", + "autoload": { + "psr-4": { + "Friendica\\Directory\\": "src" + } + }, + "require": { + "php": ">=5.3", + "pear/Net_Ping": "dev-master", + "asika/simple-console": "^1.0" + }, + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/pear/Net_Ping" + } + ] +} diff --git a/composer.lock b/composer.lock new file mode 100644 index 00000000..0c106ba5 --- /dev/null +++ b/composer.lock @@ -0,0 +1,171 @@ +{ + "_readme": [ + "This file locks the dependencies of your project to a known state", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "This file is @generated automatically" + ], + "content-hash": "f330bced2cb3855635512ca74f0fa779", + "packages": [ + { + "name": "asika/simple-console", + "version": "1.0.3", + "source": { + "type": "git", + "url": "https://github.com/asika32764/php-simple-console.git", + "reference": "0b624c1a999849dc6481a47182e58d593bf65068" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/asika32764/php-simple-console/zipball/0b624c1a999849dc6481a47182e58d593bf65068", + "reference": "0b624c1a999849dc6481a47182e58d593bf65068", + "shasum": "" + }, + "type": "library", + "autoload": { + "psr-4": { + "Asika\\SimpleConsole\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Simon Asika", + "email": "asika32764@gmail.com" + } + ], + "description": "One file console framework to help you write build scripts.", + "time": "2018-03-08T12:05:40+00:00" + }, + { + "name": "pear/net_ping", + "version": "dev-master", + "source": { + "type": "git", + "url": "https://github.com/pear/Net_Ping.git", + "reference": "d1f370b3a6072e5a493ed855804d98aadf818fa7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/pear/Net_Ping/zipball/d1f370b3a6072e5a493ed855804d98aadf818fa7", + "reference": "d1f370b3a6072e5a493ed855804d98aadf818fa7", + "shasum": "" + }, + "require": { + "pear/pear_exception": "*" + }, + "require-dev": { + "phpunit/phpunit": "*" + }, + "type": "library", + "autoload": { + "psr-0": { + "Net": "./" + } + }, + "include-path": [ + "./" + ], + "license": [ + "PHP License" + ], + "authors": [ + { + "email": "cconstantine@php.net", + "name": "Craig Constantine", + "role": "Lead" + }, + { + "email": "mj@php.net", + "name": "Martin Jansen", + "role": "Developer" + }, + { + "email": "jan@php.net", + "name": "Jan Lehnardt", + "role": "Developer" + }, + { + "email": "cox@php.net", + "name": "Thomas V.V.Cox", + "role": "Developer" + } + ], + "description": "More info available on: http://pear.php.net/package/Net_Ping", + "support": { + "issues": "http://pear.php.net/bugs/search.php?cmd=display&package_name[]=Net_Ping", + "source": "https://github.com/pear/Net_Ping" + }, + "time": "2014-02-20T19:24:17+00:00" + }, + { + "name": "pear/pear_exception", + "version": "v1.0.0", + "source": { + "type": "git", + "url": "https://github.com/pear/PEAR_Exception.git", + "reference": "8c18719fdae000b690e3912be401c76e406dd13b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/pear/PEAR_Exception/zipball/8c18719fdae000b690e3912be401c76e406dd13b", + "reference": "8c18719fdae000b690e3912be401c76e406dd13b", + "shasum": "" + }, + "require": { + "php": ">=4.4.0" + }, + "require-dev": { + "phpunit/phpunit": "*" + }, + "type": "class", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-0": { + "PEAR": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "include-path": [ + "." + ], + "license": [ + "BSD-2-Clause" + ], + "authors": [ + { + "name": "Helgi Thormar", + "email": "dufuz@php.net" + }, + { + "name": "Greg Beaver", + "email": "cellog@php.net" + } + ], + "description": "The PEAR Exception base class.", + "homepage": "https://github.com/pear/PEAR_Exception", + "keywords": [ + "exception" + ], + "time": "2015-02-10T20:07:52+00:00" + } + ], + "packages-dev": [], + "aliases": [], + "minimum-stability": "stable", + "stability-flags": { + "pear/net_ping": 20 + }, + "prefer-stable": false, + "prefer-lowest": false, + "platform": { + "php": ">=5.3" + }, + "platform-dev": [] +} -- 2.43.5 From 4da0d68bada69f8339a94d7ddc8d51a75e5a4d6f Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 4 May 2018 08:16:03 -0400 Subject: [PATCH 04/11] Add console CLI command - Fix App to be used in CLI mode --- bin/console | 10 ++++ bin/console.bat | 4 ++ bin/console.php | 6 +++ src/App.php | 17 ++++--- src/Core/Console.php | 106 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 bin/console create mode 100644 bin/console.bat create mode 100644 bin/console.php create mode 100644 src/Core/Console.php diff --git a/bin/console b/bin/console new file mode 100644 index 00000000..4d76bdc4 --- /dev/null +++ b/bin/console @@ -0,0 +1,10 @@ +#!/bin/bash + +dir=$(cd "${0%[/\\]*}" > /dev/null; pwd) + +if [[ -d /proc/cygdrive && $(which php) == $(readlink -n /proc/cygdrive)/* ]]; then + # We are in Cgywin using Windows php, so the path must be translated + dir=$(cygpath -m "$dir"); +fi + +php "${dir}/console.php" "$@" diff --git a/bin/console.bat b/bin/console.bat new file mode 100644 index 00000000..06c41a03 --- /dev/null +++ b/bin/console.bat @@ -0,0 +1,4 @@ +@echo OFF +:: in case DelayedExpansion is on and a path contains ! +setlocal DISABLEDELAYEDEXPANSION +php "%~dp0console.php" %* diff --git a/bin/console.php b/bin/console.php new file mode 100644 index 00000000..07f26bbf --- /dev/null +++ b/bin/console.php @@ -0,0 +1,6 @@ +#!/usr/bin/env php +execute(); diff --git a/src/App.php b/src/App.php index 8540892a..5ce9fbc6 100644 --- a/src/App.php +++ b/src/App.php @@ -34,17 +34,20 @@ class App $this->pager = array(); $this->scheme = ((isset($_SERVER['HTTPS']) && ($_SERVER['HTTPS'])) ? 'https' : 'http'); - $this->hostname = str_replace('www.', '', $_SERVER['SERVER_NAME']); + if (!empty($_SERVER['SERVER_NAME'])) { + $this->hostname = str_replace('www.', '', $_SERVER['SERVER_NAME']); + } set_include_path(get_include_path() - . PATH_SEPARATOR . "include/$this->hostname" . PATH_SEPARATOR . 'include' . PATH_SEPARATOR . '.'); - if (substr($_SERVER['QUERY_STRING'], 0, 2) == "q=") { - $_SERVER['QUERY_STRING'] = substr($_SERVER['QUERY_STRING'], 2); - } + if (!empty($_SERVER['QUERY_STRING'])) { + if (substr($_SERVER['QUERY_STRING'], 0, 2) == "q=") { + $_SERVER['QUERY_STRING'] = substr($_SERVER['QUERY_STRING'], 2); + } - $this->query_string = $_SERVER['QUERY_STRING']; + $this->query_string = $_SERVER['QUERY_STRING']; + } $q = isset($_GET['q']) ? $_GET['q'] : ''; $this->cmd = trim($q, '/'); @@ -120,4 +123,4 @@ class App '$baseurl' => $this->get_baseurl() )); } -} \ No newline at end of file +} diff --git a/src/Core/Console.php b/src/Core/Console.php new file mode 100644 index 00000000..82473009 --- /dev/null +++ b/src/Core/Console.php @@ -0,0 +1,106 @@ + + */ +class Console extends \Asika\SimpleConsole\Console +{ + // Disables the default help handling + protected $helpOptions = []; + protected $customHelpOptions = ['h', 'help', '?']; + + protected $subConsoles = [ + 'config' => __NAMESPACE__ . '\Console\Config', + 'probe' => __NAMESPACE__ . '\Console\Probe', + 'po2php' => __NAMESPACE__ . '\Console\PoToPhp', + ]; + + protected function getHelp() + { + $help = << [] [-v] + +Commands: + config Edit site config + probe Probe a single site + po2php Generate a strings.php file from a messages.po file + +Options: + -h|--help|-? Show help information + -v Show more debug information. +HELP; + return $help; + } + + protected function doExecute() + { + if ($this->getOption('v')) { + $this->out('Executable: ' . $this->executable); + $this->out('Arguments: ' . var_export($this->args, true)); + $this->out('Options: ' . var_export($this->options, true)); + } + + $showHelp = false; + $subHelp = false; + $command = null; + + if ($this->getOption('version')) { + $this->out('Friendica Console version ' . BUILD_ID); + + return 0; + } elseif ((count($this->options) === 0 || $this->getOption($this->customHelpOptions) === true || $this->getOption($this->customHelpOptions) === 1) && count($this->args) === 0 + ) { + $showHelp = true; + } elseif (count($this->args) >= 2 && $this->getArgument(0) == 'help') { + $command = $this->getArgument(1); + $subHelp = true; + array_shift($this->args); + array_shift($this->args); + } elseif (count($this->args) >= 1) { + $command = $this->getArgument(0); + array_shift($this->args); + } + + if (is_null($command)) { + $this->out($this->getHelp()); + return 0; + } + + $console = $this->getSubConsole($command); + + if ($subHelp) { + $console->setOption($this->customHelpOptions, true); + } + + return $console->execute(); + } + + private function getSubConsole($command) + { + if ($this->getOption('v')) { + $this->out('Command: ' . $command); + } + + if (!isset($this->subConsoles[$command])) { + throw new \Asika\SimpleConsole\CommandArgsException('Command ' . $command . ' doesn\'t exist'); + } + + $subargs = $this->args; + array_unshift($subargs, $this->executable); + + $className = $this->subConsoles[$command]; + + $subconsole = new $className($subargs); + + foreach ($this->options as $name => $value) { + $subconsole->setOption($name, $value); + } + + return $subconsole; + } + +} -- 2.43.5 From ac41f7caf3b19f984a7f06ecd44bf0d370a78ef1 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 4 May 2018 08:16:36 -0400 Subject: [PATCH 05/11] Add console commands - Add Config command - Add Probe command - Add PoToPhp command - Remove po2php script --- src/Core/Console/Config.php | 137 ++++++++++++++++++++++++ src/Core/Console/PoToPhp.php | 201 +++++++++++++++++++++++++++++++++++ src/Core/Console/Probe.php | 114 ++++++++++++++++++++ util/po2php.php | 122 --------------------- 4 files changed, 452 insertions(+), 122 deletions(-) create mode 100644 src/Core/Console/Config.php create mode 100644 src/Core/Console/PoToPhp.php create mode 100644 src/Core/Console/Probe.php delete mode 100644 util/po2php.php diff --git a/src/Core/Console/Config.php b/src/Core/Console/Config.php new file mode 100644 index 00000000..e0e87082 --- /dev/null +++ b/src/Core/Console/Config.php @@ -0,0 +1,137 @@ + + */ +class Config extends \Asika\SimpleConsole\Console +{ + protected $helpOptions = ['h', 'help', '?']; + + protected function getHelp() + { + $help = << [-h|--help|-?] [-v] + bin/console config [-h|--help|-?] [-v] + bin/console config [-h|--help|-?] [-v] + +Description + bin/console config + Lists all config values + + bin/console config + Lists all config values in the provided category + + bin/console config + Shows the value of the provided key in the category + + bin/console config + Sets the value of the provided key in the category + +Notes: + Setting config entries which are manually set in .htconfig.php may result in + conflict between database settings and the manual startup settings. + +Options + -h|--help|-? Show help information + -v Show more debug information. +HELP; + return $help; + } + + protected function doExecute() + { + $a = get_app(); + + if ($this->getOption('v')) { + $this->out('Executable: ' . $this->executable); + $this->out('Class: ' . __CLASS__); + $this->out('Arguments: ' . var_export($this->args, true)); + $this->out('Options: ' . var_export($this->options, true)); + } + + if (count($this->args) > 3) { + throw new CommandArgsException('Too many arguments'); + } + + require_once '.htconfig.php'; + $db = new dba($db_host, $db_user, $db_pass, $db_data); + unset($db_host, $db_user, $db_pass, $db_data); + + if (!$result) { + throw new \RuntimeException('Unable to connect to database'); + } + + if (count($this->args) == 3) { + Core\Config::set($this->getArgument(0), $this->getArgument(1), $this->getArgument(2)); + $this->out("config[{$this->getArgument(0)}][{$this->getArgument(1)}] = " . Core\Config::get($this->getArgument(0), + $this->getArgument(1))); + } + + if (count($this->args) == 2) { + $this->out("config[{$this->getArgument(0)}][{$this->getArgument(1)}] = " . Core\Config::get($this->getArgument(0), + $this->getArgument(1))); + } + + if (count($this->args) == 1) { + Core\Config::load($this->getArgument(0)); + + $a = get_app(); + if (!is_null($a->config[$this->getArgument(0)])) { + foreach ($a->config[$this->getArgument(0)] as $k => $x) { + $this->out("config[{$this->getArgument(0)}][{$k}] = " . $x); + } + } else { + $this->out('Config section ' . $this->getArgument(0) . ' returned nothing'); + } + } + + if (count($this->args) == 0) { + $configs = dba::select('config'); + foreach ($configs as $config) { + $this->out("config[{$config['cat']}][{$config['k']}] = " . $config['v']); + } + } + + return 0; + } + +} diff --git a/src/Core/Console/PoToPhp.php b/src/Core/Console/PoToPhp.php new file mode 100644 index 00000000..99d34dc1 --- /dev/null +++ b/src/Core/Console/PoToPhp.php @@ -0,0 +1,201 @@ + + */ +class PoToPhp extends \Asika\SimpleConsole\Console +{ + protected $helpOptions = ['h', 'help', '?']; + + const DQ_ESCAPE = "__DQ__"; + + protected function getHelp() + { + $help = << [-h|--help|-?] [-v] + +Description + Read a messages.po file and create the according strings.php in the same directory + +Options + -h|--help|-? Show help information + -v Show more debug information. +HELP; + return $help; + } + + protected function doExecute() + { + if ($this->getOption('v')) { + $this->out('Class: ' . __CLASS__); + $this->out('Arguments: ' . var_export($this->args, true)); + $this->out('Options: ' . var_export($this->options, true)); + } + + if (count($this->args) == 0) { + $this->out($this->getHelp()); + return 0; + } + + if (count($this->args) > 1) { + throw new \Asika\SimpleConsole\CommandArgsException('Too many arguments'); + } + + $a = get_app(); + + $pofile = realpath($this->getArgument(0)); + + if (!file_exists($pofile)) { + throw new \RuntimeException('Supplied file path doesn\'t exist.'); + } + + if (!is_writable(dirname($pofile))) { + throw new \RuntimeException('Supplied directory isn\'t writable.'); + } + + $outfile = dirname($pofile) . DIRECTORY_SEPARATOR . 'strings.php'; + + if (strstr($outfile, 'util')) { + $lang = 'en'; + } else { + $lang = str_replace('-', '_', basename(dirname($pofile))); + } + + $this->out('Out to ' . $outfile); + + $out = "strings["' . $k . '"] = '; + } + + if ($inv) { + $inv = false; + $out .= '"' . $v . '"'; + } + + $v = substr($l, 8, $len - 10); + $v = preg_replace_callback($escape_s_exp, [$this, 'escapeDollar'], $v); + + $inv = true; + } + + if ($k != "" && substr($l, 0, 7) == 'msgstr[') { + if ($ink) { + $ink = false; + $out .= '$a->strings["' . $k . '"] = '; + } + if ($inv) { + $inv = false; + $out .= '"' . $v . '"'; + } + + if (!$arr) { + $arr = true; + $out .= "[\n"; + } + + $match = []; + preg_match("|\[([0-9]*)\] (.*)|", $l, $match); + $out .= "\t" + . preg_replace_callback($escape_s_exp, [$this, 'escapeDollar'], $match[1]) + . ' => ' + . preg_replace_callback($escape_s_exp, [$this, 'escapeDollar'], $match[2]) + . ",\n"; + } + + if (substr($l, 0, 6) == 'msgid_') { + $ink = false; + $out .= '$a->strings["' . $k . '"] = '; + } + + if ($ink) { + $k .= trim($l, "\"\r\n"); + $k = preg_replace_callback($escape_s_exp, [$this, 'escapeDollar'], $k); + } + + if (substr($l, 0, 6) == 'msgid ') { + if ($inv) { + $inv = false; + $out .= '"' . $v . '"'; + } + + if ($k != "") { + $out .= ($arr) ? "];\n" : ";\n"; + } + + $arr = false; + $k = str_replace("msgid ", "", $l); + if ($k != '""') { + $k = trim($k, "\"\r\n"); + } else { + $k = ''; + } + + $k = preg_replace_callback($escape_s_exp, [$this, 'escapeDollar'], $k); + $ink = true; + } + + if ($inv && substr($l, 0, 6) != "msgstr") { + $v .= trim($l, "\"\r\n"); + $v = preg_replace_callback($escape_s_exp, [$this, 'escapeDollar'], $v); + } + } + + if ($inv) { + $inv = false; + $out .= '"' . $v . '"'; + } + + if ($k != '') { + $out .= ($arr ? "];\n" : ";\n"); + } + + $out = str_replace(self::DQ_ESCAPE, '\"', $out); + if (!file_put_contents($outfile, $out)) { + throw new \RuntimeException('Unable to write to ' . $outfile); + } + + return 0; + } + + private function escapeDollar($match) + { + return str_replace('$', '\$', $match[0]); + } +} diff --git a/src/Core/Console/Probe.php b/src/Core/Console/Probe.php new file mode 100644 index 00000000..41eff0a3 --- /dev/null +++ b/src/Core/Console/Probe.php @@ -0,0 +1,114 @@ + + */ +class Probe extends Console +{ + protected $helpOptions = ['h', 'help', '?']; + + protected function getHelp() + { + $help = << [-h|--help|-?] [-v] + +Description + Blocks an account in such a way that no postings or comments this account writes are accepted to this node. + +Options + -h|--help|-? Show help information + -v Show more debug information. +HELP; + return $help; + } + + protected function doExecute() + { + global $db, $a; + + $a = new \Friendica\Directory\App(); + + if ($this->getOption('v')) { + $this->out('Class: ' . __CLASS__); + $this->out('Arguments: ' . var_export($this->args, true)); + $this->out('Options: ' . var_export($this->options, true)); + } + + require_once '.htconfig.php'; + $db = new dba($db_host, $db_user, $db_pass, $db_data); + unset($db_host, $db_user, $db_pass, $db_data); + + if ($this->getOption('all')) { + $sites = q('SELECT * FROM `site-health` WHERE `health_score` >= 0'); + if (is_bool($sites)) { + throw new \RuntimeException('SQL Error'); + } elseif (!count($sites)) { + throw new \RuntimeException('No sites to probe ' . intval($this->getArgument(0))); + } else { + foreach($sites as $site) { + $this->out('Running probe for site ID ' . $site['id']); + run_site_probe($site['id'], $site); + } + } + + return 0; + } + + if (count($this->args) == 0) { + $this->out($this->getHelp()); + return 0; + } + + if (count($this->args) > 1) { + throw new CommandArgsException('Too many arguments'); + } + + if (is_numeric($this->getArgument(0))) { + $site_health = q('SELECT * FROM `site-health` WHERE `id` = %u LIMIT 1', intval($this->getArgument(0))); + + if (is_bool($site_health)) { + throw new \RuntimeException('SQL Error'); + } elseif (!count($site_health)) { + throw new \RuntimeException('Unknown site with ID ' . intval($this->getArgument(0))); + } else { + $site_health = $site_health[0]; + } + } else { + $site_health = q('SELECT * FROM `site-health` WHERE `base_url` LIKE "%%%s%%" OR `effective_base_url` LIKE "%%%s%%" LIMIT 1', + $this->getArgument(0), $this->getArgument(0)); + + if (is_bool($site_health)) { + throw new \RuntimeException('SQL Error'); + } elseif (!count($site_health)) { + throw new \RuntimeException('Unknown site with base URL ' . $this->getArgument(0)); + } else { + $site_health = $site_health[0]; + } + } + + $this->out(var_export($site_health, true)); + + $this->out('Running probe for site ID ' . $site_health['id']); + run_site_probe($site_health['id'], $site_health); + + $this->out(var_export($site_health, true)); + + return 0; + } +} diff --git a/util/po2php.php b/util/po2php.php deleted file mode 100644 index c703172a..00000000 --- a/util/po2php.php +++ /dev/null @@ -1,122 +0,0 @@ -\n\n"; - return; - } - - $pofile = $argv[1]; - $outfile = dirname($pofile)."/strings.php"; - - if(strstr($outfile,'util')) - $lang = 'en'; - else - $lang = str_replace('-','_',basename(dirname($pofile))); - - - - if (!file_exists($pofile)){ - print "Unable to find '$pofile'\n"; - return; - } - - print "Out to '$outfile'\n"; - - $out="strings["'.$k.'"] = '; } - if ($inv) { $inv = False; $out .= '"'.$v.'"'; } - - $v = substr($l,8,$len-10); - $v = preg_replace_callback($escape_s_exp,'escape_s',$v); - $inv = True; - //$out .= $v; - } - if ($k!="" && substr($l,0,7)=="msgstr["){ - if ($ink) { $ink = False; $out .= '$a->strings["'.$k.'"] = '; } - if ($inv) { $inv = False; $out .= '"'.$v.'"'; } - - if (!$arr) { - $arr=True; - $out .= "array(\n"; - } - $match=Array(); - preg_match("|\[([0-9]*)\] (.*)|", $l, $match); - $out .= "\t". - preg_replace_callback($escape_s_exp,'escape_s',$match[1]) - ." => " - .preg_replace_callback($escape_s_exp,'escape_s',$match[2]) .",\n"; - } - - if (substr($l,0,6)=="msgid_") { $ink = False; $out .= '$a->strings["'.$k.'"] = '; }; - - - if ($ink) { - $k .= trim($l,"\"\r\n"); - $k = preg_replace_callback($escape_s_exp,'escape_s',$k); - //$out .= '$a->strings['.$k.'] = '; - } - - if (substr($l,0,6)=="msgid "){ - if ($inv) { $inv = False; $out .= '"'.$v.'"'; } - if ($k!="") $out .= $arr?");\n":";\n"; - $arr=False; - $k = str_replace("msgid ","",$l); - if ($k != '""' ) { - $k = trim($k,"\"\r\n"); - } else { - $k = ""; - } - - $k = preg_replace_callback($escape_s_exp,'escape_s',$k); - $ink = True; - } - - if ($inv && substr($l,0,6)!="msgstr") { - $v .= trim($l,"\"\r\n"); - $v = preg_replace_callback($escape_s_exp,'escape_s',$v); - //$out .= '$a->strings['.$k.'] = '; - } - - - } - - if ($inv) { $inv = False; $out .= '"'.$v.'"'; } - if ($k!="") $out .= $arr?");\n":";\n"; - - file_put_contents($outfile, $out); - -} - -if (array_search(__file__,get_included_files())===0){ - po2php_run($argv,$argc); -} -- 2.43.5 From 27153fab68ea51c288d888093cb9a1c5a6113878 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 4 May 2018 08:21:47 -0400 Subject: [PATCH 06/11] Updat database structure file - Add a couple of fields in site-probe --- dfrndir.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dfrndir.sql b/dfrndir.sql index c1ff2143..21112f56 100644 --- a/dfrndir.sql +++ b/dfrndir.sql @@ -146,6 +146,8 @@ CREATE TABLE IF NOT EXISTS `site-probe` ( `site_health_id` int(10) UNSIGNED NOT NULL, `dt_performed` datetime NOT NULL, `request_time` int(10) UNSIGNED NOT NULL, + `avg_ping` int(11) DEFAULT NULL, + `speed_score` int(11) DEFAULT NULL, PRIMARY KEY (`id`), KEY `site_health_id` (`site_health_id`), KEY `dt_performed` (`dt_performed`) -- 2.43.5 From 95ea839fa7f70e34afc667eee1d25e011c676740 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 4 May 2018 08:22:01 -0400 Subject: [PATCH 07/11] Add ping stats during probe --- include/site-health.php | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/include/site-health.php b/include/site-health.php index cd42f017..7d6139d2 100644 --- a/include/site-health.php +++ b/include/site-health.php @@ -110,6 +110,16 @@ if (!function_exists('run_site_probe')) { $base_url = $entry['base_url']; $probe_location = $base_url . '/friendica/json'; + $net_ping = Net_Ping::factory(); + $net_ping->setArgs(['count' => 5]); + $result = $net_ping->ping(parse_url($base_url, PHP_URL_HOST)); + + if (is_a($result, 'Net_Ping_Result')) { + $avg_ping = $result->getAvg(); + } else { + $avg_ping = null; + } + //Prepare the CURL call. $handle = curl_init(); $options = array( @@ -172,6 +182,12 @@ if (!function_exists('run_site_probe')) { //Done with CURL now. curl_close($handle); + if ($time && $avg_ping) { + $speed_score = max(1, $avg_ping > 10 ? $time / $avg_ping : $time / 50); + } else { + $speed_score = null; + } + #TODO: if the site redirects elsewhere, notice this site and record an issue. $effective_base_url = parse_site_from_url($info['url']); $wrong_base_url = $effective_base_url !== $entry['base_url']; @@ -202,10 +218,12 @@ if (!function_exists('run_site_probe')) { //Record the probe speed in a probes table. q( - "INSERT INTO `site-probe` (`site_health_id`, `dt_performed`, `request_time`)" . - "VALUES (%u, NOW(), %u)", + "INSERT INTO `site-probe` (`site_health_id`, `dt_performed`, `request_time`, `avg_ping`, `speed_score`)" . + "VALUES (%u, NOW(), %u, %u, %u)", $entry['id'], - $time + $time, + $avg_ping, + $speed_score ); if (isset($data->addons)) { @@ -322,7 +340,7 @@ if (!function_exists('health_score_after_probe')) { } //Older than 3.3.x? - elseif (intval($versionParts[1] < 3)) { + elseif (!empty($versionParts[1]) && intval($versionParts[1] < 3)) { $current -= 5; //Somewhat outdated. } -- 2.43.5 From 4de3fcf5f5a21a64b992ec6c59375a0ea850da07 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 7 May 2018 22:00:26 -0400 Subject: [PATCH 08/11] Simplify the tag table - Remove auto increment id - Replaced nurl by profile id - Added primary key on the two remaining columns --- dfrndir.sql | 11 ++++------- include/submit.php | 8 ++++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/dfrndir.sql b/dfrndir.sql index 21112f56..1be9434a 100644 --- a/dfrndir.sql +++ b/dfrndir.sql @@ -230,13 +230,10 @@ CREATE TABLE IF NOT EXISTS `sync-timestamps` ( -- CREATE TABLE IF NOT EXISTS `tag` ( - `id` int(11) NOT NULL AUTO_INCREMENT, - `term` char(255) NOT NULL, - `nurl` char(255) NOT NULL, - PRIMARY KEY (`id`), - KEY `term` (`term`(250)), - KEY `nurl` (`nurl`(250)) -) ENGINE=MyISAM AUTO_INCREMENT=101679 DEFAULT CHARSET=utf8mb4; + `term` varchar(50) NOT NULL, + `profile_id` int(11) NOT NULL, + PRIMARY KEY (`term`,`profile_id`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- -------------------------------------------------------- diff --git a/include/submit.php b/include/submit.php index a61e7fb5..4001be85 100644 --- a/include/submit.php +++ b/include/submit.php @@ -39,8 +39,8 @@ function run_submit($url) `updated` = '%s' WHERE `id` = %d LIMIT 1", dbesc(datetime_convert()), intval($profile_id) ); - $r = q("DELETE FROM `tag` WHERE `nurl` = '%s'", - dbesc($r[0]['nurl']) + $r = q("DELETE FROM `tag` WHERE `profile_id` = %d", + intval($profile_id) ); } @@ -188,9 +188,9 @@ function run_submit($url) $t = substr($t, 0, 254); if (strlen($t)) { - $r = q("INSERT INTO `tag` (`term`, `nurl`) VALUES ('%s', '%s') ", + $r = q("INSERT INTO `tag` (`term`, `profile_id`) VALUES ('%s', %d) ", dbesc($t), - dbesc($nurl) + intval($profile_id) ); } } -- 2.43.5 From 90314232789a57da554cdb913e26ad8def92a16b Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 7 May 2018 22:02:03 -0400 Subject: [PATCH 09/11] Prevent infinite push loop between multiple directories - Add auto increment id to sync-push-queue table to add new profile urls at the end - Use REPLACE instead of INSERT to prevent unique key errors --- dfrndir.sql | 4 +++- include/sync.php | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dfrndir.sql b/dfrndir.sql index 1be9434a..07f6fa4b 100644 --- a/dfrndir.sql +++ b/dfrndir.sql @@ -190,8 +190,10 @@ CREATE TABLE IF NOT EXISTS `sync-pull-queue` ( -- CREATE TABLE IF NOT EXISTS `sync-push-queue` ( + `id` int(11) NOT NULL AUTO_INCREMENT, `url` varchar(255) CHARACTER SET utf8 NOT NULL, - PRIMARY KEY (`url`) + PRIMARY KEY (`id`), + UNIQUE KEY `url` (`url`) USING BTREE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -- -------------------------------------------------------- diff --git a/include/sync.php b/include/sync.php index 3d816cee..b569e84a 100644 --- a/include/sync.php +++ b/include/sync.php @@ -13,7 +13,7 @@ function sync_pull($url) //If we support it that is. if ($a->config['syncing']['enable_pulling']) { - q("INSERT INTO `sync-pull-queue` (`url`) VALUES ('%s')", dbesc($url)); + q("REPLACE INTO `sync-pull-queue` (`url`) VALUES ('%s')", dbesc($url)); } } @@ -28,7 +28,7 @@ function sync_push($url) //If we support it that is. if ($a->config['syncing']['enable_pushing']) { - q("INSERT INTO `sync-push-queue` (`url`) VALUES ('%s')", dbesc($url)); + q("REPLACE INTO `sync-push-queue` (`url`) VALUES ('%s')", dbesc($url)); } sync_mark($url); @@ -96,7 +96,7 @@ function get_push_targets() */ function get_push_batch(App $a) { - return q("SELECT * FROM `sync-push-queue` LIMIT %u", intval($a->config['syncing']['max_push_items'])); + return q("SELECT * FROM `sync-push-queue` ORDER BY `id` LIMIT %u", intval($a->config['syncing']['max_push_items'])); } /** -- 2.43.5 From 15563be5ccc906aa9f8d80570ae28998b33dad86 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Mon, 7 May 2018 22:02:12 -0400 Subject: [PATCH 10/11] Fix PHP Notices --- include/Scrape.php | 62 +++++++++++++++++++++++++++++----------------- include/submit.php | 10 ++++---- mod/photo.php | 5 +--- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/include/Scrape.php b/include/Scrape.php index 0c0a7326..5caae6b3 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -11,51 +11,67 @@ function attribute_contains($attr,$s) { }} -if(! function_exists('noscrape_dfrn')) { -function noscrape_dfrn($url) { +if (!function_exists('noscrape_dfrn')) { + +function noscrape_dfrn($url) +{ $submit_noscrape_start = microtime(true); $data = fetch_url($url); $submit_noscrape_request_end = microtime(true); - if(empty($data)) return false; + if (empty($data)) { + return false; + } + $parms = json_decode($data, true); - if(!$parms || !count($parms)) return false; - $parms['tags'] = implode(' ', (array)$parms['tags']); + if (!$parms || !count($parms)) { + return false; + } + + if (isset($parms['tags'])) { + $parms['tags'] = implode(' ', (array) $parms['tags']); + } else { + $parms['tags'] = ''; + } + $submit_noscrape_end = microtime(true); $parms['_timings'] = array( 'fetch' => round(($submit_noscrape_request_end - $submit_noscrape_start) * 1000), 'scrape' => round(($submit_noscrape_end - $submit_noscrape_request_end) * 1000) ); + return $parms; -}} +} + +} if(! function_exists('scrape_dfrn')) { function scrape_dfrn($url, $max_nodes=3500) { - + $minNodes = 100; //Lets do at least 100 nodes per type. $timeout = 10; //Timeout will affect batch processing. - + //Try and cheat our way into faster profiles. if(strpos($url, 'tab=profile') === false){ $url .= (strpos($url, '?') > 0 ? '&' : '?').'tab=profile'; } - + $scrape_start = microtime(true); - + $ret = array(); $s = fetch_url($url, $timeout); - + $scrape_fetch_end = microtime(true); - - if(! $s) + + if(! $s) return $ret; - + $dom = HTML5_Parser::parse($s); - + if(! $dom) return $ret; - + $items = $dom->getElementsByTagName('meta'); - + // get DFRN link elements $nodes_left = max(intval($max_nodes), $minNodes); $targets = array('hide', 'comm', 'tags'); @@ -89,7 +105,7 @@ function scrape_dfrn($url, $max_nodes=3500) { $items = $dom->getElementsByTagName('link'); // get DFRN link elements - + $nodes_left = max(intval($max_nodes), $minNodes); foreach($items as $item) { $x = $item->getAttribute('rel'); @@ -100,7 +116,7 @@ function scrape_dfrn($url, $max_nodes=3500) { } // Pull out hCard profile elements - + $nodes_left = max(intval($max_nodes), $minNodes); $items = $dom->getElementsByTagName('*'); $targets = array('fn', 'pdesc', 'photo', 'key', 'locality', 'region', 'postal-code', 'country-name'); @@ -146,18 +162,18 @@ function scrape_dfrn($url, $max_nodes=3500) { $nodes_left--; if($nodes_left <= 0 || $targets_left <= 0) break; } - + $scrape_end = microtime(true); $fetch_time = round(($scrape_fetch_end - $scrape_start) * 1000); $scrape_time = round(($scrape_end - $scrape_fetch_end) * 1000); - + $ret['_timings'] = array( 'fetch' => $fetch_time, 'scrape' => $scrape_time ); - + return $ret; - + }} diff --git a/include/submit.php b/include/submit.php index 4001be85..44075d09 100644 --- a/include/submit.php +++ b/include/submit.php @@ -131,11 +131,11 @@ function run_submit($url) `updated` = '%s' WHERE `id` = %d LIMIT 1", $params['fn'], - $params['pdesc'], - $params['locality'], - $params['region'], - $params['postal-code'], - $params['country-name'], + isset($params['pdesc']) ? $params['pdesc'] : '', + isset($params['locality']) ? $params['locality'] : '', + isset($params['region']) ? $params['region'] : '', + isset($params['postal-code']) ? $params['postal-code'] : '', + isset($params['country-name']) ? $params['country-name'] : '', dbesc($url), dbesc($nurl), intval($params['comm']), diff --git a/mod/photo.php b/mod/photo.php index c832a9e9..bd53759e 100644 --- a/mod/photo.php +++ b/mod/photo.php @@ -18,12 +18,9 @@ function photo_init(App $a) $profile_id = str_replace('.jpg', '', $photo); $r = q('SELECT * FROM `photo` WHERE `profile-id` = %d LIMIT 1', intval($profile_id)); - if (count($r)) { $data = $r[0]['data']; - } - - if (x($data) === false || (!strlen($data))) { + } else { $data = file_get_contents('images/default-profile-sm.jpg'); } -- 2.43.5 From 945c1d1c00295572a1d35a26f6f01fdc2cc27fcc Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Tue, 8 May 2018 03:50:43 -0400 Subject: [PATCH 11/11] Remove placeholder header --- src/Core/Console/Config.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Core/Console/Config.php b/src/Core/Console/Config.php index e0e87082..3eb98035 100644 --- a/src/Core/Console/Config.php +++ b/src/Core/Console/Config.php @@ -1,11 +1,5 @@