Merge branch '2020.06-rc' into stable

.travis.yml

@@ -8,7 +8,7 @@ php:
 
 services:
  - mysql
- - redis-server
+ - redis
  - memcached
 env:
  - MYSQL_HOST=localhost MYSQL_PORT=3306 MYSQL_USERNAME=travis MYSQL_PASSWORD="" MYSQL_DATABASE=test

CHANGELOG

@@ -1,3 +1,61 @@
+Version 2020.07 (2020-07-12)
+  Friendica Core:
+    Update to the translations: DE, EN GB, EN US, FR, ET, NL, PL, RU, ZH-CN [translation teams]
+    Updates to the themes (frio, vier) [MrPetovan]
+    Updated the shipped composer version, and the dependency list [annando, MrPetovan, tobiasd]
+    Updates to the documentation [MrPetovan]
+    General code refactoring and enhancements [AlfredSK, annando, MrPetovan]
+    Replace charged terms with "allowlist", "denylist" and "blocklist" [MrPetovan]
+    Enhanced the comment distribution in threads that involve diaspora*, AP and DFRN actors [annando]
+    Enhanced the profile probing mechanism [annando, MrPetovan]
+    Enhanced the post update process of the database [annando]
+    Enhanced the database performance [annando]
+    Enhanced ActivityPub attachment handling [MrPetovan]
+    Enhanced security of redirections [annando]
+    Enhanced database performance [annando]
+    Enhanced the handling of BBCode [pre] tags [MrPetovan]
+    Enhanced Markdown to BBCode conversion [MrPetovan]
+    Enhanced the speed of the network page [annando]
+    Fixed a problem recognising logins via the API [MrPetovan]
+    Fixed a problem with handling local diaspora* URLs [MrPetovan]
+    Fixed a problem with implicit mentions [annando]
+    Fixed a problem with the password reset token security [lynn-stephenson]
+    Fixed a problem with receiving non-public posts via ActivityPub [annando]
+    Fixed a problem with the photo endpoint of the API [MrPetovan]
+    Fixed a problem with pressing the ESC key in the frio-theme [MrPetovan]
+    Fixed a problem with the display if post categories [annando]
+    Fixed a problem with validation of feeds [annando]
+    Fixed a problem that prevented AP activities being fetched sometimes [annando]
+    Renamed the -q option of the console user delete command to -y [MrPetovan]
+    Added notification count to page title [MrPetovan]
+    Added handling of relative URLs during feed detection [MrPetovan]
+    Added entities [nupplaphil]
+
+  Friendica Addons:
+    Update to the translations (EN GB, NB NO, NL, PL, RU, ZH CN) [translation teams]
+    blockbot:
+      The list of accepted user agents was enhanced [annando]
+    Diaspora*:
+      Enhanced conntector settings [MrPetovan]
+    PHP Mailer SMTP:
+      Updated phpmailer version [dependabot]
+    showmore_dyn:
+      New addon to collapse long post depending on their actual height [wiwie]
+    twitter:
+      Enhaceed the handling of mobile twitter URLs [annando]
+      Enhanced the handling of quoted tweets [MrPetovan]
+      added HTML error code handling [MrPetovan]
+    various:
+      enhancements to the probe mechanism [MrPetovan]
+
+  Closed Issues:
+    3084, 3884, 8287, 8314, 8374, 8400, 8425, 8432, 8458, 8470, 8477,
+    8482, 8488, 8489, 8490, 8493, 8495, 8498, 8511, 8517, 8523, 8527,
+    8551, 8553, 8560, 8564, 8565, 8568, 8578, 8586, 8593, 8606, 8610,
+    8612, 8626, 8664, 8672, 8683, 8685, 8691, 8694, 8702, 8709, 8714,
+    8717, 8722, 8726, 8732, 8736, 8743, 8744, 8746, 8756, 8766, 8769,
+    8781, 8800, 8807, 8808, 8827, 8829, 8836, 8844, 8846, 8857, 8866
+
 Version 2020.03 "Red Hot Poker"  (2020-03-30)
   Friendica Core:
     Updates to the translations (DE, FR, JA, NL, PL, RU, ZH-CN) [translation teams]
@@ -52,7 +110,7 @@ Version 2020.03 "Red Hot Poker"  (2020-03-30)
     Update to the translations (CS, DE, FR, PL, RU, ZH-CN) [translation teams]
     General code refactoring and enhancements [AndyHee, annando, MrPetovan, nupplaphil]
     blockbot:
-      Ensure that good agents are whitelisted [valvin1]
+      Ensure that good agents are allowlisted [valvin1]
     markdown:
       Addon to use Markdown while composing a posting was added [annando]
     showmore:
@@ -902,7 +960,7 @@ Version 3.5.3 (2017-10-05)
     Updates to the documentation [tobiasd]
     Code revision and refactoring [Hypolite]
     pumpio, twitter bridges adopted to new background mechanism [annando]
-    Leistungsschutzrecht has a new source list, and a whitelist [annando]
+    Leistungsschutzrecht has a new source list, and an allowlist [annando]
     retriever marked unsupported due to unwanted side-effects [annando]
     Unicode emoji added [annando]
     Enhancement to the general content filter [annando]
@@ -1364,7 +1422,7 @@ Version 3.3.1 (2014-11-06)
 	Set default location to empty for new users. Suppress warning on user creation (issue #1193) (fabrixxm)
 	Correctly build urls with queries (issue #1190) (fabrixxm)
 	Optionally use keywords in feed as post tags with "remote self" (annando)
-	A blacklist of keywords to not use can be defined (annando)
+	A denylist of keywords to not use can be defined (annando)
 	"remote self" works also with Friendica and Diaspora contacts (annando)
 	Show exact post time after 12 hours (FX7)
 	Optionally redirect from non-SSL to SSL (annando)

CREDITS.txt

@@ -9,6 +9,7 @@ Aditoo
 AgnesElisa
 Albert
 Alberto Díaz Tormo
+Aleksandr "M.O.Z.G" Dikov
 Alex
 Alexander An
 Alexander Fortin
@@ -131,6 +132,7 @@ julia.domagalska
 Julio Cova
 Karel
 Karolina
+Keenan Pepper
 Keith Fernie
 Klaus Weidenbach
 Koyu Berteon
@@ -141,8 +143,10 @@ Leberwurscht
 Leonard Lausen
 Lionel Triay
 loma-one
+loma1
 Lorem Ipsum
 Ludovic Grossard
+Lynn Stephenson
 maase2
 Magdalena Gazda
 Mai Anh Nguyen
@@ -231,6 +235,7 @@ St John Karp
 Stanislav N.
 Steffen K9
 StefOfficiel
+steve jobs
 Sveinn í Felli
 Sven Anders
 Sylke Vicious

VERSION

@@ -1 +1 @@
-2020.03
+2020.06-rc

bin/run_xgettext.sh

@@ -32,7 +32,7 @@ case "$MODE" in
 		mkdir -p "$FULLPATH/../addon/$ADDONNAME/lang/C"
 		OUTFILE="$FULLPATH/../addon/$ADDONNAME/lang/C/messages.po"
 		FINDSTARTDIR="."
-		FINDOPTS=
+		FINDOPTS="-path ./vendor -prune -or"
 	;;
 	'single')
 		FULLPATH=$PWD
@@ -40,7 +40,7 @@ case "$MODE" in
 		mkdir -p "$FULLPATH/lang/C"
 		OUTFILE="$FULLPATH/lang/C/messages.po"
 		FINDSTARTDIR="."
-		FINDOPTS=		
+		FINDOPTS="-path ./vendor -prune -or"
 		echo "Extract strings for single addon '$ADDONNAME'"
 	;;
 	'default')
@@ -48,7 +48,7 @@ case "$MODE" in
 		OUTFILE="$FULLPATH/../view/lang/C/messages.po"
 		FINDSTARTDIR="."
 		# skip addon folder
-		FINDOPTS="( -wholename */addon -or -wholename */addons -or -wholename */addons-extra -or -wholename */smarty3 ) -prune -o"
+		FINDOPTS="( -path ./addon -or -path ./addons -or -path ./addons-extra -or -path ./tests -or -path ./view/lang -or -path ./view/smarty3 -or -path ./vendor ) -prune -or"
 		
 		F9KVERSION=$(cat ./VERSION);
 		echo "Friendica version $F9KVERSION"
@@ -58,18 +58,32 @@ esac
 
 KEYWORDS="-k -kt -ktt:1,2"
 
-echo "extract strings to $OUTFILE.."
+echo "Extract strings to $OUTFILE.."
 rm "$OUTFILE"; touch "$OUTFILE"
-for f in $(find "$FINDSTARTDIR" $FINDOPTS -name "*.php" -type f)
+
+find_result=$(find "$FINDSTARTDIR" $FINDOPTS -name "*.php" -type f)
+
+total_files=$(wc -l <<< "${find_result}")
+
+for file in $find_result
 do
-	if [ ! -d "$f" ]
+	((count++))
+	echo -ne "                                            \r"
+	echo -ne "Reading file $count/$total_files..."
+
+	# On Windows, find still outputs the name of pruned folders
+	if [ ! -d "$file" ]
 	then
-		xgettext $KEYWORDS -j -o "$OUTFILE" --from-code=UTF-8 "$f"
+		xgettext $KEYWORDS -j -o "$OUTFILE" --from-code=UTF-8 "$file" || exit 1
 		sed -i "s/CHARSET/UTF-8/g" "$OUTFILE"
 	fi
 done
+echo -ne "\n"
+
+echo "Interpolate metadata.."
+
+sed -i "s/^\"Plural-Forms.*$//g" "$OUTFILE"
 
-echo "setup base info.."
 case "$MODE" in
 	'addon'|'single')
 		sed -i "s/SOME DESCRIPTIVE TITLE./ADDON $ADDONNAME/g" "$OUTFILE"
@@ -77,25 +91,21 @@ case "$MODE" in
 		sed -i "s/FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.//g" "$OUTFILE"
 		sed -i "s/PACKAGE VERSION//g" "$OUTFILE"
 		sed -i "s/PACKAGE/Friendica $ADDONNAME addon/g" "$OUTFILE"
-		sed -i "s/CHARSET/UTF-8/g" "$OUTFILE"
-		sed -i "s/^\"Plural-Forms.*$//g" "$OUTFILE"
 	;;
 	'default')
 		sed -i "s/SOME DESCRIPTIVE TITLE./FRIENDICA Distributed Social Network/g" "$OUTFILE"
-		sed -i "s/YEAR THE PACKAGE'S COPYRIGHT HOLDER/2010, 2011, 2012, 2013 the Friendica Project/g" "$OUTFILE"
+		sed -i "s/YEAR THE PACKAGE'S COPYRIGHT HOLDER/2010-$(date +%Y) the Friendica Project/g" "$OUTFILE"
 		sed -i "s/FIRST AUTHOR <EMAIL@ADDRESS>, YEAR./Mike Macgirvin, 2010/g" "$OUTFILE"
 		sed -i "s/PACKAGE VERSION/$F9KVERSION/g" "$OUTFILE"
 		sed -i "s/PACKAGE/Friendica/g" "$OUTFILE"
-		sed -i "s/CHARSET/UTF-8/g" "$OUTFILE"
-		sed -i "s/^\"Plural-Forms.*$//g" "$OUTFILE"
 	;;
 esac
 
 if [ "" != "$1" -a "$MODE" == "default" ]
 then
 	UPDATEFILE="$(readlink -f ${FULLPATH}/$1)"
-	echo "merging new strings to $UPDATEFILE.."
+	echo "Merging new strings to $UPDATEFILE.."
 	msgmerge -U $OUTFILE $UPDATEFILE
 fi
 
-echo "done."
+echo "Done."

boot.php

@@ -33,13 +33,12 @@ use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Notify;
-use Friendica\Model\Term;
 use Friendica\Util\BasePath;
 use Friendica\Util\DateTimeFormat;
 
 define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'Red Hot Poker');
-define('FRIENDICA_VERSION',      '2020.03');
+define('FRIENDICA_VERSION',      '2020.06-rc');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
@@ -178,29 +177,6 @@ define('NOTIFY_SHARE',       Notify\Type::SHARE);
 define('NOTIFY_SYSTEM',      Notify\Type::SYSTEM);
 /* @}*/
 
-
-/** @deprecated since 2019.03, use Term::UNKNOWN instead */
-define('TERM_UNKNOWN',   Term::UNKNOWN);
-/** @deprecated since 2019.03, use Term::HASHTAG instead */
-define('TERM_HASHTAG',   Term::HASHTAG);
-/** @deprecated since 2019.03, use Term::MENTION instead */
-define('TERM_MENTION',   Term::MENTION);
-/** @deprecated since 2019.03, use Term::CATEGORY instead */
-define('TERM_CATEGORY',  Term::CATEGORY);
-/** @deprecated since 2019.03, use Term::PCATEGORY instead */
-define('TERM_PCATEGORY', Term::PCATEGORY);
-/** @deprecated since 2019.03, use Term::FILE instead */
-define('TERM_FILE',      Term::FILE);
-/** @deprecated since 2019.03, use Term::SAVEDSEARCH instead */
-define('TERM_SAVEDSEARCH', Term::SAVEDSEARCH);
-/** @deprecated since 2019.03, use Term::CONVERSATION instead */
-define('TERM_CONVERSATION', Term::CONVERSATION);
-
-/** @deprecated since 2019.03, use Term::OBJECT_TYPE_POST instead */
-define('TERM_OBJ_POST',  Term::OBJECT_TYPE_POST);
-/** @deprecated since 2019.03, use Term::OBJECT_TYPE_PHOTO instead */
-define('TERM_OBJ_PHOTO', Term::OBJECT_TYPE_PHOTO);
-
 /**
  * @name Gravity
  *

composer.json

@@ -92,6 +92,9 @@
 		}
 	},
 	"config": {
+		"platform": {
+			"php": "7.0"
+		},
 		"autoloader-suffix": "Friendica",
 		"optimize-autoloader": true,
 		"preferred-install": "dist",

database.sql

@@ -1,9 +1,186 @@
 -- ------------------------------------------
--- Friendica 2020.03-rc (Dalmatian Bellflower)
+-- Friendica 2020.06-dev (Red Hot Poker)
--- DB_UPDATE_VERSION 1338
+-- DB_UPDATE_VERSION 1353
 -- ------------------------------------------
 
 
+--
+-- TABLE gserver
+--
+CREATE TABLE IF NOT EXISTS `gserver` (
+	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
+	`url` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`nurl` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`version` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`site_name` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`info` text COMMENT '',
+	`register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '',
+	`registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users',
+	`directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)',
+	`poco` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`network` char(4) NOT NULL DEFAULT '' COMMENT '',
+	`platform` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`relay-subscribe` boolean NOT NULL DEFAULT '0' COMMENT 'Has the server subscribed to the relay system',
+	`relay-scope` varchar(10) NOT NULL DEFAULT '' COMMENT 'The scope of messages that the server wants to get',
+	`detection-method` tinyint unsigned COMMENT 'Method that had been used to detect that server',
+	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`last_poco_query` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `nurl` (`nurl`(190))
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Global servers';
+
+--
+-- TABLE clients
+--
+CREATE TABLE IF NOT EXISTS `clients` (
+	`client_id` varchar(20) NOT NULL COMMENT '',
+	`pw` varchar(20) NOT NULL DEFAULT '' COMMENT '',
+	`redirect_uri` varchar(200) NOT NULL DEFAULT '' COMMENT '',
+	`name` text COMMENT '',
+	`icon` text COMMENT '',
+	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
+	 PRIMARY KEY(`client_id`)
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
+
+--
+-- TABLE contact
+--
+CREATE TABLE IF NOT EXISTS `contact` (
+	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
+	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'Owner User id',
+	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`updated` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last contact update',
+	`self` boolean NOT NULL DEFAULT '0' COMMENT '1 if the contact is the user him/her self',
+	`remote_self` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`rel` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'The kind of the relation between the user and the contact',
+	`duplex` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`network` char(4) NOT NULL DEFAULT '' COMMENT 'Network of the contact',
+	`protocol` char(4) NOT NULL DEFAULT '' COMMENT 'Protocol of the contact',
+	`name` varchar(255) NOT NULL DEFAULT '' COMMENT 'Name that this contact is known by',
+	`nick` varchar(255) NOT NULL DEFAULT '' COMMENT 'Nick- and user name of the contact',
+	`location` varchar(255) DEFAULT '' COMMENT '',
+	`about` text COMMENT '',
+	`keywords` text COMMENT 'public keywords (interests) of the contact',
+	`gender` varchar(32) NOT NULL DEFAULT '' COMMENT 'Deprecated',
+	`xmpp` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`attag` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`avatar` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`photo` varchar(255) DEFAULT '' COMMENT 'Link to the profile photo of the contact',
+	`thumb` varchar(255) DEFAULT '' COMMENT 'Link to the profile photo (thumb size)',
+	`micro` varchar(255) DEFAULT '' COMMENT 'Link to the profile photo (micro size)',
+	`site-pubkey` text COMMENT '',
+	`issued-id` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`dfrn-id` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`url` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`nurl` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`addr` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`alias` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`pubkey` text COMMENT 'RSA public key 4096 bit',
+	`prvkey` text COMMENT 'RSA private key 4096 bit',
+	`batch` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`request` varchar(255) COMMENT '',
+	`notify` varchar(255) COMMENT '',
+	`poll` varchar(255) COMMENT '',
+	`confirm` varchar(255) COMMENT '',
+	`subscribe` varchar(255) COMMENT '',
+	`poco` varchar(255) COMMENT '',
+	`aes_allow` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`ret-aes` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`usehub` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`subhub` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`hub-verify` varchar(255) NOT NULL DEFAULT '' COMMENT '',
+	`last-update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last try to update the contact info',
+	`success_update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last successful contact update',
+	`failure_update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed update',
+	`name-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`uri-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`avatar-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`term-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`last-item` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'date of the last post',
+	`priority` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
+	`blocked` boolean NOT NULL DEFAULT '1' COMMENT 'Node-wide block status',
+	`block_reason` text COMMENT 'Node-wide block reason',
+	`readonly` boolean NOT NULL DEFAULT '0' COMMENT 'posts of the contact are readonly',
+	`writable` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`forum` boolean NOT NULL DEFAULT '0' COMMENT 'contact is a forum',
+	`prv` boolean NOT NULL DEFAULT '0' COMMENT 'contact is a private group',
+	`contact-type` tinyint NOT NULL DEFAULT 0 COMMENT '',
+	`hidden` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`archive` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`pending` boolean NOT NULL DEFAULT '1' COMMENT '',
+	`deleted` boolean NOT NULL DEFAULT '0' COMMENT 'Contact has been deleted',
+	`rating` tinyint NOT NULL DEFAULT 0 COMMENT '',
+	`unsearchable` boolean NOT NULL DEFAULT '0' COMMENT 'Contact prefers to not be searchable',
+	`sensitive` boolean NOT NULL DEFAULT '0' COMMENT 'Contact posts sensitive content',
+	`baseurl` varchar(255) DEFAULT '' COMMENT 'baseurl of the contact',
+	`gsid` int unsigned COMMENT 'Global Server ID',
+	`reason` text COMMENT '',
+	`closeness` tinyint unsigned NOT NULL DEFAULT 99 COMMENT '',
+	`info` mediumtext COMMENT '',
+	`profile-id` int unsigned COMMENT 'Deprecated',
+	`bdyear` varchar(4) NOT NULL DEFAULT '' COMMENT '',
+	`bd` date NOT NULL DEFAULT '0001-01-01' COMMENT '',
+	`notify_new_posts` boolean NOT NULL DEFAULT '0' COMMENT '',
+	`fetch_further_information` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
+	`ffi_keyword_denylist` text COMMENT '',
+	 PRIMARY KEY(`id`),
+	 INDEX `uid_name` (`uid`,`name`(190)),
+	 INDEX `self_uid` (`self`,`uid`),
+	 INDEX `alias_uid` (`alias`(32),`uid`),
+	 INDEX `pending_uid` (`pending`,`uid`),
+	 INDEX `blocked_uid` (`blocked`,`uid`),
+	 INDEX `uid_rel_network_poll` (`uid`,`rel`,`network`,`poll`(64),`archive`),
+	 INDEX `uid_network_batch` (`uid`,`network`,`batch`(64)),
+	 INDEX `addr_uid` (`addr`(32),`uid`),
+	 INDEX `nurl_uid` (`nurl`(32),`uid`),
+	 INDEX `nick_uid` (`nick`(32),`uid`),
+	 INDEX `dfrn-id` (`dfrn-id`(64)),
+	 INDEX `issued-id` (`issued-id`(64)),
+	 INDEX `gsid` (`gsid`),
+	FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contact table';
+
+--
+-- TABLE item-uri
+--
+CREATE TABLE IF NOT EXISTS `item-uri` (
+	`id` int unsigned NOT NULL auto_increment,
+	`uri` varbinary(255) NOT NULL COMMENT 'URI of an item',
+	`guid` varbinary(255) COMMENT 'A unique identifier for an item',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `uri` (`uri`),
+	 INDEX `guid` (`guid`)
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='URI and GUID for items';
+
+--
+-- TABLE permissionset
+--
+CREATE TABLE IF NOT EXISTS `permissionset` (
+	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
+	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'Owner id of this permission set',
+	`allow_cid` mediumtext COMMENT 'Access Control - list of allowed contact.id \'<19><78>\'',
+	`allow_gid` mediumtext COMMENT 'Access Control - list of allowed groups',
+	`deny_cid` mediumtext COMMENT 'Access Control - list of denied contact.id',
+	`deny_gid` mediumtext COMMENT 'Access Control - list of denied groups',
+	 PRIMARY KEY(`id`),
+	 INDEX `uid_allow_cid_allow_gid_deny_cid_deny_gid` (`allow_cid`(50),`allow_gid`(30),`deny_cid`(50),`deny_gid`(30))
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
+
+--
+-- TABLE tag
+--
+CREATE TABLE IF NOT EXISTS `tag` (
+	`id` int unsigned NOT NULL auto_increment COMMENT '',
+	`name` varchar(96) NOT NULL DEFAULT '' COMMENT '',
+	`url` varbinary(255) NOT NULL DEFAULT '' COMMENT '',
+	 PRIMARY KEY(`id`),
+	 UNIQUE INDEX `type_name_url` (`name`,`url`),
+	 INDEX `url` (`url`)
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='tags and mentions';
+
 --
 -- TABLE 2fa_app_specific_password
 --
@@ -64,7 +241,9 @@ CREATE TABLE IF NOT EXISTS `apcontact` (
 	`addr` varchar(255) COMMENT '',
 	`alias` varchar(255) COMMENT '',
 	`pubkey` text COMMENT '',
+	`subscribe` varchar(255) COMMENT '',
 	`baseurl` varchar(255) COMMENT 'baseurl of the ap contact',
+	`gsid` int unsigned COMMENT 'Global Server ID',
 	`generator` varchar(255) COMMENT 'Name of the contact\'s system',
 	`following_count` int unsigned DEFAULT 0 COMMENT 'Number of following contacts',
 	`followers_count` int unsigned DEFAULT 0 COMMENT 'Number of followers',
@@ -73,7 +252,9 @@ CREATE TABLE IF NOT EXISTS `apcontact` (
 	 PRIMARY KEY(`url`),
 	 INDEX `addr` (`addr`(32)),
 	 INDEX `alias` (`alias`(190)),
-	 INDEX `url` (`followers`(190))
+	 INDEX `followers` (`followers`(190)),
+	 INDEX `gsid` (`gsid`),
+	FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='ActivityPub compatible contacts - used in the ActivityPub implementation';
 
 --
@@ -107,7 +288,9 @@ CREATE TABLE IF NOT EXISTS `auth_codes` (
 	`redirect_uri` varchar(200) NOT NULL DEFAULT '' COMMENT '',
 	`expires` int NOT NULL DEFAULT 0 COMMENT '',
 	`scope` varchar(250) NOT NULL DEFAULT '' COMMENT '',
-	 PRIMARY KEY(`id`)
+	 PRIMARY KEY(`id`),
+	 INDEX `client_id` (`client_id`),
+	FOREIGN KEY (`client_id`) REFERENCES `clients` (`client_id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
 
 --
@@ -135,19 +318,6 @@ CREATE TABLE IF NOT EXISTS `challenge` (
 	 PRIMARY KEY(`id`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
 
---
--- TABLE clients
---
-CREATE TABLE IF NOT EXISTS `clients` (
-	`client_id` varchar(20) NOT NULL COMMENT '',
-	`pw` varchar(20) NOT NULL DEFAULT '' COMMENT '',
-	`redirect_uri` varchar(200) NOT NULL DEFAULT '' COMMENT '',
-	`name` text COMMENT '',
-	`icon` text COMMENT '',
-	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
-	 PRIMARY KEY(`client_id`)
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
-
 --
 -- TABLE config
 --
@@ -160,100 +330,6 @@ CREATE TABLE IF NOT EXISTS `config` (
 	 UNIQUE INDEX `cat_k` (`cat`,`k`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='main configuration storage';
 
---
--- TABLE contact
---
-CREATE TABLE IF NOT EXISTS `contact` (
-	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
-	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'Owner User id',
-	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`updated` datetime DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last contact update',
-	`self` boolean NOT NULL DEFAULT '0' COMMENT '1 if the contact is the user him/her self',
-	`remote_self` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`rel` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'The kind of the relation between the user and the contact',
-	`duplex` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`network` char(4) NOT NULL DEFAULT '' COMMENT 'Network of the contact',
-	`protocol` char(4) NOT NULL DEFAULT '' COMMENT 'Protocol of the contact',
-	`name` varchar(255) NOT NULL DEFAULT '' COMMENT 'Name that this contact is known by',
-	`nick` varchar(255) NOT NULL DEFAULT '' COMMENT 'Nick- and user name of the contact',
-	`location` varchar(255) DEFAULT '' COMMENT '',
-	`about` text COMMENT '',
-	`keywords` text COMMENT 'public keywords (interests) of the contact',
-	`gender` varchar(32) NOT NULL DEFAULT '' COMMENT 'Deprecated',
-	`xmpp` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`attag` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`avatar` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`photo` varchar(255) DEFAULT '' COMMENT 'Link to the profile photo of the contact',
-	`thumb` varchar(255) DEFAULT '' COMMENT 'Link to the profile photo (thumb size)',
-	`micro` varchar(255) DEFAULT '' COMMENT 'Link to the profile photo (micro size)',
-	`site-pubkey` text COMMENT '',
-	`issued-id` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`dfrn-id` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`url` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`nurl` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`addr` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`alias` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`pubkey` text COMMENT 'RSA public key 4096 bit',
-	`prvkey` text COMMENT 'RSA private key 4096 bit',
-	`batch` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`request` varchar(255) COMMENT '',
-	`notify` varchar(255) COMMENT '',
-	`poll` varchar(255) COMMENT '',
-	`confirm` varchar(255) COMMENT '',
-	`poco` varchar(255) COMMENT '',
-	`aes_allow` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`ret-aes` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`usehub` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`subhub` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`hub-verify` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`last-update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last try to update the contact info',
-	`success_update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last successful contact update',
-	`failure_update` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of the last failed update',
-	`name-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`uri-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`avatar-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`term-date` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`last-item` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'date of the last post',
-	`priority` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
-	`blocked` boolean NOT NULL DEFAULT '1' COMMENT 'Node-wide block status',
-	`block_reason` text COMMENT 'Node-wide block reason',
-	`readonly` boolean NOT NULL DEFAULT '0' COMMENT 'posts of the contact are readonly',
-	`writable` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`forum` boolean NOT NULL DEFAULT '0' COMMENT 'contact is a forum',
-	`prv` boolean NOT NULL DEFAULT '0' COMMENT 'contact is a private group',
-	`contact-type` tinyint NOT NULL DEFAULT 0 COMMENT '',
-	`hidden` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`archive` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`pending` boolean NOT NULL DEFAULT '1' COMMENT '',
-	`deleted` boolean NOT NULL DEFAULT '0' COMMENT 'Contact has been deleted',
-	`rating` tinyint NOT NULL DEFAULT 0 COMMENT '',
-	`unsearchable` boolean NOT NULL DEFAULT '0' COMMENT 'Contact prefers to not be searchable',
-	`sensitive` boolean NOT NULL DEFAULT '0' COMMENT 'Contact posts sensitive content',
-	`baseurl` varchar(255) DEFAULT '' COMMENT 'baseurl of the contact',
-	`reason` text COMMENT '',
-	`closeness` tinyint unsigned NOT NULL DEFAULT 99 COMMENT '',
-	`info` mediumtext COMMENT '',
-	`profile-id` int unsigned COMMENT 'Deprecated',
-	`bdyear` varchar(4) NOT NULL DEFAULT '' COMMENT '',
-	`bd` date NOT NULL DEFAULT '0001-01-01' COMMENT '',
-	`notify_new_posts` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`fetch_further_information` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
-	`ffi_keyword_blacklist` text COMMENT '',
-	 PRIMARY KEY(`id`),
-	 INDEX `uid_name` (`uid`,`name`(190)),
-	 INDEX `self_uid` (`self`,`uid`),
-	 INDEX `alias_uid` (`alias`(32),`uid`),
-	 INDEX `pending_uid` (`pending`,`uid`),
-	 INDEX `blocked_uid` (`blocked`,`uid`),
-	 INDEX `uid_rel_network_poll` (`uid`,`rel`,`network`,`poll`(64),`archive`),
-	 INDEX `uid_network_batch` (`uid`,`network`,`batch`(64)),
-	 INDEX `addr_uid` (`addr`(32),`uid`),
-	 INDEX `nurl_uid` (`nurl`(32),`uid`),
-	 INDEX `nick_uid` (`nick`(32),`uid`),
-	 INDEX `dfrn-id` (`dfrn-id`(64)),
-	 INDEX `issued-id` (`issued-id`(64))
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contact table';
-
 --
 -- TABLE contact-relation
 --
@@ -304,7 +380,8 @@ CREATE TABLE IF NOT EXISTS `conversation` (
 CREATE TABLE IF NOT EXISTS `diaspora-interaction` (
 	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
 	`interaction` mediumtext COMMENT 'The Diaspora interaction',
-	 PRIMARY KEY(`uri-id`)
+	 PRIMARY KEY(`uri-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Signed Diaspora Interaction';
 
 --
@@ -422,13 +499,16 @@ CREATE TABLE IF NOT EXISTS `gcontact` (
 	`alias` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`generation` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
 	`server_url` varchar(255) NOT NULL DEFAULT '' COMMENT 'baseurl of the contacts server',
+	`gsid` int unsigned COMMENT 'Global Server ID',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `nurl` (`nurl`(190)),
 	 INDEX `name` (`name`(64)),
 	 INDEX `nick` (`nick`(32)),
 	 INDEX `addr` (`addr`(64)),
 	 INDEX `hide_network_updated` (`hide`,`network`,`updated`),
-	 INDEX `updated` (`updated`)
+	 INDEX `updated` (`updated`),
+	 INDEX `gsid` (`gsid`),
+	FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='global contacts';
 
 --
@@ -482,33 +562,6 @@ CREATE TABLE IF NOT EXISTS `group_member` (
 	 UNIQUE INDEX `gid_contactid` (`gid`,`contact-id`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='privacy groups, member info';
 
---
--- TABLE gserver
---
-CREATE TABLE IF NOT EXISTS `gserver` (
-	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
-	`url` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`nurl` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`version` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`site_name` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`info` text COMMENT '',
-	`register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '',
-	`registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users',
-	`directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)',
-	`poco` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`network` char(4) NOT NULL DEFAULT '' COMMENT '',
-	`platform` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`relay-subscribe` boolean NOT NULL DEFAULT '0' COMMENT 'Has the server subscribed to the relay system',
-	`relay-scope` varchar(10) NOT NULL DEFAULT '' COMMENT 'The scope of messages that the server wants to get',
-	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`last_poco_query` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	 PRIMARY KEY(`id`),
-	 UNIQUE INDEX `nurl` (`nurl`(190))
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Global servers';
-
 --
 -- TABLE gserver-tag
 --
@@ -589,6 +642,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 	`author-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Link to the contact table with uid=0 of the author of this item',
 	`icid` int unsigned COMMENT 'Id of the item-content table entry that contains the whole item content',
 	`iaid` int unsigned COMMENT 'Id of the item-activity table entry that contains the activity data',
+	`vid` smallint unsigned COMMENT 'Id of the verb table entry that contains the activity verbs',
 	`extid` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`post-type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Post type (personal note, bookmark, ...)',
 	`global` boolean NOT NULL DEFAULT '0' COMMENT '',
@@ -666,7 +720,14 @@ CREATE TABLE IF NOT EXISTS `item` (
 	 INDEX `uid_eventid` (`uid`,`event-id`),
 	 INDEX `icid` (`icid`),
 	 INDEX `iaid` (`iaid`),
-	 INDEX `psid_wall` (`psid`,`wall`)
+	 INDEX `psid_wall` (`psid`,`wall`),
+	 INDEX `uri-id` (`uri-id`),
+	 INDEX `parent-uri-id` (`parent-uri-id`),
+	 INDEX `thr-parent-id` (`thr-parent-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`parent-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`thr-parent-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`psid`) REFERENCES `permissionset` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Structure for all posts';
 
 --
@@ -681,7 +742,8 @@ CREATE TABLE IF NOT EXISTS `item-activity` (
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `uri-hash` (`uri-hash`),
 	 INDEX `uri` (`uri`(191)),
-	 INDEX `uri-id` (`uri-id`)
+	 INDEX `uri-id` (`uri-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Activities for items';
 
 --
@@ -711,39 +773,10 @@ CREATE TABLE IF NOT EXISTS `item-content` (
 	 UNIQUE INDEX `uri-plink-hash` (`uri-plink-hash`),
 	 INDEX `uri` (`uri`(191)),
 	 INDEX `plink` (`plink`(191)),
-	 INDEX `uri-id` (`uri-id`)
+	 INDEX `uri-id` (`uri-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Content for all posts';
 
---
--- TABLE item-delivery-data
---
-CREATE TABLE IF NOT EXISTS `item-delivery-data` (
-	`iid` int unsigned NOT NULL COMMENT 'Item id',
-	`postopts` text COMMENT 'External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery',
-	`inform` mediumtext COMMENT 'Additional receivers of the linked item',
-	`queue_count` mediumint NOT NULL DEFAULT 0 COMMENT 'Initial number of delivery recipients, used as item.delivery_queue_count',
-	`queue_done` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries, used as item.delivery_queue_done',
-	`queue_failed` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of unsuccessful deliveries, used as item.delivery_queue_failed',
-	`activitypub` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via ActivityPub',
-	`dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via DFRN',
-	`legacy_dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via legacy DFRN',
-	`diaspora` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via Diaspora',
-	`ostatus` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via OStatus',
-	 PRIMARY KEY(`iid`)
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items';
-
---
--- TABLE item-uri
---
-CREATE TABLE IF NOT EXISTS `item-uri` (
-	`id` int unsigned NOT NULL auto_increment,
-	`uri` varbinary(255) NOT NULL COMMENT 'URI of an item',
-	`guid` varbinary(255) COMMENT 'A unique identifier for an item',
-	 PRIMARY KEY(`id`),
-	 UNIQUE INDEX `uri` (`uri`),
-	 INDEX `guid` (`guid`)
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='URI and GUID for items';
-
 --
 -- TABLE locks
 --
@@ -832,6 +865,8 @@ CREATE TABLE IF NOT EXISTS `notify` (
 	`link` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`iid` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id',
 	`parent` int unsigned NOT NULL DEFAULT 0 COMMENT '',
+	`uri-id` int unsigned COMMENT 'Item-uri id of the related post',
+	`parent-uri-id` int unsigned COMMENT 'Item-uri id of the parent of the related post',
 	`seen` boolean NOT NULL DEFAULT '0' COMMENT '',
 	`verb` varchar(100) NOT NULL DEFAULT '' COMMENT '',
 	`otype` varchar(10) NOT NULL DEFAULT '' COMMENT '',
@@ -850,9 +885,11 @@ CREATE TABLE IF NOT EXISTS `notify-threads` (
 	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
 	`notify-id` int unsigned NOT NULL DEFAULT 0 COMMENT '',
 	`master-parent-item` int unsigned NOT NULL DEFAULT 0 COMMENT '',
+	`master-parent-uri-id` int unsigned COMMENT 'Item-uri id of the parent of the related post',
 	`parent-item` int unsigned NOT NULL DEFAULT 0 COMMENT '',
 	`receiver-uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
-	 PRIMARY KEY(`id`)
+	 PRIMARY KEY(`id`),
+	 INDEX `master-parent-uri-id` (`master-parent-uri-id`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
 
 --
@@ -919,20 +956,6 @@ CREATE TABLE IF NOT EXISTS `pconfig` (
 	 UNIQUE INDEX `uid_cat_k` (`uid`,`cat`,`k`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='personal (per user) configuration storage';
 
---
--- TABLE permissionset
---
-CREATE TABLE IF NOT EXISTS `permissionset` (
-	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
-	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'Owner id of this permission set',
-	`allow_cid` mediumtext COMMENT 'Access Control - list of allowed contact.id \'<19><78>\'',
-	`allow_gid` mediumtext COMMENT 'Access Control - list of allowed groups',
-	`deny_cid` mediumtext COMMENT 'Access Control - list of denied contact.id',
-	`deny_gid` mediumtext COMMENT 'Access Control - list of denied groups',
-	 PRIMARY KEY(`id`),
-	 INDEX `uid_allow_cid_allow_gid_deny_cid_deny_gid` (`allow_cid`(50),`allow_gid`(30),`deny_cid`(50),`deny_gid`(30))
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
-
 --
 -- TABLE photo
 --
@@ -1003,6 +1026,55 @@ CREATE TABLE IF NOT EXISTS `poll_result` (
 	 INDEX `poll_id` (`poll_id`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='data for polls - currently unused';
 
+--
+-- TABLE post-category
+--
+CREATE TABLE IF NOT EXISTS `post-category` (
+	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
+	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
+	`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
+	`tid` int unsigned NOT NULL DEFAULT 0 COMMENT '',
+	 PRIMARY KEY(`uri-id`,`uid`,`type`,`tid`),
+	 INDEX `uri-id` (`tid`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to categories';
+
+--
+-- TABLE post-delivery-data
+--
+CREATE TABLE IF NOT EXISTS `post-delivery-data` (
+	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
+	`postopts` text COMMENT 'External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery',
+	`inform` mediumtext COMMENT 'Additional receivers of the linked item',
+	`queue_count` mediumint NOT NULL DEFAULT 0 COMMENT 'Initial number of delivery recipients, used as item.delivery_queue_count',
+	`queue_done` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries, used as item.delivery_queue_done',
+	`queue_failed` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of unsuccessful deliveries, used as item.delivery_queue_failed',
+	`activitypub` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via ActivityPub',
+	`dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via DFRN',
+	`legacy_dfrn` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via legacy DFRN',
+	`diaspora` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via Diaspora',
+	`ostatus` mediumint NOT NULL DEFAULT 0 COMMENT 'Number of successful deliveries via OStatus',
+	 PRIMARY KEY(`uri-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Delivery data for items';
+
+--
+-- TABLE post-tag
+--
+CREATE TABLE IF NOT EXISTS `post-tag` (
+	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
+	`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
+	`tid` int unsigned NOT NULL DEFAULT 0 COMMENT '',
+	`cid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Contact id of the mentioned public contact',
+	 PRIMARY KEY(`uri-id`,`type`,`tid`,`cid`),
+	 INDEX `tid` (`tid`),
+	 INDEX `cid` (`cid`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`tid`) REFERENCES `tag` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT,
+	FOREIGN KEY (`cid`) REFERENCES `contact` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='post relation to tags';
+
 --
 -- TABLE process
 --
@@ -1093,7 +1165,8 @@ CREATE TABLE IF NOT EXISTS `profile_field` (
 	 PRIMARY KEY(`id`),
 	 INDEX `uid` (`uid`),
 	 INDEX `order` (`order`),
-	 INDEX `psid` (`psid`)
+	 INDEX `psid` (`psid`),
+	FOREIGN KEY (`psid`) REFERENCES `permissionset` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Custom profile fields';
 
 --
@@ -1153,46 +1226,20 @@ CREATE TABLE IF NOT EXISTS `session` (
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='web session storage';
 
 --
--- TABLE sign
+-- TABLE storage
 --
-CREATE TABLE IF NOT EXISTS `sign` (
+CREATE TABLE IF NOT EXISTS `storage` (
-	`id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID',
+	`id` int unsigned NOT NULL auto_increment COMMENT 'Auto incremented image data id',
-	`iid` int unsigned NOT NULL DEFAULT 0 COMMENT 'item.id',
+	`data` longblob NOT NULL COMMENT 'file data',
-	`signed_text` mediumtext COMMENT '',
+	 PRIMARY KEY(`id`)
-	`signature` text COMMENT '',
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Data stored by Database storage backend';
-	`signer` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	 PRIMARY KEY(`id`),
-	 UNIQUE INDEX `iid` (`iid`)
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Diaspora signatures';
-
---
--- TABLE term
---
-CREATE TABLE IF NOT EXISTS `term` (
-	`tid` int unsigned NOT NULL auto_increment COMMENT '',
-	`oid` int unsigned NOT NULL DEFAULT 0 COMMENT '',
-	`otype` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
-	`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
-	`term` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`url` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`guid` varchar(255) NOT NULL DEFAULT '' COMMENT '',
-	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
-	`global` boolean NOT NULL DEFAULT '0' COMMENT '',
-	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
-	 PRIMARY KEY(`tid`),
-	 INDEX `term_type` (`term`(64),`type`),
-	 INDEX `oid_otype_type_term` (`oid`,`otype`,`type`,`term`(32)),
-	 INDEX `uid_otype_type_term_global_created` (`uid`,`otype`,`type`,`term`(32),`global`,`created`),
-	 INDEX `uid_otype_type_url` (`uid`,`otype`,`type`,`url`(64)),
-	 INDEX `guid` (`guid`(64))
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='item taxonomy (categories, tags, etc.) table';
 
 --
 -- TABLE thread
 --
 CREATE TABLE IF NOT EXISTS `thread` (
 	`iid` int unsigned NOT NULL DEFAULT 0 COMMENT 'sequential ID',
+	`uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the item uri',
 	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
 	`contact-id` int unsigned NOT NULL DEFAULT 0 COMMENT '',
 	`owner-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item owner',
@@ -1228,7 +1275,9 @@ CREATE TABLE IF NOT EXISTS `thread` (
 	 INDEX `uid_received` (`uid`,`received`),
 	 INDEX `uid_commented` (`uid`,`commented`),
 	 INDEX `uid_wall_received` (`uid`,`wall`,`received`),
-	 INDEX `private_wall_origin_commented` (`private`,`wall`,`origin`,`commented`)
+	 INDEX `private_wall_origin_commented` (`private`,`wall`,`origin`,`commented`),
+	 INDEX `uri-id` (`uri-id`),
+	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Thread related data';
 
 --
@@ -1241,7 +1290,9 @@ CREATE TABLE IF NOT EXISTS `tokens` (
 	`expires` int NOT NULL DEFAULT 0 COMMENT '',
 	`scope` varchar(200) NOT NULL DEFAULT '' COMMENT '',
 	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
-	 PRIMARY KEY(`id`)
+	 PRIMARY KEY(`id`),
+	 INDEX `client_id` (`client_id`),
+	FOREIGN KEY (`client_id`) REFERENCES `clients` (`client_id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
 
 --
@@ -1334,6 +1385,15 @@ CREATE TABLE IF NOT EXISTS `user-item` (
 	 INDEX `iid_uid` (`iid`,`uid`)
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='User specific item data';
 
+--
+-- TABLE verb
+--
+CREATE TABLE IF NOT EXISTS `verb` (
+	`id` smallint unsigned NOT NULL auto_increment,
+	`name` varchar(100) NOT NULL DEFAULT '' COMMENT '',
+	 PRIMARY KEY(`id`)
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Activity Verbs';
+
 --
 -- TABLE worker-ipc
 --
@@ -1366,12 +1426,227 @@ CREATE TABLE IF NOT EXISTS `workerqueue` (
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Background tasks queue entries';
 
 --
--- TABLE storage
+-- VIEW category-view
 --
-CREATE TABLE IF NOT EXISTS `storage` (
+DROP VIEW IF EXISTS `category-view`;
-	`id` int unsigned NOT NULL auto_increment COMMENT 'Auto incremented image data id',
+CREATE VIEW `category-view` AS SELECT 
-	`data` longblob NOT NULL COMMENT 'file data',
+	`post-category`.`uri-id` AS `uri-id`,
-	 PRIMARY KEY(`id`)
+	`post-category`.`uid` AS `uid`,
-) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Data stored by Database storage backend';
+	`item-uri`.`uri` AS `uri`,
+	`item-uri`.`guid` AS `guid`,
+	`post-category`.`type` AS `type`,
+	`post-category`.`tid` AS `tid`,
+	`tag`.`name` AS `name`,
+	`tag`.`url` AS `url`
+	FROM `post-category`
+			INNER JOIN `item-uri` ON `item-uri`.id = `post-category`.`uri-id`
+			LEFT JOIN `tag` ON `post-category`.`tid` = `tag`.`id`;
+
+--
+-- VIEW tag-view
+--
+DROP VIEW IF EXISTS `tag-view`;
+CREATE VIEW `tag-view` AS SELECT 
+	`post-tag`.`uri-id` AS `uri-id`,
+	`item-uri`.`uri` AS `uri`,
+	`item-uri`.`guid` AS `guid`,
+	`post-tag`.`type` AS `type`,
+	`post-tag`.`tid` AS `tid`,
+	`post-tag`.`cid` AS `cid`,
+	CASE `cid` WHEN 0 THEN `tag`.`name` ELSE `contact`.`name` END AS `name`,
+	CASE `cid` WHEN 0 THEN `tag`.`url` ELSE `contact`.`url` END AS `url`
+	FROM `post-tag`
+			INNER JOIN `item-uri` ON `item-uri`.id = `post-tag`.`uri-id`
+			LEFT JOIN `tag` ON `post-tag`.`tid` = `tag`.`id`
+			LEFT JOIN `contact` ON `post-tag`.`cid` = `contact`.`id`;
+
+--
+-- VIEW owner-view
+--
+DROP VIEW IF EXISTS `owner-view`;
+CREATE VIEW `owner-view` AS SELECT 
+	`contact`.`id` AS `id`,
+	`contact`.`uid` AS `uid`,
+	`contact`.`created` AS `created`,
+	`contact`.`updated` AS `updated`,
+	`contact`.`self` AS `self`,
+	`contact`.`remote_self` AS `remote_self`,
+	`contact`.`rel` AS `rel`,
+	`contact`.`duplex` AS `duplex`,
+	`contact`.`network` AS `network`,
+	`contact`.`protocol` AS `protocol`,
+	`contact`.`name` AS `name`,
+	`contact`.`nick` AS `nick`,
+	`contact`.`location` AS `location`,
+	`contact`.`about` AS `about`,
+	`contact`.`keywords` AS `keywords`,
+	`contact`.`gender` AS `gender`,
+	`contact`.`xmpp` AS `xmpp`,
+	`contact`.`attag` AS `attag`,
+	`contact`.`avatar` AS `avatar`,
+	`contact`.`photo` AS `photo`,
+	`contact`.`thumb` AS `thumb`,
+	`contact`.`micro` AS `micro`,
+	`contact`.`site-pubkey` AS `site-pubkey`,
+	`contact`.`issued-id` AS `issued-id`,
+	`contact`.`dfrn-id` AS `dfrn-id`,
+	`contact`.`url` AS `url`,
+	`contact`.`nurl` AS `nurl`,
+	`contact`.`addr` AS `addr`,
+	`contact`.`alias` AS `alias`,
+	`contact`.`pubkey` AS `pubkey`,
+	`contact`.`prvkey` AS `prvkey`,
+	`contact`.`batch` AS `batch`,
+	`contact`.`request` AS `request`,
+	`contact`.`notify` AS `notify`,
+	`contact`.`poll` AS `poll`,
+	`contact`.`confirm` AS `confirm`,
+	`contact`.`poco` AS `poco`,
+	`contact`.`aes_allow` AS `aes_allow`,
+	`contact`.`ret-aes` AS `ret-aes`,
+	`contact`.`usehub` AS `usehub`,
+	`contact`.`subhub` AS `subhub`,
+	`contact`.`hub-verify` AS `hub-verify`,
+	`contact`.`last-update` AS `last-update`,
+	`contact`.`success_update` AS `success_update`,
+	`contact`.`failure_update` AS `failure_update`,
+	`contact`.`name-date` AS `name-date`,
+	`contact`.`uri-date` AS `uri-date`,
+	`contact`.`avatar-date` AS `avatar-date`,
+	`contact`.`avatar-date` AS `picdate`,
+	`contact`.`term-date` AS `term-date`,
+	`contact`.`last-item` AS `last-item`,
+	`contact`.`priority` AS `priority`,
+	`contact`.`blocked` AS `blocked`,
+	`contact`.`block_reason` AS `block_reason`,
+	`contact`.`readonly` AS `readonly`,
+	`contact`.`writable` AS `writable`,
+	`contact`.`forum` AS `forum`,
+	`contact`.`prv` AS `prv`,
+	`contact`.`contact-type` AS `contact-type`,
+	`contact`.`hidden` AS `hidden`,
+	`contact`.`archive` AS `archive`,
+	`contact`.`pending` AS `pending`,
+	`contact`.`deleted` AS `deleted`,
+	`contact`.`rating` AS `rating`,
+	`contact`.`unsearchable` AS `unsearchable`,
+	`contact`.`sensitive` AS `sensitive`,
+	`contact`.`baseurl` AS `baseurl`,
+	`contact`.`reason` AS `reason`,
+	`contact`.`closeness` AS `closeness`,
+	`contact`.`info` AS `info`,
+	`contact`.`profile-id` AS `profile-id`,
+	`contact`.`bdyear` AS `bdyear`,
+	`contact`.`bd` AS `bd`,
+	`contact`.`notify_new_posts` AS `notify_new_posts`,
+	`contact`.`fetch_further_information` AS `fetch_further_information`,
+	`contact`.`ffi_keyword_denylist` AS `ffi_keyword_denylist`,
+	`user`.`parent-uid` AS `parent-uid`,
+	`user`.`guid` AS `guid`,
+	`user`.`nickname` AS `nickname`,
+	`user`.`email` AS `email`,
+	`user`.`openid` AS `openid`,
+	`user`.`timezone` AS `timezone`,
+	`user`.`language` AS `language`,
+	`user`.`register_date` AS `register_date`,
+	`user`.`login_date` AS `login_date`,
+	`user`.`default-location` AS `default-location`,
+	`user`.`allow_location` AS `allow_location`,
+	`user`.`theme` AS `theme`,
+	`user`.`pubkey` AS `upubkey`,
+	`user`.`prvkey` AS `uprvkey`,
+	`user`.`sprvkey` AS `sprvkey`,
+	`user`.`spubkey` AS `spubkey`,
+	`user`.`verified` AS `verified`,
+	`user`.`blockwall` AS `blockwall`,
+	`user`.`hidewall` AS `hidewall`,
+	`user`.`blocktags` AS `blocktags`,
+	`user`.`unkmail` AS `unkmail`,
+	`user`.`cntunkmail` AS `cntunkmail`,
+	`user`.`notify-flags` AS `notify-flags`,
+	`user`.`page-flags` AS `page-flags`,
+	`user`.`account-type` AS `account-type`,
+	`user`.`prvnets` AS `prvnets`,
+	`user`.`maxreq` AS `maxreq`,
+	`user`.`expire` AS `expire`,
+	`user`.`account_removed` AS `account_removed`,
+	`user`.`account_expired` AS `account_expired`,
+	`user`.`account_expires_on` AS `account_expires_on`,
+	`user`.`expire_notification_sent` AS `expire_notification_sent`,
+	`user`.`def_gid` AS `def_gid`,
+	`user`.`allow_cid` AS `allow_cid`,
+	`user`.`allow_gid` AS `allow_gid`,
+	`user`.`deny_cid` AS `deny_cid`,
+	`user`.`deny_gid` AS `deny_gid`,
+	`user`.`openidserver` AS `openidserver`,
+	`profile`.`publish` AS `publish`,
+	`profile`.`net-publish` AS `net-publish`,
+	`profile`.`hide-friends` AS `hide-friends`,
+	`profile`.`prv_keywords` AS `prv_keywords`,
+	`profile`.`pub_keywords` AS `pub_keywords`,
+	`profile`.`address` AS `address`,
+	`profile`.`locality` AS `locality`,
+	`profile`.`region` AS `region`,
+	`profile`.`postal-code` AS `postal-code`,
+	`profile`.`country-name` AS `country-name`,
+	`profile`.`homepage` AS `homepage`,
+	`profile`.`dob` AS `dob`
+	FROM `user`
+			INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`
+			INNER JOIN `profile` ON `profile`.`uid` = `user`.`uid`;
+
+--
+-- VIEW pending-view
+--
+DROP VIEW IF EXISTS `pending-view`;
+CREATE VIEW `pending-view` AS SELECT 
+	`register`.`id` AS `id`,
+	`register`.`hash` AS `hash`,
+	`register`.`created` AS `created`,
+	`register`.`uid` AS `uid`,
+	`register`.`password` AS `password`,
+	`register`.`language` AS `language`,
+	`register`.`note` AS `note`,
+	`contact`.`self` AS `self`,
+	`contact`.`name` AS `name`,
+	`contact`.`url` AS `url`,
+	`contact`.`micro` AS `micro`,
+	`user`.`email` AS `email`,
+	`contact`.`nick` AS `nick`
+	FROM `register`
+			INNER JOIN `contact` ON `register`.`uid` = `contact`.`uid`
+			INNER JOIN `user` ON `register`.`uid` = `user`.`uid`;
+
+--
+-- VIEW tag-search-view
+--
+DROP VIEW IF EXISTS `tag-search-view`;
+CREATE VIEW `tag-search-view` AS SELECT 
+	`post-tag`.`uri-id` AS `uri-id`,
+	`item`.`id` AS `iid`,
+	`item`.`uri` AS `uri`,
+	`item`.`guid` AS `guid`,
+	`item`.`uid` AS `uid`,
+	`item`.`private` AS `private`,
+	`item`.`wall` AS `wall`,
+	`item`.`origin` AS `origin`,
+	`item`.`gravity` AS `gravity`,
+	`item`.`received` AS `received`,
+	`tag`.`name` AS `name`
+	FROM `post-tag`
+			INNER JOIN `tag` ON `tag`.`id` = `post-tag`.`tid`
+			INNER JOIN `item` ON `item`.`uri-id` = `post-tag`.`uri-id`
+			WHERE `post-tag`.`type` = 1;
+
+--
+-- VIEW workerqueue-view
+--
+DROP VIEW IF EXISTS `workerqueue-view`;
+CREATE VIEW `workerqueue-view` AS SELECT 
+	`process`.`pid` AS `pid`,
+	`workerqueue`.`priority` AS `priority`
+	FROM `process`
+			INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid`
+			WHERE NOT `workerqueue`.`done`;
 
 

doc/API-Twitter.md

@@ -152,19 +152,29 @@ These endpoints use the [Friendica API entities](help/API-Entities).
 - [GET api/friendships/incoming](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friendships-incoming)
     - Unsupported parameters
         - `stringify_ids`
-- [GET api/followers/ids](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-followers-ids)
+
-    - Unsupported parameters:
+- - [GET api/followers/ids](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-followers-ids)
-        - `user_id`: Relationships aren't returned for other users than self
+  - [GET api/followers/list](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-followers-list)
-        - `screen_name`: Relationships aren't returned for other users than self
   - [GET api/friends/ids](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friends-ids)
-    - Unsupported parameters:
+  - [GET api/friends/list](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friends-list)
-        - `user_id`: Relationships aren't returned for other users than self
+    - Additional parameters:
-        - `screen_name`: Relationships aren't returned for other users than self
+        - `since_id`: You can use the `next_cursor` value to load the next page.
+        - `max_id`: You can use the inverse of the `previous_cursor` value to load the previous page.
+    - Unsupported parameter:
+        - `skip_status`: No status is returned even if it isn't set to true.
+    - Caveats:
+        - `cursor` trumps `since_id` trumps `max_id` if any combination is provided. 
+        - `user_id` must be the ID of a contact associated with a local user account.
+        - `screen_name` must be associated with a local user account.
+        - `screen_name` trumps `user_id` if both are provided (undocumented Twitter behavior).
+        - Will succeed but return an empty array for users hiding their contact lists.
 
 
 - [POST api/friendships/destroy](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/post-friendships-destroy)
 
 
+
+
 ## Non-implemented endpoints
 
 - [GET oauth/authenticate](https://developer.twitter.com/en/docs/basics/authentication/api-reference/authenticate)
@@ -188,8 +198,6 @@ These endpoints use the [Friendica API entities](help/API-Entities).
 - [POST lists/subscribers/destroy](https://developer.twitter.com/en/docs/accounts-and-users/create-manage-lists/api-reference/post-lists-subscribers-destroy)
 
 
-- [GET followers/list](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-followers-list)
-- [GET friends/list](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friends-list)
 - [GET friendships/lookup](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friendships-lookup)
 - [GET friendships/no_retweets/ids](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friendships-no_retweets-ids)
 - [GET friendships/outgoing](https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friendships-outgoing)

doc/Addons.md

@@ -466,6 +466,19 @@ Hook data is a `\FastRoute\RouterCollector` object that should be used to add ad
 
 **Notice**: The class whose name is provided in the route handler must be reachable via auto-loader.
 
+### probe_detect
+
+Called before trying to detect the target network of a URL.
+If any registered hook function sets the `result` key of the hook data array, it will be returned immediately.
+Hook functions should also return immediately if the hook data contains an existing result. 
+
+Hook data:
+
+- **uri** (input): the profile URI.
+- **network** (input): the target network (can be empty for auto-detection).
+- **uid** (input): the user to return the contact data for (can be empty for public contacts).
+- **result** (output): Set by the hook function to indicate a successful detection.
+
 ## Complete list of hook callbacks
 
 Here is a complete list of all hook callbacks with file locations (as of 24-Sep-2018). Please see the source for details of any hooks not documented above.

doc/BBCode.md

@@ -65,17 +65,17 @@ table.bbcodes > * > tr > th {
   <td><a href="http://friendi.ca" target="external-link">Friendica</a></td>
 </tr>
 <tr>
-  <td>[img]https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg[/img]</td>
+  <td>[img]https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg[/img]</td>
-  <td><img src="https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg" alt="Immagine/foto"></td>
+  <td><img src="https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg" alt="Immagine/foto"></td>
 </tr>
 <tr>
-  <td>[img=https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg]The Friendica Logo[/img]</td>
+  <td>[img=https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg]The Friendica Logo[/img]</td>
-  <td><img src="https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg" alt="The Friendica Logo"></td>
+  <td><img src="https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg" alt="The Friendica Logo"></td>
 </tr>
 <tr>
-  <td>[img=64x32]https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg[/img]<br>
+  <td>[img=64x32]https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg[/img]<br>
 <br>Note: provided height is simply discarded.</td>
-  <td><img src="https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg" style="width: 64px;"></td>
+  <td><img src="https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg" style="width: 64px;"></td>
 </tr>
 <tr>
   <td>[size=xx-small]small text[/size]</td>
@@ -613,15 +613,34 @@ On Mastodon this field is used for the content warning.
   <th>Result</th>
 </tr>
 <tr>
-  <td>If you need to put literal bbcode in a message, [noparse], [nobb] or [pre] are used to escape bbcode:
+  <td>If you need to put literal BBCode in a message, [noparse], [nobb] or [pre] blocks prevent BBCode conversion:
     <ul>
       <li>[noparse][b]bold[/b][/noparse]</li>
       <li>[nobb][b]bold[/b][/nobb]</li>
       <li>[pre][b]bold[/b][/pre]</li>
     </ul>
+    Note: [code] has priority over [noparse], [nobb] and [pre] which makes them display as BBCode tags in code blocks instead of being removed.
+    [code] blocks inside [noparse] will still be converted to a code block.
   </td>
   <td>[b]bold[/b]</td>
 </tr>
+<tr>
+  <td>Additionally, [noparse] and [pre] blocks prevent mention and hashtag conversion to links:
+    <ul>
+      <li>[noparse]@user@domain.tld #hashtag[/noparse]</li>
+      <li>[pre]@user@domain.tld #hashtag[/pre]</li>
+    </ul>
+  </td>
+  <td>@user@domain.tld #hashtag</td>
+</tr>
+<tr>
+  <td>Additionally, [pre] blocks preserve spaces:
+    <ul>
+      <li>[pre]&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Spaces[/pre]</li>
+    </ul>
+  </td>
+  <td>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Spaces</td>
+</tr>
 <tr>
   <td>[nosmile] is used to disable smilies on a post by post basis<br>
     <br>

doc/Chats.md

@@ -43,7 +43,7 @@ At first you have to get the current version. You can either pull it from [Githu
 
     $> cd /var/www/virtual/YOURSPACE/html/addon; git pull
 
-Or you can download a tar archive here: [jappixmini.tgz](https://github.com/friendica/friendica-addons/blob/master/jappixmini.tgz) (click at „view raw“).
+Or you can download a tar archive here: [jappixmini.tgz](https://github.com/friendica/friendica-addons/blob/stable/jappixmini.tgz) (click at „view raw“).
 
 Just unpack the file and rename the directory to „jappixmini“.
 Next, upload this directory and the .tgz-file into your addon directory of your friendica installation.

doc/Github.md

@@ -22,7 +22,7 @@ Our Git Branches
 
 There are two relevant branches in the main repo on GitHub:
 
-1. master: This branch contains stable releases only.
+1. stable: This branch contains stable releases only.
 2. develop: This branch contains the latest code.
 This is what you want to work with.
 
@@ -43,7 +43,7 @@ Release branches
 A release branch is created when the develop branch contains all features it should have.
 A release branch is used for a few things.
 
-1. It allows last-minute bug fixing before the release goes to master branch.
+1. It allows last-minute bug fixing before the release goes to stable branch.
 2. It allows meta-data changes (README, CHANGELOG, etc.) for version bumps and documentation changes.
 3. It makes sure the develop branch can receive new features that are **not** part of this release.
 

doc/Install.md

@@ -72,7 +72,7 @@ This makes the software much easier to update.
 
 The Linux commands to clone the repository into a directory "mywebsite" would be
 
-    git clone https://github.com/friendica/friendica.git -b master mywebsite
+    git clone https://github.com/friendica/friendica.git -b stable mywebsite
     cd mywebsite
     bin/composer.phar install --no-dev
 
@@ -88,7 +88,7 @@ Get the addons by going into your website folder.
 
 Clone the addon repository (separately):
 
-    git clone https://github.com/friendica/friendica-addons.git -b master addon
+    git clone https://github.com/friendica/friendica-addons.git -b stable addon
 
 If you want to use the development version of Friendica you can switch to the develop branch in the repository by running
 
@@ -435,7 +435,7 @@ provided by one of our members.
 > 
 > This is obvious as soon as you notice that the friendica-cron uses `proc_open`
 > to execute PHP scripts that also use `proc_open`, but it took me quite some time to find that out.
-> I hope this saves some time for other people using suhosin with function blacklists.
+> I hope this saves some time for other people using suhosin with function blocklists.
 
 ### Unable to create all mysql tables on MySQL 5.7.17 or newer
 

doc/Message-Flow.md

@@ -4,7 +4,7 @@ Friendica Message Flow
 This page documents some of the details of how messages get from one person to another in the Friendica network.
 There are multiple paths, using multiple protocols and message formats.
 
-Those attempting to understand these message flows should become familiar with (at the minimum) the [DFRN protocol document](https://github.com/friendica/friendica/blob/master/spec/dfrn2.pdf) and the message passing elements of the OStatus stack (salmon and Pubsubhubbub).
+Those attempting to understand these message flows should become familiar with (at the minimum) the [DFRN protocol document](https://github.com/friendica/friendica/blob/stable/spec/dfrn2.pdf) and the message passing elements of the OStatus stack (salmon and Pubsubhubbub).
 
 Most message passing involves the file include/items.php, which has functions for several feed-related import/export activities. 
 

doc/Settings.md

@@ -74,7 +74,7 @@ You can chose between the following modes:
 ##### Invitation based registry
 
 Additionally to the setting in the admin panel, you can decide if registrations are only possible using an invitation code or not.
-To enable invitation based registration, you have to set the `invitation_only` setting in the [config/local.config.php](/help/Config) file.
+To enable invitation based registration, you have to set the `invitation_only` setting to `true` in the `system` section of the [config/local.config.php](/help/Config) file.
 If you want to use this method, the registration policy has to be set to either *open* or *requires approval*.
 
 #### Check Full Names

doc/Update.md

@@ -8,7 +8,13 @@ Updating Friendica
 If you installed Friendica in the ``path/to/friendica`` folder:
 
 1. Unpack the new Friendica archive in ``path/to/friendica_new``.
-2. Copy ``config/local.config.php``, ``photo/`` and ``proxy/`` from ``path/to/friendica`` to ``path/to/friendica_new``.
+2. Copy  the following items from ``path/to/friendica`` to ``path/to/friendica_new``:
+   * ``config/local.config.php``
+   * ``proxy/``   
+The following items only need to be copied if they are located inside your friendica path:
+   * your storage folder as set in **Admin -> Site -> File Upload -> Storage base path** 
+   * your item cache as set in **Admin -> Site -> Performance -> Path to item cache**
+   * your temp folder as set in **Admin -> Site -> Advanced -> Temp path**
 3. Rename the ``path/to/friendica`` folder to ``path/to/friendica_old``.
 4. Rename the ``path/to/friendica_new`` folder to ``path/to/friendica``.
 5. Check your site. Note: it may go into maintenance mode to update the database schema.
@@ -30,11 +36,11 @@ The addon tree has to be updated separately like so:
     git pull
 
 For both repositories:
-The default branch to use is the ``master`` branch, which is the stable version of Friendica.
+The default branch to use is the ``stable`` branch, which is the stable version of Friendica.
 It is updated about four times a year on a fixed schedule.
 
 If you want to use and test bleeding edge code please checkout the ``develop`` branch.
-The new features and fixes will be merged from ``develop`` into ``master`` after a release candidate period before each release.
+The new features and fixes will be merged from ``develop`` into ``stable`` after a release candidate period before each release.
 
 Warning: The ``develop`` branch is unstable, and breaks on average once a month for at most 24 hours until a patch is submitted and merged.
 Be sure to pull frequently if you choose the ``develop`` branch.

doc/database/db_contact.md

@@ -67,6 +67,6 @@ Table contact
 | bd                        |                                                           | date         | NO   |     | 0001-01-01          |                |
 | notify_new_posts          |                                                           | tinyint(1)   | NO   |     | 0                   |                |
 | fetch_further_information |                                                           | tinyint(1)   | NO   |     | 0                   |                |
-| ffi_keyword_blacklist     |                                                           | mediumtext   | NO   |     | NULL                |                |
+| ffi_keyword_denylist     |                                                            | mediumtext   | NO   |     | NULL                |                |
 
 Return to [database documentation](help/database)

doc/de/BBCode.md

@@ -65,17 +65,17 @@ table.bbcodes > * > tr > th {
   <td><a href="http://friendi.ca" target="external-link">Friendica</a></td>
 </tr>
 <tr>
-  <td>[img]https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg[/img]</td>
+  <td>[img]https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg[/img]</td>
-  <td><img src="https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg" alt="Immagine/foto"></td>
+  <td><img src="https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg" alt="Immagine/foto"></td>
 </tr>
 <tr>
-  <td>[img=https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg]Das Friendica Logo[/img]</td>
+  <td>[img=https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg]Das Friendica Logo[/img]</td>
-  <td><img src="https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg" alt="Das Friendica Logo"></td>
+  <td><img src="https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg" alt="Das Friendica Logo"></td>
 </tr>
 <tr>
-  <td>[img=64x32]https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg[/img]<br>
+  <td>[img=64x32]https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg[/img]<br>
 <br>Note: provided height is simply discarded.</td>
-  <td><img src="https://raw.githubusercontent.com/friendica/friendica/master/images/friendica-32.jpg" style="width: 64px;"></td>
+  <td><img src="https://raw.githubusercontent.com/friendica/friendica/stable/images/friendica-32.jpg" style="width: 64px;"></td>
 </tr>
 <tr>
   <td>[size=xx-small]kleiner Text[/size]</td>

doc/de/Chats.md

@@ -49,7 +49,7 @@ Per Git:
 cd /var/www/<Pfad zu Deiner friendica-Installation>/addon; git pull
 </p>
 
-oder als normaler Download von hier: https://github.com/friendica/friendica-addons/blob/master/jappixmini.tgz (auf „view raw“ klicken)
+oder als normaler Download von hier: https://github.com/friendica/friendica-addons/blob/stable/jappixmini.tgz (auf „view raw“ klicken)
 
 Entpacke diese Datei (ggf. den entpackten Ordner in „jappixmini“ umbenennen) und lade sowohl den entpackten Ordner komplett als auch die .tgz Datei in den Addon Ordner Deiner Friendica Installation hoch.
 

doc/de/Install.md

@@ -55,7 +55,7 @@ Wenn du die Möglichkeit hierzu hast, empfehlen wir dir "git" zu nutzen, um die
 Das macht die Aktualisierung wesentlich einfacher.
 Der Linux-Code, mit dem man die Dateien direkt in ein Verzeichnis wie "meinewebseite" kopiert, ist
 
-    git clone https://github.com/friendica/friendica.git -b master mywebsite
+    git clone https://github.com/friendica/friendica.git -b stable mywebsite
     cd mywebsite
     bin/composer.phar install
 
@@ -70,7 +70,7 @@ Falls Addons installiert werden sollen: Gehe in den Friendica-Ordner
 
 Und die Addon Repository klonst:
 
-    git clone https://github.com/friendica/friendica-addons.git -b master addon
+    git clone https://github.com/friendica/friendica-addons.git -b stable addon
 
 Um das Addon-Verzeichnis aktuell zu halten, solltest du in diesem Pfad ein "git pull"-Befehl eintragen
 

doc/de/Message-Flow.md

@@ -6,7 +6,7 @@ Friendica Nachrichtenfluss
 Diese Seite soll einige Infos darüber dokumentieren, wie Nachrichten innerhalb von Friendica von einer Person zur anderen übertragen werden. 
 Es gibt verschiedene Pfade, die verschiedene Protokolle und Nachrichtenformate nutzen. 
 
-Diejenigen, die den Nachrichtenfluss genauer verstehen wollen, sollten sich mindestens mit dem DFRN-Protokoll ([Dokument mit den DFRN Spezifikationen](https://github.com/friendica/friendica/blob/master/spec/dfrn2.pdf)) und den Elementen zur Nachrichtenverarbeitung des OStatus Stack informieren (salmon und Pubsubhubbub).
+Diejenigen, die den Nachrichtenfluss genauer verstehen wollen, sollten sich mindestens mit dem DFRN-Protokoll ([Dokument mit den DFRN Spezifikationen](https://github.com/friendica/friendica/blob/stable/spec/dfrn2.pdf)) und den Elementen zur Nachrichtenverarbeitung des OStatus Stack informieren (salmon und Pubsubhubbub).
 
 Der Großteil der Nachrichtenverarbeitung nutzt die Datei include/items.php, welche Funktionen für verschiedene Feed-bezogene Import-/Exportaktivitäten liefert.
 

doc/themes.md

@@ -3,7 +3,7 @@
 * [Home](help)
 
 To change the look of friendica you have to touch the themes.
-The current default theme is [Vier](https://github.com/friendica/friendica/tree/master/view/theme/vier) but there are numerous others.
+The current default theme is [Vier](https://github.com/friendica/friendica/tree/stable/view/theme/vier) but there are numerous others.
 Have a look at [friendica-themes.com](http://friendica-themes.com) for an overview of the existing themes.
 In case none of them suits your needs, there are several ways to change a theme.
 

doc/translations.md

@@ -8,7 +8,7 @@ Friendica translations
 The Friendica translation process is based on `gettext` PO files.
 
 Basic worflow:
-1. `xgettext` is used to collect translation strings across the project in the master PO file located in `view/lang/C/messages.po`.
+1. `xgettext` is used to collect translation strings across the project in the authoritative PO file located in `view/lang/C/messages.po`.
 2. This file makes translations strings available at [the Transifex Friendica page](https://www.transifex.com/Friendica/friendica/dashboard/).
 3. The translation itself is done at Transifex by volunteers.
 4. The resulting PO files by languages are manually updated in `view/lang/<language>/messages.po`.

include/api.php

@@ -43,6 +43,7 @@ use Friendica\Model\Notify;
 use Friendica\Model\Photo;
 use Friendica\Model\User;
 use Friendica\Model\UserItem;
+use Friendica\Model\Verb;
 use Friendica\Network\FKOAuth1;
 use Friendica\Network\HTTPException;
 use Friendica\Network\HTTPException\BadRequestException;
@@ -263,7 +264,10 @@ function api_login(App $a)
 		throw new UnauthorizedException("This API requires login");
 	}
 
-	DI::auth()->setForUser($a, $record);
+	// Don't refresh the login date more often than twice a day to spare database writes
+	$login_refresh = strcmp(DateTimeFormat::utc('now - 12 hours'), $record['login_date']) > 0;
+
+	DI::auth()->setForUser($a, $record, false, false, $login_refresh);
 
 	$_SESSION["allow_api"] = true;
 
@@ -331,16 +335,16 @@ function api_call(App $a, App\Arguments $args = null)
 
 				if (!empty($info['auth']) && api_user() === false) {
 					api_login($a);
+					Logger::info(API_LOG_PREFIX . 'username {username}', ['module' => 'api', 'action' => 'call', 'username' => $a->user['username']]);
 				}
 
-				Logger::info(API_LOG_PREFIX . 'username {username}', ['module' => 'api', 'action' => 'call', 'username' => $a->user['username']]);
 				Logger::debug(API_LOG_PREFIX . 'parameters', ['module' => 'api', 'action' => 'call', 'parameters' => $_REQUEST]);
 
 				$stamp =  microtime(true);
 				$return = call_user_func($info['func'], $type);
 				$duration = floatval(microtime(true) - $stamp);
 
-				Logger::info(API_LOG_PREFIX . 'username {username}', ['module' => 'api', 'action' => 'call', 'username' => $a->user['username'], 'duration' => round($duration, 2)]);
+				Logger::info(API_LOG_PREFIX . 'duration {duration}', ['module' => 'api', 'action' => 'call', 'duration' => round($duration, 2)]);
 
 				DI::profiler()->saveLog(DI::logger(), API_LOG_PREFIX . 'performance');
 
@@ -623,7 +627,7 @@ function api_get_user(App $a, $contact_id = null)
 				'name' => $contact["name"],
 				'screen_name' => (($contact['nick']) ? $contact['nick'] : $contact['name']),
 				'location' => ($contact["location"] != "") ? $contact["location"] : ContactSelector::networkToName($contact['network'], $contact['url'], $contact['protocol']),
-				'description' => BBCode::toPlaintext($contact["about"]),
+				'description' => BBCode::toPlaintext($contact["about"] ?? ''),
 				'profile_image_url' => $contact["micro"],
 				'profile_image_url_https' => $contact["micro"],
 				'profile_image_url_profile_size' => $contact["thumb"],
@@ -697,7 +701,7 @@ function api_get_user(App $a, $contact_id = null)
 		'name' => (($uinfo[0]['name']) ? $uinfo[0]['name'] : $uinfo[0]['nick']),
 		'screen_name' => (($uinfo[0]['nick']) ? $uinfo[0]['nick'] : $uinfo[0]['name']),
 		'location' => $location,
-		'description' => BBCode::toPlaintext($description),
+		'description' => BBCode::toPlaintext($description ?? ''),
 		'profile_image_url' => $uinfo[0]['micro'],
 		'profile_image_url_https' => $uinfo[0]['micro'],
 		'profile_image_url_profile_size' => $uinfo[0]["thumb"],
@@ -1240,7 +1244,7 @@ function api_media_upload()
 				"image_type" => $media["type"],
 				"friendica_preview_url" => $media["preview"]];
 
-	Logger::log("Media uploaded: " . print_r($returndata, true), Logger::DEBUG);
+	Logger::info('Media uploaded', ['return' => $returndata]);
 
 	return ["media" => $returndata];
 }
@@ -1310,7 +1314,7 @@ api_register_func('api/media/metadata/create', 'api_media_metadata_create', true
 /**
  * @param string $type    Return format (atom, rss, xml, json)
  * @param int    $item_id
- * @return string
+ * @return array|string
  * @throws Exception
  */
 function api_status_show($type, $item_id)
@@ -1538,34 +1542,27 @@ function api_search($type)
 	$params = ['order' => ['id' => true], 'limit' => [$start, $count]];
 	if (preg_match('/^#(\w+)$/', $searchTerm, $matches) === 1 && isset($matches[1])) {
 		$searchTerm = $matches[1];
-		$condition = ["`oid` > ?
+		$condition = ["`iid` > ? AND `name` = ? AND (NOT `private` OR (`private` AND `uid` = ?))", $since_id, $searchTerm, local_user()];
-			AND (`uid` = 0 OR (`uid` = ? AND NOT `global`)) 
+		$tags = DBA::select('tag-search-view', ['uri-id'], $condition);
-			AND `otype` = ? AND `type` = ? AND `term` = ?",
+		$uriids = [];
-			$since_id, local_user(), TERM_OBJ_POST, TERM_HASHTAG, $searchTerm];
+		while ($tag = DBA::fetch($tags)) {
-		if ($max_id > 0) {
+			$uriids[] = $tag['uri-id'];
-			$condition[0] .= ' AND `oid` <= ?';
-			$condition[] = $max_id;
 		}
-		$terms = DBA::select('term', ['oid'], $condition, []);
+		DBA::close($tags);
-		$itemIds = [];
-		while ($term = DBA::fetch($terms)) {
-			$itemIds[] = $term['oid'];
-		}
-		DBA::close($terms);
 
-		if (empty($itemIds)) {
+		if (empty($uriids)) {
 			return api_format_data('statuses', $type, $data);
 		}
 
-		$preCondition = ['`id` IN (' . implode(', ', $itemIds) . ')'];
+		$condition = ['uri-id' => $uriids];
 		if ($exclude_replies) {
-			$preCondition[] = '`id` = `parent`';
+			$condition['gravity'] = GRAVITY_PARENT;
 		}
 
-		$condition = [implode(' AND ', $preCondition)];
+		$params['group_by'] = ['uri-id'];
 	} else {
 		$condition = ["`id` > ?
-			" . ($exclude_replies ? " AND `id` = `parent` " : ' ') . "
+			" . ($exclude_replies ? " AND `gravity` = " . GRAVITY_PARENT : ' ') . "
 			AND (`uid` = 0 OR (`uid` = ? AND NOT `global`))
 			AND `body` LIKE CONCAT('%',?,'%')",
 			$since_id, api_user(), $_REQUEST['q']];
@@ -1653,7 +1650,8 @@ function api_statuses_home_timeline($type)
 		$condition[] = $max_id;
 	}
 	if ($exclude_replies) {
-		$condition[0] .= ' AND `item`.`parent` = `item`.`id`';
+		$condition[0] .= ' AND `item`.`gravity` = ?';
+		$condition[] = GRAVITY_PARENT;
 	}
 	if ($conversation_id > 0) {
 		$condition[0] .= " AND `item`.`parent` = ?";
@@ -2040,7 +2038,7 @@ function api_statuses_repeat($type)
 
 	Logger::log('API: api_statuses_repeat: '.$id);
 
-	$fields = ['body', 'title', 'attach', 'tag', 'author-name', 'author-link', 'author-avatar', 'guid', 'created', 'plink'];
+	$fields = ['uri-id', 'body', 'title', 'attach', 'author-name', 'author-link', 'author-avatar', 'guid', 'created', 'plink'];
 	$item = Item::selectFirst($fields, ['id' => $id, 'private' => [Item::PUBLIC, Item::UNLISTED]]);
 
 	if (DBA::isResult($item) && $item['body'] != "") {
@@ -2048,7 +2046,7 @@ function api_statuses_repeat($type)
 			$pos = strpos($item['body'], "[share");
 			$post = substr($item['body'], $pos);
 		} else {
-			$post = share_header($item['author-name'], $item['author-link'], $item['author-avatar'], $item['guid'], $item['created'], $item['plink']);
+			$post = BBCode::getShareOpeningTag($item['author-name'], $item['author-link'], $item['author-avatar'], $item['plink'], $item['created'], $item['guid']);
 
 			if (!empty($item['title'])) {
 				$post .= '[h3]' . $item['title'] . "[/h3]\n";
@@ -2058,7 +2056,6 @@ function api_statuses_repeat($type)
 			$post .= "[/share]";
 		}
 		$_REQUEST['body'] = $post;
-		$_REQUEST['tag'] = $item['tag'];
 		$_REQUEST['attach'] = $item['attach'];
 		$_REQUEST['profile_uid'] = api_user();
 		$_REQUEST['api_source'] = true;
@@ -2068,6 +2065,8 @@ function api_statuses_repeat($type)
 		}
 
 		$item_id = item_post($a);
+
+		/// @todo Copy tags from the original post to the new one
 	} else {
 		throw new ForbiddenException();
 	}
@@ -2234,12 +2233,7 @@ function api_statuses_user_timeline($type)
 		throw new ForbiddenException();
 	}
 
-	Logger::log(
+	Logger::info('api_statuses_user_timeline', ['api_user' => api_user(), 'user_info' => $user_info, '_REQUEST' => $_REQUEST]);
-		"api_statuses_user_timeline: api_user: ". api_user() .
-			"\nuser_info: ".print_r($user_info, true) .
-			"\n_REQUEST:  ".print_r($_REQUEST, true),
-		Logger::DEBUG
-	);
 
 	$since_id        = $_REQUEST['since_id'] ?? 0;
 	$max_id          = $_REQUEST['max_id'] ?? 0;
@@ -2260,7 +2254,8 @@ function api_statuses_user_timeline($type)
 	}
 
 	if ($exclude_replies) {
-		$condition[0] .= ' AND `item`.`parent` = `item`.`id`';
+		$condition[0] .= ' AND `item`.`gravity` = ?';
+		$condition[] = GRAVITY_PARENT;
 	}
 
 	if ($conversation_id > 0) {
@@ -2497,10 +2492,10 @@ function api_format_messages($item, $recipient, $sender)
 		if ($_GET['getText'] == 'html') {
 			$ret['text'] = BBCode::convert($item['body'], false);
 		} elseif ($_GET['getText'] == 'plain') {
-			$ret['text'] = trim(HTML::toPlaintext(BBCode::convert(api_clean_plain_items($item['body']), false, 2, true), 0));
+			$ret['text'] = trim(HTML::toPlaintext(BBCode::convert(api_clean_plain_items($item['body']), false, BBCode::API, true), 0));
 		}
 	} else {
-		$ret['text'] = $item['title'] . "\n" . HTML::toPlaintext(BBCode::convert(api_clean_plain_items($item['body']), false, 2, true), 0);
+		$ret['text'] = $item['title'] . "\n" . HTML::toPlaintext(BBCode::convert(api_clean_plain_items($item['body']), false, BBCode::API, true), 0);
 	}
 	if (!empty($_GET['getUserObjects']) && $_GET['getUserObjects'] == 'false') {
 		unset($ret['sender']);
@@ -2526,7 +2521,7 @@ function api_convert_item($item)
 	$attachments = api_get_attachments($body);
 
 	// Workaround for ostatus messages where the title is identically to the body
-	$html = BBCode::convert(api_clean_plain_items($body), false, 2, true);
+	$html = BBCode::convert(api_clean_plain_items($body), false, BBCode::API, true);
 	$statusbody = trim(HTML::toPlaintext($html, 0));
 
 	// handle data: images
@@ -3033,7 +3028,7 @@ function api_format_item($item, $type = "json", $status_user = null, $author_use
 	$retweeted_item = [];
 	$quoted_item = [];
 
-	if ($item["id"] == $item["parent"]) {
+	if ($item['gravity'] == GRAVITY_PARENT) {
 		$body = $item['body'];
 		$retweeted_item = api_share_as_retweet($item);
 		if ($body != $item['body']) {
@@ -3310,7 +3305,8 @@ function api_lists_statuses($type)
 		$condition[] = $max_id;
 	}
 	if ($exclude_replies > 0) {
-		$condition[0] .= ' AND `item`.`parent` = `item`.`id`';
+		$condition[0] .= ' AND `item`.`gravity` = ?';
+		$condition[] = GRAVITY_PARENT;
 	}
 	if ($conversation_id > 0) {
 		$condition[0] .= " AND `item`.`parent` = ?";
@@ -3582,96 +3578,6 @@ function api_statusnet_version($type)
 api_register_func('api/gnusocial/version', 'api_statusnet_version', false);
 api_register_func('api/statusnet/version', 'api_statusnet_version', false);
 
-/**
- *
- * @param string $type Return type (atom, rss, xml, json)
- *
- * @param int $rel A contact relationship constant
- * @return array|string|void
- * @throws BadRequestException
- * @throws ForbiddenException
- * @throws ImagickException
- * @throws InternalServerErrorException
- * @throws UnauthorizedException
- * @todo use api_format_data() to return data
- */
-function api_ff_ids($type, int $rel)
-{
-	if (!api_user()) {
-		throw new ForbiddenException();
-	}
-
-	$a = DI::app();
-
-	api_get_user($a);
-
-	$stringify_ids = $_REQUEST['stringify_ids'] ?? false;
-
-	$contacts = DBA::p("SELECT `pcontact`.`id`
-		FROM `contact`
-		INNER JOIN `contact` AS `pcontact`
-		    ON `contact`.`nurl` = `pcontact`.`nurl`
-		    AND `pcontact`.`uid` = 0
-		WHERE `contact`.`uid` = ?
-		AND NOT `contact`.`self`
-		AND `contact`.`rel` IN (?, ?)",
-		api_user(),
-		$rel,
-		Contact::FRIEND
-	);
-
-	$ids = [];
-	foreach (DBA::toArray($contacts) as $contact) {
-		if ($stringify_ids) {
-			$ids[] = $contact['id'];
-		} else {
-			$ids[] = intval($contact['id']);
-		}
-	}
-
-	return api_format_data('ids', $type, ['id' => $ids]);
-}
-
-/**
- * Returns the ID of every user the user is following.
- *
- * @param string $type Return type (atom, rss, xml, json)
- *
- * @return array|string
- * @throws BadRequestException
- * @throws ForbiddenException
- * @throws ImagickException
- * @throws InternalServerErrorException
- * @throws UnauthorizedException
- * @see https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friends-ids
- */
-function api_friends_ids($type)
-{
-	return api_ff_ids($type, Contact::SHARING);
-}
-
-/**
- * Returns the ID of every user following the user.
- *
- * @param string $type Return type (atom, rss, xml, json)
- *
- * @return array|string
- * @throws BadRequestException
- * @throws ForbiddenException
- * @throws ImagickException
- * @throws InternalServerErrorException
- * @throws UnauthorizedException
- * @see https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-followers-ids
- */
-function api_followers_ids($type)
-{
-	return api_ff_ids($type, Contact::FOLLOWER);
-}
-
-/// @TODO move to top of file or somewhere better
-api_register_func('api/friends/ids', 'api_friends_ids', true);
-api_register_func('api/followers/ids', 'api_followers_ids', true);
-
 /**
  * Sends a new direct message.
  *
@@ -4167,26 +4073,18 @@ function api_fr_photoalbum_delete($type)
 		throw new BadRequestException("no albumname specified");
 	}
 	// check if album is existing
-	$r = q(
+
-		"SELECT DISTINCT `resource-id` FROM `photo` WHERE `uid` = %d AND `album` = '%s'",
+	$photos = DBA::selectToArray('photo', ['resource-id'], ['uid' => api_user(), 'album' => $album], ['group_by' => ['resource-id']]);
-		intval(api_user()),
+	if (!DBA::isResult($photos)) {
-		DBA::escape($album)
-	);
-	if (!DBA::isResult($r)) {
 		throw new BadRequestException("album not available");
 	}
 
+	$resourceIds = array_column($photos, 'resource-id');
+
 	// function for setting the items to "deleted = 1" which ensures that comments, likes etc. are not shown anymore
 	// to the user and the contacts of the users (drop_items() performs the federation of the deletion to other networks
-	foreach ($r as $rr) {
+	$condition = ['uid' => api_user(), 'resource-id' => $resourceIds, 'type' => 'photo'];
-		$condition = ['uid' => local_user(), 'resource-id' => $rr['resource-id'], 'type' => 'photo'];
+	Item::deleteForUser($condition, api_user());
-		$photo_item = Item::selectFirstForUser(local_user(), ['id'], $condition);
-
-		if (!DBA::isResult($photo_item)) {
-			throw new InternalServerErrorException("problem with deleting items occured");
-		}
-		Item::deleteForUser(['id' => $photo_item['id']], api_user());
-	}
 
 	// now let's delete all photos from the album
 	$result = Photo::delete(['uid' => api_user(), 'album' => $album]);
@@ -4463,19 +4361,13 @@ function api_fr_photo_delete($type)
 
 	// return success of deletion or error message
 	if ($result) {
-		// retrieve the id of the parent element (the photo element)
-		$condition = ['uid' => local_user(), 'resource-id' => $photo_id, 'type' => 'photo'];
-		$photo_item = Item::selectFirstForUser(local_user(), ['id'], $condition);
-
-		if (!DBA::isResult($photo_item)) {
-			throw new InternalServerErrorException("problem with deleting items occured");
-		}
 		// function for setting the items to "deleted = 1" which ensures that comments, likes etc. are not shown anymore
 		// to the user and the contacts of the users (drop_items() do all the necessary magic to avoid orphans in database and federate deletion)
-		Item::deleteForUser(['id' => $photo_item['id']], api_user());
+		$condition = ['uid' => api_user(), 'resource-id' => $photo_id, 'type' => 'photo'];
+		Item::deleteForUser($condition, api_user());
 
-		$answer = ['result' => 'deleted', 'message' => 'photo with id `' . $photo_id . '` has been deleted from server.'];
+		$result = ['result' => 'deleted', 'message' => 'photo with id `' . $photo_id . '` has been deleted from server.'];
-		return api_format_data("photo_delete", $type, ['$result' => $answer]);
+		return api_format_data("photo_delete", $type, ['$result' => $result]);
 	} else {
 		throw new InternalServerErrorException("unknown error on deleting photo from database table");
 	}
@@ -4734,13 +4626,8 @@ function save_media_to_database($mediatype, $media, $type, $album, $allow_cid, $
 		}
 	}
 
-	if ($filetype == "") {
+	$filetype = Images::getMimeTypeBySource($src, $filename, $filetype);
-		$filetype = Images::guessType($filename);
+
-	}
-	$imagedata = @getimagesize($src);
-	if ($imagedata) {
-		$filetype = $imagedata['mime'];
-	}
 	Logger::log(
 		"File upload src: " . $src . " - filename: " . $filename .
 		" - size: " . $filesize . " - type: " . $filetype,
@@ -4839,7 +4726,7 @@ function save_media_to_database($mediatype, $media, $type, $album, $allow_cid, $
 		Logger::log("photo upload: new profile image upload ended", Logger::DEBUG);
 	}
 
-	if (isset($r) && $r) {
+	if (!empty($r)) {
 		// create entry in 'item'-table on new uploads to enable users to comment/like/dislike the photo
 		if ($photo_id == null && $mediatype == "photo") {
 			post_photo_item($resource_id, $allow_cid, $deny_cid, $allow_gid, $deny_gid, $filetype, $visibility);
@@ -4986,8 +4873,8 @@ function prepare_photo_data($type, $scale, $photo_id)
 	}
 
 	// retrieve item element for getting activities (like, dislike etc.) related to photo
-	$condition = ['uid' => local_user(), 'resource-id' => $photo_id, 'type' => 'photo'];
+	$condition = ['uid' => api_user(), 'resource-id' => $photo_id, 'type' => 'photo'];
-	$item = Item::selectFirstForUser(local_user(), ['id'], $condition);
+	$item = Item::selectFirst(['id', 'uid', 'uri', 'parent', 'allow_cid', 'deny_cid', 'allow_gid', 'deny_gid'], $condition);
 	if (!DBA::isResult($item)) {
 		throw new NotFoundException('Photo-related item not found.');
 	}
@@ -4996,7 +4883,7 @@ function prepare_photo_data($type, $scale, $photo_id)
 
 	// retrieve comments on photo
 	$condition = ["`parent` = ? AND `uid` = ? AND (`gravity` IN (?, ?) OR `type`='photo')",
-		$item[0]['parent'], api_user(), GRAVITY_PARENT, GRAVITY_COMMENT];
+		$item['parent'], api_user(), GRAVITY_PARENT, GRAVITY_COMMENT];
 
 	$statuses = Item::selectForUser(api_user(), [], $condition);
 
@@ -5016,10 +4903,10 @@ function prepare_photo_data($type, $scale, $photo_id)
 	$data['photo']['friendica_comments'] = $comments;
 
 	// include info if rights on photo and rights on item are mismatching
-	$rights_mismatch = $data['photo']['allow_cid'] != $item[0]['allow_cid'] ||
+	$rights_mismatch = $data['photo']['allow_cid'] != $item['allow_cid'] ||
-		$data['photo']['deny_cid'] != $item[0]['deny_cid'] ||
+		$data['photo']['deny_cid'] != $item['deny_cid'] ||
-		$data['photo']['allow_gid'] != $item[0]['allow_gid'] ||
+		$data['photo']['allow_gid'] != $item['allow_gid'] ||
-		$data['photo']['deny_cid'] != $item[0]['deny_cid'];
+		$data['photo']['deny_gid'] != $item['deny_gid'];
 	$data['photo']['rights_mismatch'] = $rights_mismatch;
 
 	return $data;
@@ -5113,8 +5000,7 @@ function api_get_announce($item)
 	}
 
 	$fields = ['author-id', 'author-name', 'author-link', 'author-avatar'];
-	$activity = Item::activityToIndex(Activity::ANNOUNCE);
+	$condition = ['parent-uri' => $item['uri'], 'gravity' => GRAVITY_ACTIVITY, 'uid' => [0, $item['uid']], 'vid' => Verb::getID(Activity::ANNOUNCE)];
-	$condition = ['parent-uri' => $item['uri'], 'gravity' => GRAVITY_ACTIVITY, 'uid' => [0, $item['uid']], 'activity' => $activity];
 	$announce = Item::selectFirstForUser($item['uid'], $fields, $condition, ['order' => ['received' => true]]);
 	if (!DBA::isResult($announce)) {
 		return [];
@@ -5210,7 +5096,7 @@ function api_in_reply_to($item)
 	$in_reply_to['user_id_str'] = null;
 	$in_reply_to['screen_name'] = null;
 
-	if (($item['thr-parent'] != $item['uri']) && (intval($item['parent']) != intval($item['id']))) {
+	if (($item['thr-parent'] != $item['uri']) && ($item['gravity'] != GRAVITY_PARENT)) {
 		$parent = Item::selectFirst(['id'], ['uid' => $item['uid'], 'uri' => $item['thr-parent']]);
 		if (DBA::isResult($parent)) {
 			$in_reply_to['status_id'] = intval($parent['id']);

include/conversation.php

@@ -22,7 +22,6 @@
 use Friendica\App;
 use Friendica\Content\ContactSelector;
 use Friendica\Content\Feature;
-use Friendica\Content\Pager;
 use Friendica\Content\Text\BBCode;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
@@ -34,7 +33,8 @@ use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Item;
 use Friendica\Model\Profile;
-use Friendica\Model\Term;
+use Friendica\Model\Tag;
+use Friendica\Model\Verb;
 use Friendica\Object\Post;
 use Friendica\Object\Thread;
 use Friendica\Protocol\Activity;
@@ -144,103 +144,12 @@ function localize_item(&$item)
 		$item['body'] = item_redir_and_replace_images($extracted['body'], $extracted['images'], $item['contact-id']);
 	}
 
-	/*
+	/// @todo The following functionality needs to be cleaned up. 
-	heluecht 2018-06-19: from my point of view this whole code part is useless.
+	if (!empty($item['verb'])) {
-	It just renders the body message of technical posts (Like, dislike, ...).
-	But: The body isn't visible at all. So we do this stuff just because we can.
-	Even if these messages were visible, this would only mean that something went wrong.
-	During the further steps of the database restructuring I would like to address this issue.
-	*/
-
 		$activity = DI::activity();
 
 		$xmlhead = "<" . "?xml version='1.0' encoding='UTF-8' ?" . ">";
-	if ($activity->match($item['verb'], Activity::LIKE)
-		|| $activity->match($item['verb'], Activity::DISLIKE)
-		|| $activity->match($item['verb'], Activity::ATTEND)
-		|| $activity->match($item['verb'], Activity::ATTENDNO)
-		|| $activity->match($item['verb'], Activity::ATTENDMAYBE)) {
 
-		$fields = ['author-link', 'author-name', 'verb', 'object-type', 'resource-id', 'body', 'plink'];
-		$obj = Item::selectFirst($fields, ['uri' => $item['parent-uri']]);
-		if (!DBA::isResult($obj)) {
-			return;
-		}
-
-		$author  = '[url=' . $item['author-link'] . ']' . $item['author-name'] . '[/url]';
-		$objauthor =  '[url=' . $obj['author-link'] . ']' . $obj['author-name'] . '[/url]';
-
-		switch ($obj['verb']) {
-			case Activity::POST:
-				switch ($obj['object-type']) {
-					case Activity\ObjectType::EVENT:
-						$post_type = DI::l10n()->t('event');
-						break;
-					default:
-						$post_type = DI::l10n()->t('status');
-				}
-				break;
-			default:
-				if ($obj['resource-id']) {
-					$post_type = DI::l10n()->t('photo');
-					$m = [];
-					preg_match("/\[url=([^]]*)\]/", $obj['body'], $m);
-					$rr['plink'] = $m[1];
-				} else {
-					$post_type = DI::l10n()->t('status');
-				}
-		}
-
-		$plink = '[url=' . $obj['plink'] . ']' . $post_type . '[/url]';
-
-		$bodyverb = '';
-		if ($activity->match($item['verb'], Activity::LIKE)) {
-			$bodyverb = DI::l10n()->t('%1$s likes %2$s\'s %3$s');
-		} elseif ($activity->match($item['verb'], Activity::DISLIKE)) {
-			$bodyverb = DI::l10n()->t('%1$s doesn\'t like %2$s\'s %3$s');
-		} elseif ($activity->match($item['verb'], Activity::ATTEND)) {
-			$bodyverb = DI::l10n()->t('%1$s attends %2$s\'s %3$s');
-		} elseif ($activity->match($item['verb'], Activity::ATTENDNO)) {
-			$bodyverb = DI::l10n()->t('%1$s doesn\'t attend %2$s\'s %3$s');
-		} elseif ($activity->match($item['verb'], Activity::ATTENDMAYBE)) {
-			$bodyverb = DI::l10n()->t('%1$s attends maybe %2$s\'s %3$s');
-		}
-
-		$item['body'] = sprintf($bodyverb, $author, $objauthor, $plink);
-	}
-
-	if ($activity->match($item['verb'], Activity::FRIEND)) {
-
-		if ($item['object-type']=="" || $item['object-type']!== Activity\ObjectType::PERSON) return;
-
-		$Aname = $item['author-name'];
-		$Alink = $item['author-link'];
-
-		$xmlhead="<"."?xml version='1.0' encoding='UTF-8' ?".">";
-
-		$obj = XML::parseString($xmlhead.$item['object']);
-		$links = XML::parseString($xmlhead."<links>".XML::unescape($obj->link)."</links>");
-
-		$Bname = $obj->title;
-		$Blink = "";
-		$Bphoto = "";
-		foreach ($links->link as $l) {
-			$atts = $l->attributes();
-			switch ($atts['rel']) {
-				case "alternate": $Blink = $atts['href']; break;
-				case "photo": $Bphoto = $atts['href']; break;
-			}
-		}
-
-		$A = '[url=' . Contact::magicLink($Alink) . ']' . $Aname . '[/url]';
-		$B = '[url=' . Contact::magicLink($Blink) . ']' . $Bname . '[/url]';
-		if ($Bphoto != "") {
-			$Bphoto = '[url=' . Contact::magicLink($Blink) . '][img]' . $Bphoto . '[/img][/url]';
-		}
-
-		$item['body'] = DI::l10n()->t('%1$s is now friends with %2$s', $A, $B)."\n\n\n".$Bphoto;
-
-	}
 		if (stristr($item['verb'], Activity::POKE)) {
 			$verb = urldecode(substr($item['verb'], strpos($item['verb'],'#') + 1));
 			if (!$verb) {
@@ -253,8 +162,6 @@ function localize_item(&$item)
 			$Aname = $item['author-name'];
 			$Alink = $item['author-link'];
 
-		$xmlhead = "<" . "?xml version='1.0' encoding='UTF-8' ?" . ">";
-
 			$obj = XML::parseString($xmlhead . $item['object']);
 
 			$Bname = $obj->title;
@@ -282,7 +189,7 @@ function localize_item(&$item)
 			$txt = DI::l10n()->t('%1$s poked %2$s');
 
 			// now translate the verb
-		$poked_t = trim(sprintf($txt, "", ""));
+			$poked_t = trim(sprintf($txt, '', ''));
 			$txt = str_replace($poked_t, DI::l10n()->t($verb), $txt);
 
 			// then do the sprintf on the translation string
@@ -335,31 +242,8 @@ function localize_item(&$item)
 			$tag = sprintf('#[url=%s]%s[/url]', $parsedobj->id, $parsedobj->content);
 			$item['body'] = DI::l10n()->t('%1$s tagged %2$s\'s %3$s with %4$s', $author, $objauthor, $plink, $tag);
 		}
-
-	if ($activity->match($item['verb'], Activity::FAVORITE)) {
-		if ($item['object-type'] == "") {
-			return;
 	}
 
-		$Aname = $item['author-name'];
-		$Alink = $item['author-link'];
-
-		$xmlhead = "<" . "?xml version='1.0' encoding='UTF-8' ?" . ">";
-
-		$obj = XML::parseString($xmlhead.$item['object']);
-		if (strlen($obj->id)) {
-			$fields = ['author-link', 'author-name', 'plink'];
-			$target = Item::selectFirst($fields, ['uri' => $obj->id, 'uid' => $item['uid']]);
-			if (DBA::isResult($target) && $target['plink']) {
-				$Bname = $target['author-name'];
-				$Blink = $target['author-link'];
-				$A = '[url=' . Contact::magicLink($Alink) . ']' . $Aname . '[/url]';
-				$B = '[url=' . Contact::magicLink($Blink) . ']' . $Bname . '[/url]';
-				$P = '[url=' . $target['plink'] . ']' . DI::l10n()->t('post/item') . '[/url]';
-				$item['body'] = DI::l10n()->t('%1$s marked %2$s\'s %3$s as favorite', $A, $B, $P)."\n";
-			}
-		}
-	}
 	$matches = null;
 	if (preg_match_all('/@\[url=(.*?)\]/is', $item['body'], $matches, PREG_SET_ORDER)) {
 		foreach ($matches as $mtch) {
@@ -493,7 +377,7 @@ function conversation(App $a, array $items, $mode, $update, $preview = false, $o
 				. "<script> var profile_uid = " . $_SESSION['uid']
 				. "; var netargs = '" . substr(DI::args()->getCommand(), 8)
 				. '?f='
-				. (!empty($_GET['cid'])    ? '&cid='    . rawurlencode($_GET['cid'])    : '')
+				. (!empty($_GET['contactid']) ? '&contactid=' . rawurlencode($_GET['contactid']) : '')
 				. (!empty($_GET['search'])    ? '&search='    . rawurlencode($_GET['search'])    : '')
 				. (!empty($_GET['star'])      ? '&star='      . rawurlencode($_GET['star'])      : '')
 				. (!empty($_GET['order'])     ? '&order='     . rawurlencode($_GET['order'])     : '')
@@ -643,7 +527,7 @@ function conversation(App $a, array $items, $mode, $update, $preview = false, $o
 					$profile_name = $item['author-link'];
 				}
 
-				$tags = Term::populateTagsFromItem($item);
+				$tags = Tag::populateFromItem($item);
 
 				$author = ['uid' => 0, 'id' => $item['author-id'],
 					'network' => $item['author-network'], 'url' => $item['author-link']];
@@ -787,7 +671,7 @@ function conversation(App $a, array $items, $mode, $update, $preview = false, $o
 
 				$item['pagedrop'] = $page_dropping;
 
-				if ($item['id'] == $item['parent']) {
+				if ($item['gravity'] == GRAVITY_PARENT) {
 					$item_object = new Post($item);
 					$conv->addParent($item_object);
 				}
@@ -876,7 +760,11 @@ function conversation_fetch_comments($thread_items, $pinned) {
  * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  */
 function conversation_add_children(array $parents, $block_authors, $order, $uid) {
+	if (count($parents) > 1) {
 		$max_comments = DI::config()->get('system', 'max_comments', 100);
+	} else {
+		$max_comments = DI::config()->get('system', 'max_display_comments', 1000);
+	}
 
 	$params = ['order' => ['uid', 'commented' => true]];
 
@@ -887,8 +775,33 @@ function conversation_add_children(array $parents, $block_authors, $order, $uid)
 	$items = [];
 
 	foreach ($parents AS $parent) {
-		$condition = ["`item`.`parent-uri` = ? AND `item`.`uid` IN (0, ?) ",
+		$condition = ["`item`.`parent-uri` = ? AND `item`.`uid` IN (0, ?) AND (`vid` != ? OR `vid` IS NULL)",
-			$parent['uri'], $uid];
+			$parent['uri'], $uid, Verb::getID(Activity::FOLLOW)];
+		$items = conversation_fetch_items($parent, $items, $condition, $block_authors, $params);
+	}
+
+	foreach ($items as $index => $item) {
+		if ($item['uid'] == 0) {
+			$items[$index]['writable'] = in_array($item['network'], Protocol::FEDERATED);
+		}
+	}
+
+	$items = conv_sort($items, $order);
+
+	return $items;
+}
+
+/**
+ * Fetch conversation items
+ *
+ * @param array $parent
+ * @param array $items
+ * @param array $condition
+ * @param boolean $block_authors
+ * @param array $params
+ * @return array
+ */
+function conversation_fetch_items(array $parent, array $items, array $condition, bool $block_authors, array $params) {
 	if ($block_authors) {
 		$condition[0] .= " AND NOT `author`.`hidden`";
 	}
@@ -900,16 +813,6 @@ function conversation_add_children(array $parents, $block_authors, $order, $uid)
 	if (count($comments) != 0) {
 		$items = array_merge($items, $comments);
 	}
-	}
-
-	foreach ($items as $index => $item) {
-		if ($item['uid'] == 0) {
-			$items[$index]['writable'] = in_array($item['network'], Protocol::FEDERATED);
-		}
-	}
-
-	$items = conv_sort($items, $order);
-
 	return $items;
 }
 
@@ -924,7 +827,7 @@ function item_photo_menu($item) {
 	$block_link = '';
 	$ignore_link = '';
 
-	if (local_user() && local_user() == $item['uid'] && $item['parent'] == $item['id'] && !$item['self']) {
+	if (local_user() && local_user() == $item['uid'] && $item['gravity'] == GRAVITY_PARENT && !$item['self']) {
 		$sub_link = 'javascript:dosubthread(' . $item['id'] . '); return false;';
 	}
 
@@ -953,15 +856,15 @@ function item_photo_menu($item) {
 
 	if (!empty($pcid)) {
 		$contact_url = 'contact/' . $pcid;
-		$posts_link = 'contact/' . $pcid . '/posts';
+		$posts_link  = $contact_url . '/posts';
-		$block_link = 'contact/' . $pcid . '/block';
+		$block_link  = $contact_url . '/block';
-		$ignore_link = 'contact/' . $pcid . '/ignore';
+		$ignore_link = $contact_url . '/ignore';
 	}
 
 	if ($cid && !$item['self']) {
-		$poke_link = 'poke?c=' . $cid;
 		$contact_url = 'contact/' . $cid;
-		$posts_link = 'contact/' . $cid . '/posts';
+		$poke_link   = $contact_url . '/poke';
+		$posts_link  = $contact_url . '/posts';
 
 		if (in_array($network, [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA])) {
 			$pm_url = 'message/new/' . $cid;
@@ -1049,7 +952,7 @@ function builtin_activity_puller($item, &$conv_responses) {
 				return;
 		}
 
-		if (!empty($item['verb']) && DI::activity()->match($item['verb'], $verb) && ($item['id'] != $item['parent'])) {
+		if (!empty($item['verb']) && DI::activity()->match($item['verb'], $verb) && ($item['gravity'] != GRAVITY_PARENT)) {
 			$author = ['uid' => 0, 'id' => $item['author-id'],
 				'network' => $item['author-network'], 'url' => $item['author-link']];
 			$url = Contact::magicLinkByContact($author);
@@ -1295,6 +1198,8 @@ function status_editor(App $a, $x, $notes_cid = 0, $popup = false)
 		//jot nav tab (used in some themes)
 		'$message' => DI::l10n()->t('Message'),
 		'$browser' => DI::l10n()->t('Browser'),
+
+		'$compose_link_title' => DI::l10n()->t('Open Compose page'),
 	]);
 
 
@@ -1317,7 +1222,7 @@ function get_item_children(array &$item_list, array $parent, $recursive = true)
 {
 	$children = [];
 	foreach ($item_list as $i => $item) {
-		if ($item['id'] != $item['parent']) {
+		if ($item['gravity'] != GRAVITY_PARENT) {
 			if ($recursive) {
 				// Fallback to parent-uri if thr-parent is not set
 				$thr_parent = $item['thr-parent'];
@@ -1465,7 +1370,7 @@ function conv_sort(array $item_list, $order)
 
 	// Extract the top level items
 	foreach ($item_array as $item) {
-		if ($item['id'] == $item['parent']) {
+		if ($item['gravity'] == GRAVITY_PARENT) {
 			$parents[] = $item;
 		}
 	}

include/enotify.php

@@ -107,12 +107,24 @@ function notification($params)
 		$item_id = 0;
 	}
 
+	if (isset($params['item']['uri-id'])) {
+		$uri_id = $params['item']['uri-id'];
+	} else {
+		$uri_id = 0;
+	}
+
 	if (isset($params['parent'])) {
 		$parent_id = $params['parent'];
 	} else {
 		$parent_id = 0;
 	}
 
+	if (isset($params['item']['parent-uri-id'])) {
+		$parent_uri_id = $params['item']['parent-uri-id'];
+	} else {
+		$parent_uri_id = 0;
+	}
+
 	$epreamble = '';
 	$preamble  = '';
 	$subject   = '';
@@ -453,18 +465,25 @@ function notification($params)
 	if ($show_in_notification_page) {
 		$notification = DI::notify()->insert([
 			'name'          => $params['source_name'] ?? '',
-			'name_cache' => substr(strip_tags(BBCode::convert($params['source_name'] ?? '')), 0, 255),
+			'name_cache'    => substr(strip_tags(BBCode::convert($params['source_name'])), 0, 255),
 			'url'           => $params['source_link'] ?? '',
 			'photo'         => $params['source_photo'] ?? '',
 			'link'          => $itemlink ?? '',
 			'uid'           => $params['uid'] ?? 0,
-			'iid'        => $item_id ?? 0,
+			'iid'           => $item_id,
-			'parent'     => $parent_id ?? 0,
+			'uri-id'        => $uri_id,
+			'parent'        => $parent_id,
+			'parent-uri-id' => $parent_uri_id,
 			'type'          => $params['type'] ?? '',
 			'verb'          => $params['verb'] ?? '',
 			'otype'         => $params['otype'] ?? '',
 		]);
 
+		// Notification insertion can be intercepted by an addon registering the 'enotify_store' hook
+		if (!$notification) {
+			return false;
+		}
+
 		$notification->msg = Renderer::replaceMacros($epreamble, ['$itemlink' => $notification->link]);
 
 		DI::notify()->update($notification);
@@ -487,6 +506,7 @@ function notification($params)
 				Logger::log("notify_id:" . intval($notify_id) . ", parent: " . intval($params['parent']) . "uid: " . intval($params['uid']), Logger::DEBUG);
 
 				$fields = ['notify-id' => $notify_id, 'master-parent-item' => $params['parent'],
+					'master-parent-uri-id' => $parent_uri_id,
 					'receiver-uid' => $params['uid'], 'parent-item' => 0];
 				DBA::insert('notify-threads', $fields);
 
@@ -574,7 +594,7 @@ function check_user_notification($itemid) {
  * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  */
 function check_item_notification($itemid, $uid, $notification_type) {
-	$fields = ['id', 'mention', 'tag', 'parent', 'title', 'body',
+	$fields = ['id', 'uri-id', 'mention', 'parent', 'parent-uri-id', 'title', 'body',
 		'author-link', 'author-name', 'author-avatar', 'author-id',
 		'guid', 'parent-uri', 'uri', 'contact-id', 'network'];
 	$condition = ['id' => $itemid, 'gravity' => [GRAVITY_PARENT, GRAVITY_COMMENT], 'deleted' => false];

include/items.php

@@ -19,433 +19,56 @@
  *
  */
 
-use Friendica\Core\Hook;
+/**
-use Friendica\Core\Logger;
+ * @deprecated since 2020.06
-use Friendica\Core\Protocol;
+ * @see \Friendica\Content\PageInfo::getFooterFromData
-use Friendica\Core\Renderer;
+ */
-use Friendica\Core\Session;
-use Friendica\Database\DBA;
-use Friendica\DI;
-use Friendica\Model\Item;
-use Friendica\Protocol\DFRN;
-use Friendica\Protocol\Feed;
-use Friendica\Protocol\OStatus;
-use Friendica\Util\Network;
-use Friendica\Util\ParseUrl;
-use Friendica\Util\Strings;
-
-require_once __DIR__ . '/../mod/share.php';
-
 function add_page_info_data(array $data, $no_photos = false)
 {
-	Hook::callAll('page_info_data', $data);
+	return "\n" . \Friendica\Content\PageInfo::getFooterFromData($data, $no_photos);
-
-	if (empty($data['type'])) {
-		return '';
-	}
-
-	// It maybe is a rich content, but if it does have everything that a link has,
-	// then treat it that way
-	if (($data["type"] == "rich") && is_string($data["title"]) &&
-		is_string($data["text"]) && !empty($data["images"])) {
-		$data["type"] = "link";
-	}
-
-	$data["title"] = $data["title"] ?? '';
-
-	if ((($data["type"] != "link") && ($data["type"] != "video") && ($data["type"] != "photo")) || ($data["title"] == $data["url"])) {
-		return "";
-	}
-
-	if ($no_photos && ($data["type"] == "photo")) {
-		return "";
-	}
-
-	// Escape some bad characters
-	$data["url"] = str_replace(["[", "]"], ["[", "]"], htmlentities($data["url"], ENT_QUOTES, 'UTF-8', false));
-	$data["title"] = str_replace(["[", "]"], ["[", "]"], htmlentities($data["title"], ENT_QUOTES, 'UTF-8', false));
-
-	$text = "[attachment type='".$data["type"]."'";
-
-	if (empty($data["text"])) {
-		$data["text"] = $data["title"];
-	}
-
-	if (empty($data["text"])) {
-		$data["text"] = $data["url"];
-	}
-
-	if (!empty($data["url"])) {
-		$text .= " url='".$data["url"]."'";
-	}
-
-	if (!empty($data["title"])) {
-		$text .= " title='".$data["title"]."'";
-	}
-
-	// Only embedd a picture link when it seems to be a valid picture ("width" is set)
-	if (!empty($data["images"]) && !empty($data["images"][0]["width"])) {
-		$preview = str_replace(["[", "]"], ["[", "]"], htmlentities($data["images"][0]["src"], ENT_QUOTES, 'UTF-8', false));
-		// if the preview picture is larger than 500 pixels then show it in a larger mode
-		// But only, if the picture isn't higher than large (To prevent huge posts)
-		if (!DI::config()->get('system', 'always_show_preview') && ($data["images"][0]["width"] >= 500)
-			&& ($data["images"][0]["width"] >= $data["images"][0]["height"])) {
-			$text .= " image='".$preview."'";
-		} else {
-			$text .= " preview='".$preview."'";
-		}
-	}
-
-	$text .= "]".$data["text"]."[/attachment]";
-
-	$hashtags = "";
-	if (isset($data["keywords"]) && count($data["keywords"])) {
-		$hashtags = "\n";
-		foreach ($data["keywords"] as $keyword) {
-			/// @TODO make a positive list of allowed characters
-			$hashtag = str_replace([' ', '+', '/', '.', '#', '@', "'", '"', '’', '`', '(', ')', '„', '“'], '', $keyword);
-			$hashtags .= "#[url=" . DI::baseUrl() . "/search?tag=" . $hashtag . "]" . $hashtag . "[/url] ";
-		}
-	}
-
-	return "\n".$text.$hashtags;
-}
-
-function query_page_info($url, $photo = "", $keywords = false, $keyword_blacklist = "")
-{
-	$data = ParseUrl::getSiteinfoCached($url, true);
-
-	if ($photo != "") {
-		$data["images"][0]["src"] = $photo;
-	}
-
-	Logger::log('fetch page info for ' . $url . ' ' . print_r($data, true), Logger::DEBUG);
-
-	if (!$keywords && isset($data["keywords"])) {
-		unset($data["keywords"]);
-	}
-
-	if (($keyword_blacklist != "") && isset($data["keywords"])) {
-		$list = explode(", ", $keyword_blacklist);
-
-		foreach ($list as $keyword) {
-			$keyword = trim($keyword);
-
-			$index = array_search($keyword, $data["keywords"]);
-			if ($index !== false) {
-				unset($data["keywords"][$index]);
-			}
-		}
-	}
-
-	return $data;
-}
-
-function add_page_keywords($url, $photo = "", $keywords = false, $keyword_blacklist = "")
-{
-	$data = query_page_info($url, $photo, $keywords, $keyword_blacklist);
-
-	$tags = "";
-	if (isset($data["keywords"]) && count($data["keywords"])) {
-		foreach ($data["keywords"] as $keyword) {
-			$hashtag = str_replace([" ", "+", "/", ".", "#", "'"],
-				["", "", "", "", "", ""], $keyword);
-
-			if ($tags != "") {
-				$tags .= ", ";
-			}
-
-			$tags .= "#[url=" . DI::baseUrl() . "/search?tag=" . $hashtag . "]" . $hashtag . "[/url]";
-		}
-	}
-
-	return $tags;
-}
-
-function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_blacklist = "")
-{
-	$data = query_page_info($url, $photo, $keywords, $keyword_blacklist);
-
-	$text = '';
-
-	if (is_array($data)) {
-		$text = add_page_info_data($data, $no_photos);
-	}
-
-	return $text;
-}
-
-function add_page_info_to_body($body, $texturl = false, $no_photos = false)
-{
-	Logger::log('add_page_info_to_body: fetch page info for body ' . $body, Logger::DEBUG);
-
-	$URLSearchString = "^\[\]";
-
-	// Fix for Mastodon where the mentions are in a different format
-	$body = preg_replace("/\[url\=([$URLSearchString]*)\]([#!@])(.*?)\[\/url\]/ism",
-		'$2[url=$1]$3[/url]', $body);
-
-	// Adding these spaces is a quick hack due to my problems with regular expressions :)
-	preg_match("/[^!#@]\[url\]([$URLSearchString]*)\[\/url\]/ism", " " . $body, $matches);
-
-	if (!$matches) {
-		preg_match("/[^!#@]\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism", " " . $body, $matches);
-	}
-
-	// Convert urls without bbcode elements
-	if (!$matches && $texturl) {
-		preg_match("/([^\]\='".'"'."]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", " ".$body, $matches);
-
-		// Yeah, a hack. I really hate regular expressions :)
-		if ($matches) {
-			$matches[1] = $matches[2];
-		}
-	}
-
-	if ($matches) {
-		$footer = add_page_info($matches[1], $no_photos);
-	}
-
-	// Remove the link from the body if the link is attached at the end of the post
-	if (isset($footer) && (trim($footer) != "") && (strpos($footer, $matches[1]))) {
-		$removedlink = trim(str_replace($matches[1], "", $body));
-		if (($removedlink == "") || strstr($body, $removedlink)) {
-			$body = $removedlink;
-		}
-
-		$removedlink = preg_replace("/\[url\=" . preg_quote($matches[1], '/') . "\](.*?)\[\/url\]/ism", '', $body);
-		if (($removedlink == "") || strstr($body, $removedlink)) {
-			$body = $removedlink;
-		}
-	}
-
-	// Add the page information to the bottom
-	if (isset($footer) && (trim($footer) != "")) {
-		$body .= $footer;
-	}
-
-	return $body;
 }
 
 /**
- *
+ * @deprecated since 2020.06
- * consume_feed - process atom feed and update anything/everything we might need to update
+ * @see \Friendica\Content\PageInfo::queryUrl
- *
+ */
- * $xml = the (atom) feed to consume - RSS isn't as fully supported but may work for simple feeds.
+function query_page_info($url, $photo = "", $keywords = false, $keyword_denylist = "")
- *
+{
- * $importer = the contact_record (joined to user_record) of the local user who owns this relationship.
+	return \Friendica\Content\PageInfo::queryUrl($url, $photo, $keywords, $keyword_denylist);
- *             It is this person's stuff that is going to be updated.
+}
- * $contact =  the person who is sending us stuff. If not set, we MAY be processing a "follow" activity
+
- *             from an external network and MAY create an appropriate contact record. Otherwise, we MUST
+/**
- *             have a contact record.
+ * @deprecated since 2020.06
- * $hub = should we find a hub declation in the feed, pass it back to our calling process, who might (or
+ * @see \Friendica\Content\PageInfo::getTagsFromUrl()
- *        might not) try and subscribe to it.
+ */
- * $datedir sorts in reverse order
+function get_page_keywords($url, $photo = "", $keywords = false, $keyword_denylist = "")
- * $pass - by default ($pass = 0) we cannot guarantee that a parent item has been
+{
- *      imported prior to its children being seen in the stream unless we are certain
+	return $keywords ? \Friendica\Content\PageInfo::getTagsFromUrl($url, $photo, $keyword_denylist) : [];
- *      of how the feed is arranged/ordered.
+}
- * With $pass = 1, we only pull parent items out of the stream.
+
- * With $pass = 2, we only pull children (comments/likes).
+/**
- *
+ * @deprecated since 2020.06
- * So running this twice, first with pass 1 and then with pass 2 will do the right
+ * @see \Friendica\Content\PageInfo::getFooterFromUrl
- * thing regardless of feed ordering. This won't be adequate in a fully-threaded
+ */
- * model where comments can have sub-threads. That would require some massive sorting
+function add_page_info($url, $no_photos = false, $photo = "", $keywords = false, $keyword_denylist = "")
- * to get all the feed items into a mostly linear ordering, and might still require
+{
- * recursion.
+	return "\n" . \Friendica\Content\PageInfo::getFooterFromUrl($url, $no_photos, $photo, $keywords, $keyword_denylist);
- *
+}
- * @param       $xml
+
- * @param array $importer
+/**
- * @param array $contact
+ * @deprecated since 2020.06
- * @param       $hub
+ * @see \Friendica\Content\PageInfo::appendToBody
- * @throws ImagickException
+ */
- * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+function add_page_info_to_body($body, $texturl = false, $no_photos = false)
+{
+	return \Friendica\Content\PageInfo::appendToBody($body, $texturl, $no_photos);
+}
+
+/**
+ * @deprecated since 2020.06
+ * @see \Friendica\Protocol\Feed::consume
  */
 function consume_feed($xml, array $importer, array $contact, &$hub)
 {
-	if ($contact['network'] === Protocol::OSTATUS) {
+	\Friendica\Protocol\Feed::consume($xml, $importer, $contact, $hub);
-		Logger::log("Consume OStatus messages ", Logger::DEBUG);
-		OStatus::import($xml, $importer, $contact, $hub);
-
-		return;
-	}
-
-	if ($contact['network'] === Protocol::FEED) {
-		Logger::log("Consume feeds", Logger::DEBUG);
-		Feed::import($xml, $importer, $contact);
-
-		return;
-	}
-
-	if ($contact['network'] === Protocol::DFRN) {
-		Logger::log("Consume DFRN messages", Logger::DEBUG);
-		$dfrn_importer = DFRN::getImporter($contact["id"], $importer["uid"]);
-		if (!empty($dfrn_importer)) {
-			Logger::log("Now import the DFRN feed");
-			DFRN::import($xml, $dfrn_importer, true);
-			return;
-		}
-	}
-}
-
-function subscribe_to_hub($url, array $importer, array $contact, $hubmode = 'subscribe')
-{
-	/*
-	 * Diaspora has different message-ids in feeds than they do
-	 * through the direct Diaspora protocol. If we try and use
-	 * the feed, we'll get duplicates. So don't.
-	 */
-	if ($contact['network'] === Protocol::DIASPORA) {
-		return;
-	}
-
-	// Without an importer we don't have a user id - so we quit
-	if (empty($importer)) {
-		return;
-	}
-
-	$user = DBA::selectFirst('user', ['nickname'], ['uid' => $importer['uid']]);
-
-	// No user, no nickname, we quit
-	if (!DBA::isResult($user)) {
-		return;
-	}
-
-	$push_url = DI::baseUrl() . '/pubsub/' . $user['nickname'] . '/' . $contact['id'];
-
-	// Use a single verify token, even if multiple hubs
-	$verify_token = ((strlen($contact['hub-verify'])) ? $contact['hub-verify'] : Strings::getRandomHex());
-
-	$params= 'hub.mode=' . $hubmode . '&hub.callback=' . urlencode($push_url) . '&hub.topic=' . urlencode($contact['poll']) . '&hub.verify=async&hub.verify_token=' . $verify_token;
-
-	Logger::log('subscribe_to_hub: ' . $hubmode . ' ' . $contact['name'] . ' to hub ' . $url . ' endpoint: '  . $push_url . ' with verifier ' . $verify_token);
-
-	if (!strlen($contact['hub-verify']) || ($contact['hub-verify'] != $verify_token)) {
-		DBA::update('contact', ['hub-verify' => $verify_token], ['id' => $contact['id']]);
-	}
-
-	$postResult = Network::post($url, $params);
-
-	Logger::log('subscribe_to_hub: returns: ' . $postResult->getReturnCode(), Logger::DEBUG);
-
-	return;
-
-}
-
-function drop_items(array $items)
-{
-	$uid = 0;
-
-	if (!Session::isAuthenticated()) {
-		return;
-	}
-
-	if (!empty($items)) {
-		foreach ($items as $item) {
-			$owner = Item::deleteForUser(['id' => $item], local_user());
-
-			if ($owner && !$uid) {
-				$uid = $owner;
-			}
-		}
-	}
-}
-
-function drop_item($id, $return = '')
-{
-	$a = DI::app();
-
-	// locate item to be deleted
-
-	$fields = ['id', 'uid', 'guid', 'contact-id', 'deleted', 'gravity', 'parent'];
-	$item = Item::selectFirstForUser(local_user(), $fields, ['id' => $id]);
-
-	if (!DBA::isResult($item)) {
-		notice(DI::l10n()->t('Item not found.') . EOL);
-		DI::baseUrl()->redirect('network');
-	}
-
-	if ($item['deleted']) {
-		return 0;
-	}
-
-	$contact_id = 0;
-
-	// check if logged in user is either the author or owner of this item
-	if (Session::getRemoteContactID($item['uid']) == $item['contact-id']) {
-		$contact_id = $item['contact-id'];
-	}
-
-	if ((local_user() == $item['uid']) || $contact_id) {
-		// Check if we should do HTML-based delete confirmation
-		if (!empty($_REQUEST['confirm'])) {
-			// <form> can't take arguments in its "action" parameter
-			// so add any arguments as hidden inputs
-			$query = explode_querystring(DI::args()->getQueryString());
-			$inputs = [];
-
-			foreach ($query['args'] as $arg) {
-				if (strpos($arg, 'confirm=') === false) {
-					$arg_parts = explode('=', $arg);
-					$inputs[] = ['name' => $arg_parts[0], 'value' => $arg_parts[1]];
-				}
-			}
-
-			return Renderer::replaceMacros(Renderer::getMarkupTemplate('confirm.tpl'), [
-				'$method' => 'get',
-				'$message' => DI::l10n()->t('Do you really want to delete this item?'),
-				'$extra_inputs' => $inputs,
-				'$confirm' => DI::l10n()->t('Yes'),
-				'$confirm_url' => $query['base'],
-				'$confirm_name' => 'confirmed',
-				'$cancel' => DI::l10n()->t('Cancel'),
-			]);
-		}
-		// Now check how the user responded to the confirmation query
-		if (!empty($_REQUEST['canceled'])) {
-			DI::baseUrl()->redirect('display/' . $item['guid']);
-		}
-
-		$is_comment = ($item['gravity'] == GRAVITY_COMMENT) ? true : false;
-		$parentitem = null;
-		if (!empty($item['parent'])){
-			$fields = ['guid'];
-			$parentitem = Item::selectFirstForUser(local_user(), $fields, ['id' => $item['parent']]);
-		}
-
-		// delete the item
-		Item::deleteForUser(['id' => $item['id']], local_user());
-
-		$return_url = hex2bin($return);
-
-		// removes update_* from return_url to ignore Ajax refresh
-		$return_url = str_replace("update_", "", $return_url);
-
-		// Check if delete a comment
-		if ($is_comment) {
-			// Return to parent guid
-			if (!empty($parentitem)) {
-				DI::baseUrl()->redirect('display/' . $parentitem['guid']);
-				//NOTREACHED
-			}
-			// In case something goes wrong
-			else {
-				DI::baseUrl()->redirect('network');
-				//NOTREACHED
-			}
-		}
-		else {
-			// if unknown location or deleting top level post called from display
-			if (empty($return_url) || strpos($return_url, 'display') !== false) {
-				DI::baseUrl()->redirect('network');
-				//NOTREACHED
-			} else {
-				DI::baseUrl()->redirect($return_url);
-				//NOTREACHED
-			}
-		}
-	} else {
-		notice(DI::l10n()->t('Permission denied.') . EOL);
-		DI::baseUrl()->redirect('display/' . $item['guid']);
-		//NOTREACHED
-	}
 }

library/OAuth1.php

@@ -96,7 +96,7 @@ abstract class OAuthSignatureMethod
 	 * @param OAuthToken    $token
 	 * @return string
 	 */
-	abstract public function build_signature(OAuthRequest $request, OAuthConsumer $consumer, OAuthToken $token);
+	abstract public function build_signature(OAuthRequest $request, OAuthConsumer $consumer, OAuthToken $token = null);
 
 	/**
 	 * Verifies that a given signature is correct
@@ -107,7 +107,7 @@ abstract class OAuthSignatureMethod
 	 * @param string        $signature
 	 * @return bool
 	 */
-	public function check_signature($request, $consumer, $token, $signature)
+	public function check_signature(OAuthRequest $request, OAuthConsumer $consumer, $signature, OAuthToken $token = null)
 	{
 		$built = $this->build_signature($request, $consumer, $token);
 		return ($built == $signature);
@@ -134,7 +134,7 @@ class OAuthSignatureMethod_HMAC_SHA1 extends OAuthSignatureMethod
 	 * @param OAuthToken    $token
 	 * @return string
 	 */
-	public function build_signature(OAuthRequest $request, OAuthConsumer $consumer, OAuthToken $token)
+	public function build_signature(OAuthRequest $request, OAuthConsumer $consumer, OAuthToken $token = null)
 	{
 		$base_string = $request->get_signature_base_string();
 		$request->base_string = $base_string;
@@ -179,7 +179,7 @@ class OAuthSignatureMethod_PLAINTEXT extends OAuthSignatureMethod
 	 * @param $token
 	 * @return string
 	 */
-	public function build_signature(OAuthRequest $request, OAuthConsumer $consumer, OAuthToken $token)
+	public function build_signature(OAuthRequest $request, OAuthConsumer $consumer, OAuthToken $token = null)
 	{
 		$key_parts = array(
 			$consumer->secret,
@@ -223,7 +223,7 @@ abstract class OAuthSignatureMethod_RSA_SHA1 extends OAuthSignatureMethod
 	// Either way should return a string representation of the certificate
 	protected abstract function fetch_private_cert(&$request);
 
-	public function build_signature(OAuthRequest $request, OAuthConsumer $consumer, OAuthToken $token)
+	public function build_signature(OAuthRequest $request, OAuthConsumer $consumer, OAuthToken $token = null)
 	{
 		$base_string = $request->get_signature_base_string();
 		$request->base_string = $base_string;
@@ -243,7 +243,7 @@ abstract class OAuthSignatureMethod_RSA_SHA1 extends OAuthSignatureMethod
 		return base64_encode($signature);
 	}
 
-	public function check_signature($request, $consumer, $token, $signature)
+	public function check_signature(OAuthRequest $request, OAuthConsumer $consumer, $signature, OAuthToken $token = null)
 	{
 		$decoded_sig = base64_decode($signature);
 
@@ -358,7 +358,7 @@ class OAuthRequest
 	 * @param array|null    $parameters
 	 * @return OAuthRequest
 	 */
-	public static function from_consumer_and_token(OAuthConsumer $consumer, OAuthToken $token, $http_method, $http_url, array $parameters = NULL)
+	public static function from_consumer_and_token(OAuthConsumer $consumer, $http_method, $http_url, array $parameters = null, OAuthToken $token = null)
 	{
 		@$parameters or $parameters = array();
 		$defaults = array(
@@ -788,11 +788,10 @@ class OAuthServer
 		$valid_sig = $signature_method->check_signature(
 			$request,
 			$consumer,
-			$token,
+			$signature,
-			$signature
+			$token
 		);
 
-
 		if (!$valid_sig) {
 			throw new OAuthException("Invalid signature");
 		}

mod/cal.php

@@ -37,17 +37,18 @@ use Friendica\Model\Event;
 use Friendica\Model\Item;
 use Friendica\Model\Profile;
 use Friendica\Module\BaseProfile;
+use Friendica\Network\HTTPException;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Temporal;
 
 function cal_init(App $a)
 {
 	if (DI::config()->get('system', 'block_public') && !Session::isAuthenticated()) {
-		throw new \Friendica\Network\HTTPException\ForbiddenException(DI::l10n()->t('Access denied.'));
+		throw new HTTPException\ForbiddenException(DI::l10n()->t('Access denied.'));
 	}
 
 	if ($a->argc < 2) {
-		throw new \Friendica\Network\HTTPException\ForbiddenException(DI::l10n()->t('Access denied.'));
+		throw new HTTPException\ForbiddenException(DI::l10n()->t('Access denied.'));
 	}
 
 	Nav::setSelected('events');
@@ -55,7 +56,7 @@ function cal_init(App $a)
 	$nick = $a->argv[1];
 	$user = DBA::selectFirst('user', [], ['nickname' => $nick, 'blocked' => false]);
 	if (!DBA::isResult($user)) {
-		throw new \Friendica\Network\HTTPException\NotFoundException();
+		throw new HTTPException\NotFoundException();
 	}
 
 	$a->data['user'] = $user;
@@ -67,18 +68,22 @@ function cal_init(App $a)
 		return;
 	}
 
-	$profile = Profile::getByNickname($nick, $a->profile_uid);
+	$a->profile = Profile::getByNickname($nick, $a->profile_uid);
 
-	$account_type = Contact::getAccountType($profile);
+	if (empty($a->profile)) {
+		throw new HTTPException\NotFoundException(DI::l10n()->t('User not found.'));
+	}
+
+	$account_type = Contact::getAccountType($a->profile);
 
 	$tpl = Renderer::getMarkupTemplate('widget/vcard.tpl');
 
 	$vcard_widget = Renderer::replaceMacros($tpl, [
-		'$name' => $profile['name'],
+		'$name' => $a->profile['name'],
-		'$photo' => $profile['photo'],
+		'$photo' => $a->profile['photo'],
-		'$addr' => $profile['addr'] ?: '',
+		'$addr' => $a->profile['addr'] ?: '',
 		'$account_type' => $account_type,
-		'$about' => BBCode::convert($profile['about'] ?: ''),
+		'$about' => BBCode::convert($a->profile['about']),
 	]);
 
 	$cal_widget = Widget\CalendarExport::getHTML();

mod/dfrn_confirm.php

@@ -27,9 +27,9 @@
  *   2. We may be the target or other side of the conversation to scenario 1, and will
  *      interact with that process on our own user's behalf.
  *
- *  @see PDF with dfrn specs: https://github.com/friendica/friendica/blob/master/spec/dfrn2.pdf
+ *  @see PDF with dfrn specs: https://github.com/friendica/friendica/blob/stable/spec/dfrn2.pdf
  *    You also find a graphic which describes the confirmation process at
- *    https://github.com/friendica/friendica/blob/master/spec/dfrn2_contact_confirmation.png
+ *    https://github.com/friendica/friendica/blob/stable/spec/dfrn2_contact_confirmation.png
  */
 
 use Friendica\App;
@@ -214,7 +214,7 @@ function dfrn_confirm_post(App $a, $handsfree = null)
 			$params['page'] = 2;
 		}
 
-		Logger::log('Confirm: posting data to ' . $dfrn_confirm . ': ' . print_r($params, true), Logger::DATA);
+		Logger::debug('Confirm: posting data', ['confirm'  => $dfrn_confirm, 'parameter' => $params]);
 
 		/*
 		 *
@@ -372,9 +372,9 @@ function dfrn_confirm_post(App $a, $handsfree = null)
 		$forum = (($page == 1) ? 1 : 0);
 		$prv   = (($page == 2) ? 1 : 0);
 
-		Logger::log('dfrn_confirm: requestee contacted: ' . $node);
+		Logger::notice('requestee contacted', ['node' => $node]);
 
-		Logger::log('dfrn_confirm: request: POST=' . print_r($_POST, true), Logger::DATA);
+		Logger::debug('request', ['POST' => $_POST]);
 
 		// If $aes_key is set, both of these items require unpacking from the hex transport encoding.
 

mod/dfrn_notify.php

@@ -19,7 +19,7 @@
  *
  * The dfrn notify endpoint
  *
- * @see PDF with dfrn specs: https://github.com/friendica/friendica/blob/master/spec/dfrn2.pdf
+ * @see PDF with dfrn specs: https://github.com/friendica/friendica/blob/stable/spec/dfrn2.pdf
  */
 
 use Friendica\App;

mod/dfrn_poll.php

@@ -379,7 +379,7 @@ function dfrn_poll_post(App $a)
 		// NOTREACHED
 	} else {
 		// Update the writable flag if it changed
-		Logger::log('dfrn_poll: post request feed: ' . print_r($_POST, true), Logger::DATA);
+		Logger::debug('post request feed', ['post' => $_POST]);
 		if ($dfrn_version >= 2.21) {
 			if ($perm === 'rw') {
 				$writable = 1;
@@ -521,7 +521,7 @@ function dfrn_poll_content(App $a)
 			if (strlen($s) && strstr($s, '<?xml')) {
 				$xml = XML::parseString($s);
 
-				Logger::log('dfrn_poll: profile: parsed xml: ' . print_r($xml, true), Logger::DATA);
+				Logger::debug(' profile: parsed', ['xml' => $xml]);
 
 				Logger::log('dfrn_poll: secure profile: challenge: ' . $xml->challenge . ' expecting ' . $hash);
 				Logger::log('dfrn_poll: secure profile: sec: ' . $xml->sec . ' expecting ' . $sec);

mod/dfrn_request.php

@@ -19,9 +19,9 @@
  *
  *Handles communication associated with the issuance of friend requests.
  *
- * @see PDF with dfrn specs: https://github.com/friendica/friendica/blob/master/spec/dfrn2.pdf
+ * @see PDF with dfrn specs: https://github.com/friendica/friendica/blob/stable/spec/dfrn2.pdf
  *    You also find a graphic which describes the confirmation process at
- *    https://github.com/friendica/friendica/blob/master/spec/dfrn2_contact_request.png
+ *    https://github.com/friendica/friendica/blob/stable/spec/dfrn2_contact_request.png
  */
 
 use Friendica\App;
@@ -297,8 +297,8 @@ function dfrn_request_post(App $a)
 		$data = Probe::uri($url);
 		$network = $data["network"];
 
-		// Canonicalise email-style profile locator
+		// Canonicalize email-style profile locator
-		$url = Probe::webfingerDfrn($url, $hcard);
+		$url = Probe::webfingerDfrn($data['url'], $hcard);
 
 		if (substr($url, 0, 5) === 'stat:') {
 			// Every time we detect the remote subscription we define this as OStatus.

mod/display.php

@@ -42,7 +42,7 @@ use Friendica\Util\Strings;
 function display_init(App $a)
 {
 	if (ActivityPub::isRequest()) {
-		Objects::rawContent();
+		Objects::rawContent(['guid' => $a->argv[1] ?? null]);
 	}
 
 	if (DI::config()->get('system', 'block_public') && !Session::isAuthenticated()) {
@@ -54,7 +54,7 @@ function display_init(App $a)
 	$item = null;
 	$item_user = local_user();
 
-	$fields = ['id', 'parent', 'author-id', 'body', 'uid', 'guid'];
+	$fields = ['id', 'parent', 'author-id', 'body', 'uid', 'guid', 'gravity'];
 
 	// If there is only one parameter, then check if this parameter could be a guid
 	if ($a->argc == 2) {
@@ -101,12 +101,12 @@ function display_init(App $a)
 	}
 
 	if (!empty($_SERVER['HTTP_ACCEPT']) && strstr($_SERVER['HTTP_ACCEPT'], 'application/atom+xml')) {
-		Logger::log('Directly serving XML for id '.$item["id"], Logger::DEBUG);
+		Logger::log('Directly serving XML for id '.$item['id'], Logger::DEBUG);
-		displayShowFeed($item["id"], false);
+		displayShowFeed($item['id'], false);
 	}
 
-	if ($item["id"] != $item["parent"]) {
+	if ($item['gravity'] != GRAVITY_PARENT) {
-		$parent = Item::selectFirstForUser($item_user, $fields, ['id' => $item["parent"]]);
+		$parent = Item::selectFirstForUser($item_user, $fields, ['id' => $item['parent']]);
 		$item = $parent ?: $item;
 	}
 
@@ -116,11 +116,7 @@ function display_init(App $a)
 		$nickname = str_replace(Strings::normaliseLink(DI::baseUrl()) . '/profile/', '', Strings::normaliseLink($profiledata['url']));
 
 		if (!empty($a->user['nickname']) && $nickname != $a->user['nickname']) {
-			$profile = DBA::fetchFirst("SELECT `profile`.* , `contact`.`avatar-date` AS picdate, `user`.* FROM `profile`
+			$profile = DBA::selectFirst('owner-view', [], ['nickname' => $nickname]);
-				INNER JOIN `contact` on `contact`.`uid` = `profile`.`uid` INNER JOIN `user` ON `profile`.`uid` = `user`.`uid`
-				WHERE `user`.`nickname` = ? AND `contact`.`self` LIMIT 1",
-				$nickname
-			);
 			if (DBA::isResult($profile)) {
 				$profiledata = $profile;
 			}
@@ -187,6 +183,8 @@ function display_content(App $a, $update = false, $update_uid = 0)
 
 	$item = null;
 
+	$force = (bool)($_REQUEST['force'] ?? false);
+
 	if ($update) {
 		$item_id = $_REQUEST['item_id'];
 		$item = Item::selectFirst(['uid', 'parent', 'parent-uri'], ['id' => $item_id]);
@@ -209,8 +207,8 @@ function display_content(App $a, $update = false, $update_uid = 0)
 				$condition = ['guid' => $a->argv[1], 'uid' => local_user()];
 				$item = Item::selectFirstForUser(local_user(), $fields, $condition);
 				if (DBA::isResult($item)) {
-					$item_id = $item["id"];
+					$item_id = $item['id'];
-					$item_parent = $item["parent"];
+					$item_parent = $item['parent'];
 					$item_parent_uri = $item['parent-uri'];
 				}
 			}
@@ -218,8 +216,8 @@ function display_content(App $a, $update = false, $update_uid = 0)
 			if (($item_parent == 0) && remote_user()) {
 				$item = Item::selectFirst($fields, ['guid' => $a->argv[1], 'private' => Item::PRIVATE, 'origin' => true]);
 				if (DBA::isResult($item) && Contact::isFollower(remote_user(), $item['uid'])) {
-					$item_id = $item["id"];
+					$item_id = $item['id'];
-					$item_parent = $item["parent"];
+					$item_parent = $item['parent'];
 					$item_parent_uri = $item['parent-uri'];
 				}
 			}
@@ -228,8 +226,8 @@ function display_content(App $a, $update = false, $update_uid = 0)
 				$condition = ['private' => [Item::PUBLIC, Item::UNLISTED], 'guid' => $a->argv[1], 'uid' => 0];
 				$item = Item::selectFirstForUser(local_user(), $fields, $condition);
 				if (DBA::isResult($item)) {
-					$item_id = $item["id"];
+					$item_id = $item['id'];
-					$item_parent = $item["parent"];
+					$item_parent = $item['parent'];
 					$item_parent_uri = $item['parent-uri'];
 				}
 			}
@@ -285,7 +283,7 @@ function display_content(App $a, $update = false, $update_uid = 0)
 	}
 
 	// We need the editor here to be able to reshare an item.
-	if ($is_owner) {
+	if ($is_owner && !$update) {
 		$x = [
 			'is_owner' => true,
 			'allow_location' => $a->user['allow_location'],
@@ -308,7 +306,7 @@ function display_content(App $a, $update = false, $update_uid = 0)
 		$unseen = false;
 	}
 
-	if ($update && !$unseen) {
+	if ($update && !$unseen && !$force) {
 		return '';
 	}
 

mod/editpost.php

@@ -132,6 +132,8 @@ function editpost_content(App $a)
 		'$message' => DI::l10n()->t('Message'),
 		'$browser' => DI::l10n()->t('Browser'),
 		'$shortpermset' => DI::l10n()->t('permissions'),
+
+		'$compose_link_title' => DI::l10n()->t('Open Compose page'),
 	]);
 
 	return $o;

mod/events.php

@@ -66,7 +66,7 @@ function events_init(App $a)
 function events_post(App $a)
 {
 
-	Logger::log('post: ' . print_r($_REQUEST, true), Logger::DATA);
+	Logger::debug('post', ['request' => $_REQUEST]);
 
 	if (!local_user()) {
 		return;

mod/fbrowser.php

@@ -39,31 +39,26 @@ function fbrowser_content(App $a)
 
 	switch ($a->argv[1]) {
 		case "image":
-			$path = [["", DI::l10n()->t("Photos")]];
+			$path = ['' => DI::l10n()->t('Photos')];
 			$albums = false;
 			$sql_extra = "";
 			$sql_extra2 = " ORDER BY created DESC LIMIT 0, 10";
 
 			if ($a->argc==2) {
-				$albums = q("SELECT distinct(`album`) AS `album` FROM `photo` WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' ",
+				$photos = q("SELECT distinct(`album`) AS `album` FROM `photo` WHERE `uid` = %d AND `album` != '%s' AND `album` != '%s' ",
 					intval(local_user()),
 					DBA::escape('Contact Photos'),
 					DBA::escape(DI::l10n()->t('Contact Photos'))
 				);
 
-				function _map_folder1($el)
+				$albums = array_column($photos, 'album');
-				{
-					return [bin2hex($el['album']),$el['album']];
-				};
-
-				$albums = array_map("_map_folder1", $albums);
 			}
 
 			if ($a->argc == 3) {
-				$album = hex2bin($a->argv[2]);
+				$album = $a->argv[2];
 				$sql_extra = sprintf("AND `album` = '%s' ", DBA::escape($album));
 				$sql_extra2 = "";
-				$path[] = [$a->argv[2], $album];
+				$path[$album] = $album;
 			}
 
 			$r = q("SELECT `resource-id`, ANY_VALUE(`id`) AS `id`, ANY_VALUE(`filename`) AS `filename`, ANY_VALUE(`type`) AS `type`,

mod/follow.php

@@ -28,6 +28,7 @@ use Friendica\Model\Profile;
 use Friendica\Model\Item;
 use Friendica\Network\Probe;
 use Friendica\Database\DBA;
+use Friendica\Model\User;
 use Friendica\Util\Strings;
 
 function follow_post(App $a)
@@ -40,7 +41,6 @@ function follow_post(App $a)
 		DI::baseUrl()->redirect('contact');
 	}
 
-	$uid = local_user();
 	$url = Probe::cleanURI($_REQUEST['url']);
 	$return_path = 'follow?url=' . urlencode($url);
 
@@ -48,7 +48,7 @@ function follow_post(App $a)
 	// This is just a precaution if maybe this page is called somewhere directly via POST
 	$_SESSION['fastlane'] = $url;
 
-	$result = Contact::createFromProbe($uid, $url, true);
+	$result = Contact::createFromProbe($a->user, $url, true);
 
 	if ($result['success'] == false) {
 		// Possibly it is a remote item and not an account
@@ -95,88 +95,63 @@ function follow_content(App $a)
 	$submit = DI::l10n()->t('Submit Request');
 
 	// Don't try to add a pending contact
-	$r = q("SELECT `pending` FROM `contact` WHERE `uid` = %d AND ((`rel` != %d) OR (`network` = '%s')) AND
+	$user_contact = DBA::selectFirst('contact', ['pending'], ["`uid` = ? AND ((`rel` != ?) OR (`network` = ?)) AND
-		(`nurl` = '%s' OR `alias` = '%s' OR `alias` = '%s') AND
+		(`nurl` = ? OR `alias` = ? OR `alias` = ?) AND `network` != ?", 
-		`network` != '%s' LIMIT 1",
+		$uid, Contact::FOLLOWER, Protocol::DFRN, Strings::normaliseLink($url),
-		intval(local_user()), DBA::escape(Contact::FOLLOWER), DBA::escape(Protocol::DFRN), DBA::escape(Strings::normaliseLink($url)),
+		Strings::normaliseLink($url), $url, Protocol::STATUSNET]);
-		DBA::escape(Strings::normaliseLink($url)), DBA::escape($url), DBA::escape(Protocol::STATUSNET));
 
-	if ($r) {
+	if (DBA::isResult($user_contact)) {
-		if ($r[0]['pending']) {
+		if ($user_contact['pending']) {
 			notice(DI::l10n()->t('You already added this contact.'));
 			$submit = '';
-			//$a->internalRedirect($_SESSION['return_path']);
-			// NOTREACHED
 		}
 	}
 
-	$ret = Probe::uri($url);
+	$contact = Contact::getByURL($url, 0, [], true);
-
+	if (empty($contact)) {
-	$protocol = Contact::getProtocol($ret['url'], $ret['network']);
-
-	if (($protocol == Protocol::DIASPORA) && !DI::config()->get('system', 'diaspora_enabled')) {
-		notice(DI::l10n()->t("Diaspora support isn't enabled. Contact can't be added."));
-		$submit = '';
-		//$a->internalRedirect($_SESSION['return_path']);
-		// NOTREACHED
-	}
-
-	if (($protocol == Protocol::OSTATUS) && DI::config()->get('system', 'ostatus_disabled')) {
-		notice(DI::l10n()->t("OStatus support is disabled. Contact can't be added."));
-		$submit = '';
-		//$a->internalRedirect($_SESSION['return_path']);
-		// NOTREACHED
-	}
-
-	if ($protocol == Protocol::PHANTOM) {
 		// Possibly it is a remote item and not an account
 		follow_remote_item($url);
 
 		notice(DI::l10n()->t("The network type couldn't be detected. Contact can't be added."));
 		$submit = '';
-		//$a->internalRedirect($_SESSION['return_path']);
+		$contact = ['url' => $url, 'network' => Protocol::PHANTOM, 'name' => $url, 'keywords' => ''];
-		// NOTREACHED
+	}
+
+	$protocol = Contact::getProtocol($contact['url'], $contact['network']);
+
+	if (($protocol == Protocol::DIASPORA) && !DI::config()->get('system', 'diaspora_enabled')) {
+		notice(DI::l10n()->t("Diaspora support isn't enabled. Contact can't be added."));
+		$submit = '';
+	}
+
+	if (($protocol == Protocol::OSTATUS) && DI::config()->get('system', 'ostatus_disabled')) {
+		notice(DI::l10n()->t("OStatus support is disabled. Contact can't be added."));
+		$submit = '';
 	}
 
 	if ($protocol == Protocol::MAIL) {
-		$ret['url'] = $ret['addr'];
+		$contact['url'] = $contact['addr'];
 	}
 
-	if (($protocol === Protocol::DFRN) && !DBA::isResult($r)) {
+	if (($protocol === Protocol::DFRN) && !DBA::isResult($contact)) {
-		$request = $ret['request'];
+		$request = $contact['request'];
 		$tpl = Renderer::getMarkupTemplate('dfrn_request.tpl');
 	} else {
 		$request = DI::baseUrl() . '/follow';
 		$tpl = Renderer::getMarkupTemplate('auto_request.tpl');
 	}
 
-	$r = q("SELECT `url` FROM `contact` WHERE `uid` = %d AND `self` LIMIT 1", intval($uid));
+	$owner = User::getOwnerDataById($uid);
-
+	if (empty($owner)) {
-	if (!$r) {
 		notice(DI::l10n()->t('Permission denied.'));
 		DI::baseUrl()->redirect($return_path);
 		// NOTREACHED
 	}
 
-	$myaddr = $r[0]['url'];
+	$myaddr = $owner['url'];
-	$gcontact_id = 0;
 
 	// Makes the connection request for friendica contacts easier
-	$_SESSION['fastlane'] = $ret['url'];
+	$_SESSION['fastlane'] = $contact['url'];
-
-	$r = q("SELECT `id`, `location`, `about`, `keywords` FROM `gcontact` WHERE `nurl` = '%s'",
-		Strings::normaliseLink($ret['url']));
-
-	if (!$r) {
-		$r = [['location' => '', 'about' => '', 'keywords' => '']];
-	} else {
-		$gcontact_id = $r[0]['id'];
-	}
-
-	if ($protocol === Protocol::DIASPORA) {
-		$r[0]['location'] = '';
-		$r[0]['about'] = '';
-	}
 
 	$o = Renderer::replaceMacros($tpl, [
 		'$header'        => DI::l10n()->t('Connect/Follow'),
@@ -188,30 +163,27 @@ function follow_content(App $a)
 		'$cancel'        => DI::l10n()->t('Cancel'),
 
 		'$request'       => $request,
-		'$name'          => $ret['name'],
+		'$name'          => $contact['name'],
-		'$url'           => $ret['url'],
+		'$url'           => $contact['url'],
-		'$zrl'           => Profile::zrl($ret['url']),
+		'$zrl'           => Profile::zrl($contact['url']),
 		'$myaddr'        => $myaddr,
-		'$keywords'      => $r[0]['keywords'],
+		'$keywords'      => $contact['keywords'],
 
-		'$does_know_you' => ['knowyou', DI::l10n()->t('%s knows you', $ret['name'])],
+		'$does_know_you' => ['knowyou', DI::l10n()->t('%s knows you', $contact['name'])],
 		'$addnote_field' => ['dfrn-request-message', DI::l10n()->t('Add a personal note:')],
 	]);
 
 	DI::page()['aside'] = '';
 
-	$profiledata = Contact::getDetailsByURL($ret['url']);
+	if ($protocol != Protocol::PHANTOM) {
-	if ($profiledata) {
+		Profile::load($a, '', $contact, false);
-		Profile::load($a, '', $profiledata, false);
-	}
 
-	if ($gcontact_id <> 0) {
 		$o .= Renderer::replaceMacros(Renderer::getMarkupTemplate('section_title.tpl'),
 			['$title' => DI::l10n()->t('Status Messages and Posts')]
 		);
 
 		// Show last public posts
-		$o .= Contact::getPostsFromUrl($ret['url']);
+		$o .= Contact::getPostsFromUrl($contact['url']);
 	}
 
 	return $o;

mod/item.php

@@ -29,11 +29,12 @@
  */
 
 use Friendica\App;
-use Friendica\Content\Pager;
+use Friendica\Content\Item as ItemHelper;
 use Friendica\Content\Text\BBCode;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
+use Friendica\Core\Renderer;
 use Friendica\Core\Session;
 use Friendica\Core\System;
 use Friendica\Core\Worker;
@@ -46,7 +47,7 @@ use Friendica\Model\FileTag;
 use Friendica\Model\Item;
 use Friendica\Model\Notify\Type;
 use Friendica\Model\Photo;
-use Friendica\Model\Term;
+use Friendica\Model\Tag;
 use Friendica\Network\HTTPException;
 use Friendica\Object\EMail\ItemCCEMail;
 use Friendica\Protocol\Activity;
@@ -67,7 +68,10 @@ function item_post(App $a) {
 
 	if (!empty($_REQUEST['dropitems'])) {
 		$arr_drop = explode(',', $_REQUEST['dropitems']);
-		drop_items($arr_drop);
+		foreach ($arr_drop as $item) {
+			Item::deleteForUser(['id' => $item], $uid);
+		}
+
 		$json = ['success' => 1];
 		System::jsonExit($json);
 	}
@@ -101,14 +105,9 @@ function item_post(App $a) {
 	$toplevel_item_id = intval($_REQUEST['parent'] ?? 0);
 	$thr_parent_uri = trim($_REQUEST['parent_uri'] ?? '');
 
-	$thread_parent_id = 0;
-	$thread_parent_contact = null;
-
 	$toplevel_item = null;
 	$parent_user = null;
 
-	$parent_contact = null;
-
 	$objecttype = null;
 	$profile_uid = ($_REQUEST['profile_uid'] ?? 0) ?: local_user();
 	$posttype = ($_REQUEST['post_type'] ?? '') ?: Item::PT_ARTICLE;
@@ -123,11 +122,9 @@ function item_post(App $a) {
 		// if this isn't the top-level parent of the conversation, find it
 		if (DBA::isResult($toplevel_item)) {
 			// The URI and the contact is taken from the direct parent which needn't to be the top parent
-			$thread_parent_id = $toplevel_item['id'];
 			$thr_parent_uri = $toplevel_item['uri'];
-			$thread_parent_contact = Contact::getDetailsByURL($toplevel_item["author-link"]);
 
-			if ($toplevel_item['id'] != $toplevel_item['parent']) {
+			if ($toplevel_item['gravity'] != GRAVITY_PARENT) {
 				$toplevel_item = Item::selectFirst([], ['id' => $toplevel_item['parent']]);
 			}
 		}
@@ -253,7 +250,7 @@ function item_post(App $a) {
 		$verb              = $orig_post['verb'];
 		$objecttype        = $orig_post['object-type'];
 		$app               = $orig_post['app'];
-		$categories        = $orig_post['file'];
+		$categories        = $orig_post['file'] ?? '';
 		$title             = Strings::escapeTags(trim($_REQUEST['title']));
 		$body              = trim($body);
 		$private           = $orig_post['private'];
@@ -370,74 +367,63 @@ function item_post(App $a) {
 
 	// get contact info for owner
 	if ($profile_uid == local_user() || $allow_comment) {
-		$contact_record = $author;
+		$contact_record = $author ?: [];
 	} else {
-		$contact_record = DBA::selectFirst('contact', [], ['uid' => $profile_uid, 'self' => true]);
+		$contact_record = DBA::selectFirst('contact', [], ['uid' => $profile_uid, 'self' => true]) ?: [];
 	}
 
 	// Look for any tags and linkify them
-	$str_tags = '';
 	$inform   = '';
-
-	$tags = BBCode::getTags($body);
-
-	if ($thread_parent_id && !\Friendica\Content\Feature::isEnabled($uid, 'explicit_mentions')) {
-		$tags = item_add_implicit_mentions($tags, $thread_parent_contact, $thread_parent_id);
-	}
-
-	$tagged = [];
-
 	$private_forum = false;
+	$private_id = null;
 	$only_to_forum = false;
 	$forum_contact = [];
 
-	if (count($tags)) {
+	$body = BBCode::performWithEscapedTags($body, ['noparse', 'pre', 'code', 'img'], function ($body) use ($profile_uid, $network, $str_contact_allow, &$inform, &$private_forum, &$private_id, &$only_to_forum, &$forum_contact) {
+		$tags = BBCode::getTags($body);
+
+		$tagged = [];
+
 		foreach ($tags as $tag) {
 			$tag_type = substr($tag, 0, 1);
 
-			if ($tag_type == Term::TAG_CHARACTER[Term::HASHTAG]) {
+			if ($tag_type == Tag::TAG_CHARACTER[Tag::HASHTAG]) {
 				continue;
 			}
 
-			/*
+			/* If we already tagged 'Robert Johnson', don't try and tag 'Robert'.
-			 * If we already tagged 'Robert Johnson', don't try and tag 'Robert'.
 			 * Robert Johnson should be first in the $tags array
 			 */
-			$fullnametagged = false;
-			/// @TODO $tagged is initialized above if () block and is not filled, maybe old-lost code?
 			foreach ($tagged as $nextTag) {
 				if (stristr($nextTag, $tag . ' ')) {
-					$fullnametagged = true;
+					continue 2;
-					break;
 				}
 			}
-			if ($fullnametagged) {
-				continue;
-			}
 
-			$success = handle_tag($body, $inform, $str_tags, local_user() ? local_user() : $profile_uid, $tag, $network);
+			$success = ItemHelper::replaceTag($body, $inform, local_user() ? local_user() : $profile_uid, $tag, $network);
 			if ($success['replaced']) {
 				$tagged[] = $tag;
 			}
 			// When the forum is private or the forum is addressed with a "!" make the post private
-			if (is_array($success['contact']) && (!empty($success['contact']['prv']) || ($tag_type == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]))) {
+			if (!empty($success['contact']['prv']) || ($tag_type == Tag::TAG_CHARACTER[Tag::EXCLUSIVE_MENTION])) {
 				$private_forum = $success['contact']['prv'];
-				$only_to_forum = ($tag_type == Term::TAG_CHARACTER[Term::EXCLUSIVE_MENTION]);
+				$only_to_forum = ($tag_type == Tag::TAG_CHARACTER[Tag::EXCLUSIVE_MENTION]);
 				$private_id = $success['contact']['id'];
 				$forum_contact = $success['contact'];
-			} elseif (is_array($success['contact']) && !empty($success['contact']['forum']) &&
+			} elseif (!empty($success['contact']['forum']) && ($str_contact_allow == '<' . $success['contact']['id'] . '>')) {
-				($str_contact_allow == '<' . $success['contact']['id'] . '>')) {
 				$private_forum = false;
 				$only_to_forum = true;
 				$private_id = $success['contact']['id'];
 				$forum_contact = $success['contact'];
 			}
 		}
-	}
+
+		return $body;
+	});
 
 	$original_contact_id = $contact_id;
 
-	if (!$toplevel_item_id && count($forum_contact) && ($private_forum || $only_to_forum)) {
+	if (!$toplevel_item_id && !empty($forum_contact) && ($private_forum || $only_to_forum)) {
 		// we tagged a forum in a top level post. Now we change the post
 		$private = $private_forum;
 
@@ -578,9 +564,9 @@ function item_post(App $a) {
 	$datarray['gravity']       = $gravity;
 	$datarray['network']       = $network;
 	$datarray['contact-id']    = $contact_id;
-	$datarray['owner-name']    = $contact_record['name'];
+	$datarray['owner-name']    = $contact_record['name'] ?? '';
-	$datarray['owner-link']    = $contact_record['url'];
+	$datarray['owner-link']    = $contact_record['url'] ?? '';
-	$datarray['owner-avatar']  = $contact_record['thumb'];
+	$datarray['owner-avatar']  = $contact_record['thumb'] ?? '';
 	$datarray['owner-id']      = Contact::getIdForURL($datarray['owner-link']);
 	$datarray['author-name']   = $author['name'];
 	$datarray['author-link']   = $author['url'];
@@ -599,7 +585,6 @@ function item_post(App $a) {
 	$datarray['app']           = $app;
 	$datarray['location']      = $location;
 	$datarray['coord']         = $coord;
-	$datarray['tag']           = $str_tags;
 	$datarray['file']          = $categories;
 	$datarray['inform']        = $inform;
 	$datarray['verb']          = $verb;
@@ -656,7 +641,7 @@ function item_post(App $a) {
 
 	// Check for hashtags in the body and repair or add hashtag links
 	if ($preview || $orig_post) {
-		Item::setHashtags($datarray);
+		$datarray['body'] = Item::setHashtags($datarray['body']);
 	}
 
 	// preview mode - prepare the body for display and send it via json
@@ -664,6 +649,7 @@ function item_post(App $a) {
 		// We set the datarray ID to -1 because in preview mode the dataray
 		// doesn't have an ID.
 		$datarray["id"] = -1;
+		$datarray["uri-id"] = -1;
 		$datarray["item_id"] = -1;
 		$datarray["author-network"] = Protocol::DFRN;
 
@@ -696,7 +682,6 @@ function item_post(App $a) {
 		$fields = [
 			'title' => $datarray['title'],
 			'body' => $datarray['body'],
-			'tag' => $datarray['tag'],
 			'attach' => $datarray['attach'],
 			'file' => $datarray['file'],
 			'rendered-html' => $datarray['rendered-html'],
@@ -750,12 +735,18 @@ function item_post(App $a) {
 		throw new HTTPException\InternalServerErrorException(DI::l10n()->t('Item couldn\'t be fetched.'));
 	}
 
+	Tag::storeFromBody($datarray['uri-id'], $datarray['body']);
+
+	if (!\Friendica\Content\Feature::isEnabled($uid, 'explicit_mentions') && ($datarray['gravity'] == GRAVITY_COMMENT)) {
+		Tag::createImplicitMentions($datarray['uri-id'], $datarray['thr-parent-id']);
+	}
+
 	// update filetags in pconfig
 	FileTag::updatePconfig($uid, $categories_old, $categories_new, 'category');
 
 	// These notifications are sent if someone else is commenting other your wall
-	if ($toplevel_item_id) {
 	if ($contact_record != $author) {
+		if ($toplevel_item_id) {
 			notification([
 				'type'         => Type::COMMENT,
 				'notify_flags' => $user['notify-flags'],
@@ -773,9 +764,7 @@ function item_post(App $a) {
 				'parent'       => $toplevel_item_id,
 				'parent_uri'   => $toplevel_item['uri']
 			]);
-		}
+		} elseif (empty($forum_contact)) {
-	} else {
-		if (($contact_record != $author) && !count($forum_contact)) {
 			notification([
 				'type'         => Type::WALL,
 				'notify_flags' => $user['notify-flags'],
@@ -863,7 +852,9 @@ function item_content(App $a)
 
 	if (($a->argc >= 3) && ($a->argv[1] === 'drop') && intval($a->argv[2])) {
 		if (DI::mode()->isAjax()) {
-			$o = Item::deleteForUser(['id' => $a->argv[2]], local_user());
+			Item::deleteForUser(['id' => $a->argv[2]], local_user());
+			// ajax return: [<item id>, 0 (no perm) | <owner id>]
+			System::jsonExit([intval($a->argv[2]), local_user()]);
 		} else {
 			if (!empty($a->argv[3])) {
 				$o = drop_item($a->argv[2], $a->argv[3]);
@@ -872,203 +863,110 @@ function item_content(App $a)
 				$o = drop_item($a->argv[2]);
 			}
 		}
-
-		if (DI::mode()->isAjax()) {
-			// ajax return: [<item id>, 0 (no perm) | <owner id>]
-			System::jsonExit([intval($a->argv[2]), intval($o)]);
-		}
 	}
 
 	return $o;
 }
 
 /**
- * This function removes the tag $tag from the text $body and replaces it with
+ * @param int    $id
- * the appropriate link.
+ * @param string $return
- *
+ * @return string
- * @param App     $a
- * @param string  $body     the text to replace the tag in
- * @param string  $inform   a comma-seperated string containing everybody to inform
- * @param string  $str_tags string to add the tag to
- * @param integer $profile_uid
- * @param string  $tag      the tag to replace
- * @param string  $network  The network of the post
- *
- * @return array|bool ['replaced' => $replaced, 'contact' => $contact];
- * @throws ImagickException
  * @throws HTTPException\InternalServerErrorException
  */
-function handle_tag(&$body, &$inform, &$str_tags, $profile_uid, $tag, $network = "")
+function drop_item(int $id, string $return = '')
 {
-	$replaced = false;
+	// locate item to be deleted
-	$r = null;
+	$fields = ['id', 'uid', 'guid', 'contact-id', 'deleted', 'gravity', 'parent'];
+	$item = Item::selectFirstForUser(local_user(), $fields, ['id' => $id]);
 
-	//is it a person tag?
+	if (!DBA::isResult($item)) {
-	if (Term::isType($tag, Term::MENTION, Term::IMPLICIT_MENTION, Term::EXCLUSIVE_MENTION)) {
+		notice(DI::l10n()->t('Item not found.') . EOL);
-		$tag_type = substr($tag, 0, 1);
+		DI::baseUrl()->redirect('network');
-		//is it already replaced?
-		if (strpos($tag, '[url=')) {
-			//append tag to str_tags
-			if (!stristr($str_tags, $tag)) {
-				if (strlen($str_tags)) {
-					$str_tags .= ',';
-				}
-				$str_tags .= $tag;
 	}
 
-			// Checking for the alias that is used for OStatus
+	if ($item['deleted']) {
-			$pattern = "/[@!]\[url\=(.*?)\](.*?)\[\/url\]/ism";
+		return '';
-			if (preg_match($pattern, $tag, $matches)) {
-				$data = Contact::getDetailsByURL($matches[1]);
-
-				if ($data["alias"] != "") {
-					$newtag = '@[url=' . $data["alias"] . ']' . $data["nick"] . '[/url]';
-
-					if (!stripos($str_tags, '[url=' . $data["alias"] . ']')) {
-						if (strlen($str_tags)) {
-							$str_tags .= ',';
 	}
 
-						$str_tags .= $newtag;
+	$contact_id = 0;
-					}
+
-				}
+	// check if logged in user is either the author or owner of this item
+	if (Session::getRemoteContactID($item['uid']) == $item['contact-id']) {
+		$contact_id = $item['contact-id'];
 	}
 
-			return $replaced;
+	if ((local_user() == $item['uid']) || $contact_id) {
+		// Check if we should do HTML-based delete confirmation
+		if (!empty($_REQUEST['confirm'])) {
+			// <form> can't take arguments in its "action" parameter
+			// so add any arguments as hidden inputs
+			$query = explode_querystring(DI::args()->getQueryString());
+			$inputs = [];
+
+			foreach ($query['args'] as $arg) {
+				if (strpos($arg, 'confirm=') === false) {
+					$arg_parts = explode('=', $arg);
+					$inputs[] = ['name' => $arg_parts[0], 'value' => $arg_parts[1]];
+				}
 			}
 
-		//get the person's name
+			return Renderer::replaceMacros(Renderer::getMarkupTemplate('confirm.tpl'), [
-		$name = substr($tag, 1);
+				'$method' => 'get',
+				'$message' => DI::l10n()->t('Do you really want to delete this item?'),
+				'$extra_inputs' => $inputs,
+				'$confirm' => DI::l10n()->t('Yes'),
+				'$confirm_url' => $query['base'],
+				'$confirm_name' => 'confirmed',
+				'$cancel' => DI::l10n()->t('Cancel'),
+			]);
+		}
+		// Now check how the user responded to the confirmation query
+		if (!empty($_REQUEST['canceled'])) {
+			DI::baseUrl()->redirect('display/' . $item['guid']);
+		}
 
-		// Sometimes the tag detection doesn't seem to work right
+		$is_comment = $item['gravity'] == GRAVITY_COMMENT;
-		// This is some workaround
+		$parentitem = null;
-		$nameparts = explode(" ", $name);
+		if (!empty($item['parent'])) {
-		$name = $nameparts[0];
+			$fields = ['guid'];
+			$parentitem = Item::selectFirstForUser(local_user(), $fields, ['id' => $item['parent']]);
+		}
 
-		// Try to detect the contact in various ways
+		// delete the item
-		if (strpos($name, 'http://')) {
+		Item::deleteForUser(['id' => $item['id']], local_user());
-			// At first we have to ensure that the contact exists
-			Contact::getIdForURL($name);
 
-			// Now we should have something
+		$return_url = hex2bin($return);
-			$contact = Contact::getDetailsByURL($name);
+
-		} elseif (strpos($name, '@')) {
+		// removes update_* from return_url to ignore Ajax refresh
-			// This function automatically probes when no entry was found
+		$return_url = str_replace("update_", "", $return_url);
-			$contact = Contact::getDetailsByAddr($name);
+
+		// Check if delete a comment
+		if ($is_comment) {
+			// Return to parent guid
+			if (!empty($parentitem)) {
+				DI::baseUrl()->redirect('display/' . $parentitem['guid']);
+				//NOTREACHED
+			} // In case something goes wrong
+			else {
+				DI::baseUrl()->redirect('network');
+				//NOTREACHED
+			}
 		} else {
-			$contact = false;
+			// if unknown location or deleting top level post called from display
-			$fields = ['id', 'url', 'nick', 'name', 'alias', 'network', 'forum', 'prv'];
+			if (empty($return_url) || strpos($return_url, 'display') !== false) {
-
+				DI::baseUrl()->redirect('network');
-			if (strrpos($name, '+')) {
+				//NOTREACHED
-				// Is it in format @nick+number?
+			} else {
-				$tagcid = intval(substr($name, strrpos($name, '+') + 1));
+				DI::baseUrl()->redirect($return_url);
-				$contact = DBA::selectFirst('contact', $fields, ['id' => $tagcid, 'uid' => $profile_uid]);
+				//NOTREACHED
-			}
-
-			// select someone by nick or attag in the current network
-			if (!DBA::isResult($contact) && ($network != "")) {
-				$condition = ["(`nick` = ? OR `attag` = ?) AND `network` = ? AND `uid` = ?",
-						$name, $name, $network, $profile_uid];
-				$contact = DBA::selectFirst('contact', $fields, $condition);
-			}
-
-			//select someone by name in the current network
-			if (!DBA::isResult($contact) && ($network != "")) {
-				$condition = ['name' => $name, 'network' => $network, 'uid' => $profile_uid];
-				$contact = DBA::selectFirst('contact', $fields, $condition);
-			}
-
-			// select someone by nick or attag in any network
-			if (!DBA::isResult($contact)) {
-				$condition = ["(`nick` = ? OR `attag` = ?) AND `uid` = ?", $name, $name, $profile_uid];
-				$contact = DBA::selectFirst('contact', $fields, $condition);
-			}
-
-			// select someone by name in any network
-			if (!DBA::isResult($contact)) {
-				$condition = ['name' => $name, 'uid' => $profile_uid];
-				$contact = DBA::selectFirst('contact', $fields, $condition);
-			}
-		}
-
-		// Check if $contact has been successfully loaded
-		if (DBA::isResult($contact)) {
-			if (strlen($inform) && (isset($contact["notify"]) || isset($contact["id"]))) {
-				$inform .= ',';
-			}
-
-			if (isset($contact["id"])) {
-				$inform .= 'cid:' . $contact["id"];
-			} elseif (isset($contact["notify"])) {
-				$inform  .= $contact["notify"];
-			}
-
-			$profile = $contact["url"];
-			$alias   = $contact["alias"];
-			$newname = ($contact["name"] ?? '') ?: $contact["nick"];
-		}
-
-		//if there is an url for this persons profile
-		if (isset($profile) && ($newname != "")) {
-			$replaced = true;
-			// create profile link
-			$profile = str_replace(',', '%2c', $profile);
-			$newtag = $tag_type.'[url=' . $profile . ']' . $newname . '[/url]';
-			$body = str_replace($tag_type . $name, $newtag, $body);
-			// append tag to str_tags
-			if (!stristr($str_tags, $newtag)) {
-				if (strlen($str_tags)) {
-					$str_tags .= ',';
-				}
-				$str_tags .= $newtag;
-			}
-
-			/*
-			 * Status.Net seems to require the numeric ID URL in a mention if the person isn't
-			 * subscribed to you. But the nickname URL is OK if they are. Grrr. We'll tag both.
-			 */
-			if (!empty($alias)) {
-				$newtag = '@[url=' . $alias . ']' . $newname . '[/url]';
-				if (!stripos($str_tags, '[url=' . $alias . ']')) {
-					if (strlen($str_tags)) {
-						$str_tags .= ',';
-					}
-					$str_tags .= $newtag;
-				}
-			}
-		}
-	}
-
-	return ['replaced' => $replaced, 'contact' => $contact];
-}
-
-function item_add_implicit_mentions(array $tags, array $thread_parent_contact, $thread_parent_id)
-{
-	if (DI::config()->get('system', 'disable_implicit_mentions')) {
-		// Add a tag if the parent contact is from ActivityPub or OStatus (This will notify them)
-		if (in_array($thread_parent_contact['network'], [Protocol::OSTATUS, Protocol::ACTIVITYPUB])) {
-			$contact = Term::TAG_CHARACTER[Term::MENTION] . '[url=' . $thread_parent_contact['url'] . ']' . $thread_parent_contact['nick'] . '[/url]';
-			if (!stripos(implode($tags), '[url=' . $thread_parent_contact['url'] . ']')) {
-				$tags[] = $contact;
 			}
 		}
 	} else {
-		$implicit_mentions = [
+		notice(DI::l10n()->t('Permission denied.'));
-			$thread_parent_contact['url'] => $thread_parent_contact['nick']
+		DI::baseUrl()->redirect('display/' . $item['guid']);
-		];
+		//NOTREACHED
-
-		$parent_terms = Term::tagArrayFromItemId($thread_parent_id, [Term::MENTION, Term::IMPLICIT_MENTION]);
-
-		foreach ($parent_terms as $parent_term) {
-			$implicit_mentions[$parent_term['url']] = $parent_term['term'];
 	}
 
-		foreach ($implicit_mentions as $url => $label) {
+	return '';
-			if ($url != \Friendica\Model\Profile::getMyURL() && !stripos(implode($tags), '[url=' . $url . ']')) {
-				$tags[] = Term::TAG_CHARACTER[Term::IMPLICIT_MENTION] . '[url=' . $url . ']' . $label . '[/url]';
-			}
-		}
-	}
-
-	return $tags;
 }

mod/lostpass.php

@@ -41,10 +41,10 @@ function lostpass_post(App $a)
 		DI::baseUrl()->redirect();
 	}
 
-	$pwdreset_token = Strings::getRandomName(12) . random_int(1000, 9999);
+	$pwdreset_token = Strings::getRandomHex(32);
 
 	$fields = [
-		'pwdreset' => $pwdreset_token,
+		'pwdreset' => hash('sha256', $pwdreset_token),
 		'pwdreset_time' => DateTimeFormat::utcNow()
 	];
 	$result = DBA::update('user', $fields, ['uid' => $user['uid']]);
@@ -95,7 +95,7 @@ function lostpass_content(App $a)
 	if ($a->argc > 1) {
 		$pwdreset_token = $a->argv[1];
 
-		$user = DBA::selectFirst('user', ['uid', 'username', 'nickname', 'email', 'pwdreset_time', 'language'], ['pwdreset' => $pwdreset_token]);
+		$user = DBA::selectFirst('user', ['uid', 'username', 'nickname', 'email', 'pwdreset_time', 'language'], ['pwdreset' => hash('sha256', $pwdreset_token)]);
 		if (!DBA::isResult($user)) {
 			notice(DI::l10n()->t("Request could not be verified. \x28You may have previously submitted it.\x29 Password reset failed."));
 

mod/message.php

@@ -352,13 +352,7 @@ function message_content(App $a)
 			$messages = DBA::toArray($messages_stmt);
 
 			DBA::update('mail', ['seen' => 1], ['parent-uri' => $message['parent-uri'], 'uid' => local_user()]);
-
+			DBA::update('notify', ['seen' => 1], ['type' => Type::MAIL, 'parent' => $message['id'], 'uid' => local_user()]);
-			if ($message['convid']) {
-				// Clear Diaspora private message notifications
-				DBA::update('notify', ['seen' => 1], ['type' => Type::MAIL, 'parent' => $message['convid'], 'uid' => local_user()]);
-			}
-			// Clear DFRN private message notifications
-			DBA::update('notify', ['seen' => 1], ['type' => Type::MAIL, 'parent' => $message['parent-uri'], 'uid' => local_user()]);
 		} else {
 			$messages = false;
 		}

mod/msearch.php

@@ -77,6 +77,8 @@ function msearch_post(App $a)
 		];
 	}
 
+	DBA::close($search_stmt);
+
 	$output = ['total' => $total, 'items_page' => $perpage, 'page' => $page, 'results' => $results];
 
 	echo json_encode($output);

mod/network.php

@@ -37,8 +37,8 @@ use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Group;
 use Friendica\Model\Item;
+use Friendica\Model\Post\Category;
 use Friendica\Model\Profile;
-use Friendica\Model\Term;
 use Friendica\Module\Security\Login;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Proxy as ProxyUtils;
@@ -58,8 +58,8 @@ function network_init(App $a)
 	$group_id = (($a->argc > 1 && is_numeric($a->argv[1])) ? intval($a->argv[1]) : 0);
 
 	$cid = 0;
-	if (!empty($_GET['cid'])) {
+	if (!empty($_GET['contactid'])) {
-		$cid = $_GET['cid'];
+		$cid = $_GET['contactid'];
 		$_GET['nets'] = '';
 		$group_id = 0;
 	}
@@ -379,25 +379,25 @@ function networkFlatView(App $a, $update = 0)
 
 	networkPager($a, $pager, $update);
 
-	$item_params = ['order' => ['id' => true]];
 
 	if (strlen($file)) {
-		$term_condition = ["`term` = ? AND `otype` = ? AND `type` = ? AND `uid` = ?",
+		$item_params = ['order' => ['uri-id' => true]];
-			$file, Term::OBJECT_TYPE_POST, Term::FILE, local_user()];
+		$term_condition = ['name' => $file, 'type' => Category::FILE, 'uid' => local_user()];
-		$term_params = ['order' => ['tid' => true], 'limit' => [$pager->getStart(), $pager->getItemsPerPage()]];
+		$term_params = ['order' => ['uri-id' => true], 'limit' => [$pager->getStart(), $pager->getItemsPerPage()]];
-		$result = DBA::select('term', ['oid'], $term_condition, $term_params);
+		$result = DBA::select('category-view', ['uri-id'], $term_condition, $term_params);
 
 		$posts = [];
 		while ($term = DBA::fetch($result)) {
-			$posts[] = $term['oid'];
+			$posts[] = $term['uri-id'];
 		}
 		DBA::close($result);
 
 		if (count($posts) == 0) {
 			return '';
 		}
-		$item_condition = ['uid' => local_user(), 'id' => $posts];
+		$item_condition = ['uid' => local_user(), 'uri-id' => $posts];
 	} else {
+		$item_params = ['order' => ['id' => true]];
 		$item_condition = ['uid' => local_user()];
 		$item_params['limit'] = [$pager->getStart(), $pager->getItemsPerPage()];
 
@@ -466,7 +466,7 @@ function networkThreadedView(App $a, $update, $parent)
 
 	$o = '';
 
-	$cid   = intval($_GET['cid']   ?? 0);
+	$cid   = intval($_GET['contactid'] ?? 0);
 	$star  = intval($_GET['star']      ?? 0);
 	$bmark = intval($_GET['bmark']     ?? 0);
 	$conv  = intval($_GET['conv']      ?? 0);
@@ -709,7 +709,7 @@ function networkThreadedView(App $a, $update, $parent)
 			}
 			if ($order === 'post') {
 				// Only show toplevel posts when updating posts in this order mode
-				$sql_extra4 .= " AND `item`.`id` = `item`.`parent`";
+				$sql_extra4 .= " AND `item`.`gravity` = " . GRAVITY_PARENT;
 			}
 		}
 
@@ -786,15 +786,21 @@ function networkThreadedView(App $a, $update, $parent)
 			$top_limit = DateTimeFormat::utcNow();
 		}
 
+		// Handle bad performance situations when the distance between top and bottom is too high
+		// See issue https://github.com/friendica/friendica/issues/8619
+		if (strtotime($top_limit) - strtotime($bottom_limit) > 86400) {
+			// Set the bottom limit to one day in the past at maximum
+			$bottom_limit = DateTimeFormat::utc(date('c', strtotime($top_limit) - 86400));
+		}
+
 		$items = DBA::p("SELECT `item`.`parent-uri` AS `uri`, 0 AS `item_id`, `item`.$ordering AS `order_date`, `author`.`url` AS `author-link` FROM `item`
-			STRAIGHT_JOIN (SELECT `oid` FROM `term` WHERE `term` IN
+			STRAIGHT_JOIN (SELECT `uri-id` FROM `tag-search-view` WHERE `name` IN
-				(SELECT SUBSTR(`term`, 2) FROM `search` WHERE `uid` = ? AND `term` LIKE '#%') AND `otype` = ? AND `type` = ? AND `uid` = 0) AS `term`
+				(SELECT SUBSTR(`term`, 2) FROM `search` WHERE `uid` = ? AND `term` LIKE '#%') AND `uid` = 0) AS `tag-search`
-			ON `item`.`id` = `term`.`oid`
+			ON `item`.`uri-id` = `tag-search`.`uri-id`
 			STRAIGHT_JOIN `contact` AS `author` ON `author`.`id` = `item`.`author-id`
 			WHERE `item`.`uid` = 0 AND `item`.$ordering < ? AND `item`.$ordering > ? AND `item`.`gravity` = ?
 				AND NOT `author`.`hidden` AND NOT `author`.`blocked`" . $sql_tag_nets,
-			local_user(), TERM_OBJ_POST, TERM_HASHTAG,
+			local_user(), $top_limit, $bottom_limit, GRAVITY_PARENT);
-			$top_limit, $bottom_limit, GRAVITY_PARENT);
 
 		$data = DBA::toArray($items);
 
@@ -892,8 +898,8 @@ function network_tabs(App $a)
 	$cmd = DI::args()->getCommand();
 
 	$def_param = [];
-	if (!empty($_GET['cid'])) {
+	if (!empty($_GET['contactid'])) {
-		$def_param['cid'] = $_GET['cid'];
+		$def_param['contactid'] = $_GET['contactid'];
 	}
 
 	// tabs

mod/ostatus_subscribe.php

@@ -48,7 +48,6 @@ function ostatus_subscribe_content(App $a)
 		}
 
 		$contact = Probe::uri($_REQUEST['url']);
-
 		if (!$contact) {
 			DI::pConfig()->delete($uid, 'ostatus', 'legacy_contact');
 			return $o . DI::l10n()->t('Couldn\'t fetch information for contact.');
@@ -91,7 +90,7 @@ function ostatus_subscribe_content(App $a)
 
 	$probed = Probe::uri($url);
 	if ($probed['network'] == Protocol::OSTATUS) {
-		$result = Contact::createFromProbe($uid, $url, true, Protocol::OSTATUS);
+		$result = Contact::createFromProbe($a->user, $probed['url'], true, Protocol::OSTATUS);
 		if ($result['success']) {
 			$o .= ' - ' . DI::l10n()->t('success');
 		} else {

mod/photos.php

@@ -36,6 +36,7 @@ use Friendica\Model\Contact;
 use Friendica\Model\Item;
 use Friendica\Model\Photo;
 use Friendica\Model\Profile;
+use Friendica\Model\Tag;
 use Friendica\Model\User;
 use Friendica\Module\BaseProfile;
 use Friendica\Network\Probe;
@@ -81,7 +82,7 @@ function photos_init(App $a) {
 			'$photo' => $profile['photo'],
 			'$addr' => $profile['addr'] ?? '',
 			'$account_type' => $account_type,
-			'$about' => BBCode::convert($profile['about'] ?? ''),
+			'$about' => BBCode::convert($profile['about']),
 		]);
 
 		$albums = Photo::getAlbums($a->data['user']['uid']);
@@ -309,7 +310,7 @@ function photos_post(App $a)
 		$desc        = !empty($_POST['desc'])      ? Strings::escapeTags(trim($_POST['desc']))      : '';
 		$rawtags     = !empty($_POST['newtag'])    ? Strings::escapeTags(trim($_POST['newtag']))    : '';
 		$item_id     = !empty($_POST['item_id'])   ? intval($_POST['item_id'])                      : 0;
-		$albname     = !empty($_POST['albname'])   ? Strings::escapeTags(trim($_POST['albname']))   : '';
+		$albname     = !empty($_POST['albname'])   ? trim($_POST['albname'])                        : '';
 		$origaname   = !empty($_POST['origaname']) ? Strings::escapeTags(trim($_POST['origaname'])) : '';
 
 		$aclFormatter = DI::aclFormatter();
@@ -421,16 +422,14 @@ function photos_post(App $a)
 		}
 
 		if ($item_id) {
-			$item = Item::selectFirst(['tag', 'inform'], ['id' => $item_id, 'uid' => $page_owner_uid]);
+			$item = Item::selectFirst(['tag', 'inform', 'uri-id'], ['id' => $item_id, 'uid' => $page_owner_uid]);
 
 			if (DBA::isResult($item)) {
-				$old_tag    = $item['tag'];
 				$old_inform = $item['inform'];
 			}
 		}
 
 		if (strlen($rawtags)) {
-			$str_tags = '';
 			$inform   = '';
 
 			// if the new tag doesn't have a namespace specifier (@foo or #foo) give it a hashtag
@@ -510,30 +509,25 @@ function photos_post(App $a)
 
 						if ($profile) {
 							if (!empty($contact)) {
-								$taginfo[] = [$newname, $profile, $notify, $contact, '@[url=' . str_replace(',', '%2c', $profile) . ']' . $newname . '[/url]'];
+								$taginfo[] = [$newname, $profile, $notify, $contact];
 							} else {
-								$taginfo[] = [$newname, $profile, $notify, null, $str_tags .= '@[url=' . $profile . ']' . $newname . '[/url]'];
+								$taginfo[] = [$newname, $profile, $notify, null];
-							}
-
-							if (strlen($str_tags)) {
-								$str_tags .= ',';
 							}
 
 							$profile = str_replace(',', '%2c', $profile);
-							$str_tags .= '@[url=' . $profile . ']' . $newname . '[/url]';
+
+							if (!empty($item['uri-id'])) {
+								Tag::store($item['uri-id'], Tag::MENTION, $newname, $profile);
+							}	
 						}
 					} elseif (strpos($tag, '#') === 0) {
 						$tagname = substr($tag, 1);
-						$str_tags .= '#[url=' . DI::baseUrl() . "/search?tag=" . $tagname . ']' . $tagname . '[/url],';
+						if (!empty($item['uri-id'])) {
+							Tag::store($item['uri-id'], Tag::HASHTAG, $tagname);
 						}
 					}
 				}
-
-			$newtag = $old_tag ?? '';
-			if (strlen($newtag) && strlen($str_tags)) {
-				$newtag .= ',';
 			}
-			$newtag .= $str_tags;
 
 			$newinform = $old_inform ?? '';
 			if (strlen($newinform) && strlen($inform)) {
@@ -541,7 +535,7 @@ function photos_post(App $a)
 			}
 			$newinform .= $inform;
 
-			$fields = ['tag' => $newtag, 'inform' => $newinform, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()];
+			$fields = ['inform' => $newinform, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()];
 			$condition = ['id' => $item_id];
 			Item::update($fields, $condition);
 
@@ -585,7 +579,6 @@ function photos_post(App $a)
 					$arr['gravity']       = GRAVITY_PARENT;
 					$arr['object-type']   = Activity\ObjectType::PERSON;
 					$arr['target-type']   = Activity\ObjectType::IMAGE;
-					$arr['tag']           = $tagged[4];
 					$arr['inform']        = $tagged[2];
 					$arr['origin']        = 1;
 					$arr['body']          = DI::l10n()->t('%1$s was tagged in %2$s by %3$s', '[url=' . $tagged[1] . ']' . $tagged[0] . '[/url]', '[url=' . DI::baseUrl() . '/photos/' . $owner_record['nickname'] . '/image/' . $photo['resource-id'] . ']' . DI::l10n()->t('a photo') . '[/url]', '[url=' . $owner_record['url'] . ']' . $owner_record['name'] . '[/url]') ;
@@ -615,10 +608,10 @@ function photos_post(App $a)
 	Hook::callAll('photo_post_init', $_POST);
 
 	// Determine the album to use
-	$album    = !empty($_REQUEST['album'])    ? Strings::escapeTags(trim($_REQUEST['album']))    : '';
+	$album    = trim($_REQUEST['album'] ?? '');
-	$newalbum = !empty($_REQUEST['newalbum']) ? Strings::escapeTags(trim($_REQUEST['newalbum'])) : '';
+	$newalbum = trim($_REQUEST['newalbum'] ?? '');
 
-	Logger::log('mod/photos.php: photos_post(): album= ' . $album . ' newalbum= ' . $newalbum , Logger::DEBUG);
+	Logger::info('album= ' . $album . ' newalbum= ' . $newalbum);
 
 	if (!strlen($album)) {
 		if (strlen($newalbum)) {
@@ -706,9 +699,7 @@ function photos_post(App $a)
 		return;
 	}
 
-	if ($type == "") {
+	$type = Images::getMimeTypeBySource($src, $filename, $type);
-		$type = Images::guessType($filename);
-	}
 
 	Logger::log('photos: upload: received file: ' . $filename . ' as ' . $src . ' ('. $type . ') ' . $filesize . ' bytes', Logger::DEBUG);
 
@@ -787,7 +778,7 @@ function photos_post(App $a)
 
 	// Create item container
 	$lat = $lon = null;
-	if ($exif && $exif['GPS'] && Feature::isEnabled($page_owner_uid, 'photo_location')) {
+	if (!empty($exif['GPS']) && Feature::isEnabled($page_owner_uid, 'photo_location')) {
 		$lat = Photo::getGps($exif['GPS']['GPSLatitude'], $exif['GPS']['GPSLatitudeRef']);
 		$lon = Photo::getGps($exif['GPS']['GPSLongitude'], $exif['GPS']['GPSLongitudeRef']);
 	}
@@ -1296,7 +1287,7 @@ function photos_content(App $a)
 		}
 
 		if (!empty($link_item['parent']) && !empty($link_item['uid'])) {
-			$condition = ["`parent` = ? AND `parent` != `id`",  $link_item['parent']];
+			$condition = ["`parent` = ? AND `gravity` != ?",  $link_item['parent'], GRAVITY_PARENT];
 			$total = DBA::count('item', $condition);
 
 			$pager = new Pager(DI::l10n(), DI::args()->getQueryString());
@@ -1316,8 +1307,9 @@ function photos_content(App $a)
 
 		$tags = null;
 
-		if (!empty($link_item['id']) && !empty($link_item['tag'])) {
+		if (!empty($link_item['id'])) {
-			$arr = explode(',', $link_item['tag']);
+			$tag_text = Tag::getCSVByURIId($link_item['uri-id']);
+			$arr = explode(',', $tag_text);
 			// parse tags and add links
 			$tag_arr = [];
 			foreach ($arr as $tag) {
@@ -1464,7 +1456,7 @@ function photos_content(App $a)
 
 					if (($activity->match($item['verb'], Activity::LIKE) ||
 					     $activity->match($item['verb'], Activity::DISLIKE)) &&
-					    ($item['id'] != $item['parent'])) {
+					    ($item['gravity'] != GRAVITY_PARENT)) {
 						continue;
 					}
 

mod/ping.php

@@ -30,6 +30,8 @@ use Friendica\Model\Contact;
 use Friendica\Model\Group;
 use Friendica\Model\Item;
 use Friendica\Model\Notify\Type;
+use Friendica\Model\Verb;
+use Friendica\Protocol\Activity;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Temporal;
 use Friendica\Util\Proxy as ProxyUtils;
@@ -134,9 +136,10 @@ function ping_init(App $a)
 
 		$notifs = ping_get_notifications(local_user());
 
-		$condition = ["`unseen` AND `uid` = ? AND `contact-id` != ?", local_user(), local_user()];
+		$condition = ["`unseen` AND `uid` = ? AND `contact-id` != ? AND (`vid` != ? OR `vid` IS NULL)",
+			local_user(), local_user(), Verb::getID(Activity::FOLLOW)];
 		$fields = ['id', 'parent', 'verb', 'author-name', 'unseen', 'author-link', 'author-avatar', 'contact-avatar',
-			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid', 'wall'];
+			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid', 'wall', 'activity'];
 		$params = ['order' => ['received' => true]];
 		$items = Item::selectForUser(local_user(), $fields, $condition, $params);
 
@@ -464,13 +467,13 @@ function ping_get_notifications($uid)
 
 			if ($notification["visible"]
 				&& !$notification["deleted"]
-				&& empty($result[$notification["parent"]])
+				&& empty($result[$notification['parent']])
 			) {
 				// Should we condense the notifications or show them all?
 				if (DI::pConfig()->get(local_user(), 'system', 'detailed_notif')) {
 					$result[$notification["id"]] = $notification;
 				} else {
-					$result[$notification["parent"]] = $notification;
+					$result[$notification['parent']] = $notification;
 				}
 			}
 		}

mod/poco.php

@@ -81,10 +81,7 @@ function poco_init(App $a) {
 	}
 
 	if (!$system_mode && !$global) {
-		$user = DBA::fetchFirst("SELECT `user`.`uid`, `user`.`nickname` FROM `user`
+		$user = DBA::selectFirst('owner-view', ['uid', 'nickname'], ['nickname' => $nickname, 'hide-friends' => false]);
-			INNER JOIN `profile` ON `user`.`uid` = `profile`.`uid`
-			WHERE `user`.`nickname` = ? AND NOT `profile`.`hide-friends`",
-			$nickname);
 		if (!DBA::isResult($user)) {
 			throw new \Friendica\Network\HTTPException\NotFoundException();
 		}
@@ -147,16 +144,7 @@ function poco_init(App $a) {
 		);
 	} elseif ($system_mode) {
 		Logger::log("Start system mode query", Logger::DEBUG);
-		$contacts = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation`, `profile`.`pub_keywords`,
+		$contacts = DBA::selectToArray('owner-view', [], ['net-publish' => true], ['limit' => [$startIndex, $itemsPerPage]]);
-				`profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`,
-				`profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry`, `user`.`account-type`
-			FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid`
-				INNER JOIN `user` ON `user`.`uid` = `contact`.`uid`
-			WHERE `self` = 1 AND `profile`.`net-publish`
-			LIMIT %d, %d",
-			intval($startIndex),
-			intval($itemsPerPage)
-		);
 	} else {
 		Logger::log("Start query for user " . $user['nickname'], Logger::DEBUG);
 		$contacts = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0
@@ -216,7 +204,10 @@ function poco_init(App $a) {
 		}
 	}
 
-	if (is_array($contacts)) {
+	if (!is_array($contacts)) {
+		throw new \Friendica\Network\HTTPException\InternalServerErrorException();
+	}
+
 	if (DBA::isResult($contacts)) {
 		foreach ($contacts as $contact) {
 			if (!isset($contact['updated'])) {
@@ -233,28 +224,6 @@ function poco_init(App $a) {
 				}
 			}
 
-				if (($contact['about'] == "") && isset($contact['pabout'])) {
-					$contact['about'] = $contact['pabout'];
-				}
-				if ($contact['location'] == "") {
-					if (isset($contact['plocation'])) {
-						$contact['location'] = $contact['plocation'];
-					}
-					if (isset($contact['pregion']) && ( $contact['pregion'] != "")) {
-						if ($contact['location'] != "") {
-							$contact['location'] .= ", ";
-						}
-						$contact['location'] .= $contact['pregion'];
-					}
-
-					if (isset($contact['pcountry']) && ( $contact['pcountry'] != "")) {
-						if ($contact['location'] != "") {
-							$contact['location'] .= ", ";
-						}
-						$contact['location'] .= $contact['pcountry'];
-					}
-				}
-
 			if (($contact['keywords'] == "") && isset($contact['pub_keywords'])) {
 				$contact['keywords'] = $contact['pub_keywords'];
 			}
@@ -346,21 +315,21 @@ function poco_init(App $a) {
 				$entry['address'] = [];
 
 				// Deactivated. It just reveals too much data. (Although its from the default profile)
-					//if (isset($rr['paddress']))
+				//if (isset($rr['address']))
-					//	 $entry['address']['streetAddress'] = $rr['paddress'];
+				//	 $entry['address']['streetAddress'] = $rr['address'];
 
-					if (isset($contact['plocation'])) {
+				if (isset($contact['locality'])) {
-						$entry['address']['locality'] = $contact['plocation'];
+					$entry['address']['locality'] = $contact['locality'];
 				}
-					if (isset($contact['pregion'])) {
+				if (isset($contact['region'])) {
-						$entry['address']['region'] = $contact['pregion'];
+					$entry['address']['region'] = $contact['region'];
 				}
 				// See above
-					//if (isset($rr['ppostalcode']))
+				//if (isset($rr['postal-code']))
-					//	 $entry['address']['postalCode'] = $rr['ppostalcode'];
+				//	 $entry['address']['postalCode'] = $rr['postal-code'];
 
-					if (isset($contact['pcountry'])) {
+				if (isset($contact['country'])) {
-						$entry['address']['country'] = $contact['pcountry'];
+					$entry['address']['country'] = $contact['country'];
 				}
 			}
 
@@ -372,9 +341,6 @@ function poco_init(App $a) {
 	} else {
 		$ret['entry'][] = [];
 	}
-	} else {
-		throw new \Friendica\Network\HTTPException\InternalServerErrorException();
-	}
 
 	Logger::log("End of poco", Logger::DEBUG);
 

mod/poke.php

@@ -1,191 +0,0 @@
-<?php
-/**
- * Poke, prod, finger, or otherwise do unspeakable things to somebody - who must be a connection in your address book
- * This function can be invoked with the required arguments (verb and cid and private and possibly parent) silently via ajax or
- * other web request. You must be logged in and connected to a profile.
- * If the required arguments aren't present, we'll display a simple form to choose a recipient and a verb.
- * parent is a special argument which let's you attach this activity as a comment to an existing conversation, which
- * may have started with somebody else poking (etc.) somebody, but this isn't necessary. This can be used in the more pokes
- * addon version to have entire conversations where Alice poked Bob, Bob fingered Alice, Alice hugged Bob, etc.
- *
- * private creates a private conversation with the recipient. Otherwise your profile's default post privacy is used.
- *
- * @file mod/poke.php
- */
-
-use Friendica\App;
-use Friendica\Core\Hook;
-use Friendica\Core\Logger;
-use Friendica\Core\Renderer;
-use Friendica\Core\System;
-use Friendica\Database\DBA;
-use Friendica\DI;
-use Friendica\Model\Item;
-use Friendica\Protocol\Activity;
-use Friendica\Util\Strings;
-use Friendica\Util\XML;
-
-function poke_init(App $a)
-{
-	if (!local_user()) {
-		return;
-	}
-
-	$uid = local_user();
-
-	if (empty($_GET['verb'])) {
-		return;
-	}
-
-	$verb = Strings::escapeTags(trim($_GET['verb']));
-
-	$verbs = DI::l10n()->getPokeVerbs();
-
-	if (!array_key_exists($verb, $verbs)) {
-		return;
-	}
-
-	$activity = Activity::POKE . '#' . urlencode($verbs[$verb][0]);
-
-	$contact_id = intval($_GET['cid']);
-	if (!$contact_id) {
-		return;
-	}
-
-	$parent = (!empty($_GET['parent']) ? intval($_GET['parent']) : 0);
-
-
-	Logger::log('poke: verb ' . $verb . ' contact ' . $contact_id, Logger::DEBUG);
-
-
-	$r = q("SELECT * FROM `contact` WHERE `id` = %d AND `uid` = %d LIMIT 1",
-		intval($contact_id),
-		intval($uid)
-	);
-
-	if (!DBA::isResult($r)) {
-		Logger::log('poke: no contact ' . $contact_id);
-		return;
-	}
-
-	$target = $r[0];
-
-	if ($parent) {
-		$fields = ['uri', 'private', 'allow_cid', 'allow_gid', 'deny_cid', 'deny_gid'];
-		$condition = ['id' => $parent, 'parent' => $parent, 'uid' => $uid];
-		$item = Item::selectFirst($fields, $condition);
-
-		if (DBA::isResult($item)) {
-			$parent_uri = $item['uri'];
-			$private    = $item['private'];
-			$allow_cid  = $item['allow_cid'];
-			$allow_gid  = $item['allow_gid'];
-			$deny_cid   = $item['deny_cid'];
-			$deny_gid   = $item['deny_gid'];
-		}
-	} else {
-		$private = (!empty($_GET['private']) ? intval($_GET['private']) : Item::PUBLIC);
-
-		$allow_cid     = ($private ? '<' . $target['id']. '>' : $a->user['allow_cid']);
-		$allow_gid     = ($private ? '' : $a->user['allow_gid']);
-		$deny_cid      = ($private ? '' : $a->user['deny_cid']);
-		$deny_gid      = ($private ? '' : $a->user['deny_gid']);
-	}
-
-	$poster = $a->contact;
-
-	$uri = Item::newURI($uid);
-
-	$arr = [];
-
-	$arr['guid']          = System::createUUID();
-	$arr['uid']           = $uid;
-	$arr['uri']           = $uri;
-	$arr['parent-uri']    = (!empty($parent_uri) ? $parent_uri : $uri);
-	$arr['wall']          = 1;
-	$arr['contact-id']    = $poster['id'];
-	$arr['owner-name']    = $poster['name'];
-	$arr['owner-link']    = $poster['url'];
-	$arr['owner-avatar']  = $poster['thumb'];
-	$arr['author-name']   = $poster['name'];
-	$arr['author-link']   = $poster['url'];
-	$arr['author-avatar'] = $poster['thumb'];
-	$arr['title']         = '';
-	$arr['allow_cid']     = $allow_cid;
-	$arr['allow_gid']     = $allow_gid;
-	$arr['deny_cid']      = $deny_cid;
-	$arr['deny_gid']      = $deny_gid;
-	$arr['visible']       = 1;
-	$arr['verb']          = $activity;
-	$arr['private']       = $private;
-	$arr['object-type']   = Activity\ObjectType::PERSON;
-
-	$arr['origin']        = 1;
-	$arr['body']          = '[url=' . $poster['url'] . ']' . $poster['name'] . '[/url]' . ' ' . DI::l10n()->t($verbs[$verb][0]) . ' ' . '[url=' . $target['url'] . ']' . $target['name'] . '[/url]';
-
-	$arr['object'] = '<object><type>' . Activity\ObjectType::PERSON . '</type><title>' . $target['name'] . '</title><id>' . $target['url'] . '</id>';
-	$arr['object'] .= '<link>' . XML::escape('<link rel="alternate" type="text/html" href="' . $target['url'] . '" />' . "\n");
-
-	$arr['object'] .= XML::escape('<link rel="photo" type="image/jpeg" href="' . $target['photo'] . '" />' . "\n");
-	$arr['object'] .= '</link></object>' . "\n";
-
-	Item::insert($arr);
-
-	Hook::callAll('post_local_end', $arr);
-
-	return;
-}
-
-function poke_content(App $a)
-{
-	if (!local_user()) {
-		notice(DI::l10n()->t('Permission denied.') . EOL);
-		return;
-	}
-
-	if (empty($_GET['c'])) {
-		return;
-	}
-
-	$contact = DBA::selectFirst('contact', ['id', 'name'], ['id' => $_GET['c'], 'uid' => local_user()]);
-	if (!DBA::isResult($contact)) {
-		return;
-	}
-
-	$name = $contact['name'];
-	$id = $contact['id'];
-
-	$head_tpl = Renderer::getMarkupTemplate('poke_head.tpl');
-	DI::page()['htmlhead'] .= Renderer::replaceMacros($head_tpl,[
-		'$baseurl' => DI::baseUrl()->get(true),
-	]);
-
-	$parent = (!empty($_GET['parent']) ? intval($_GET['parent']) : '0');
-
-
-	$verbs = DI::l10n()->getPokeVerbs();
-
-	$shortlist = [];
-	foreach ($verbs as $k => $v) {
-		if ($v[1] !== 'NOTRANSLATION') {
-			$shortlist[] = [$k, $v[1]];
-		}
-	}
-
-	$tpl = Renderer::getMarkupTemplate('poke_content.tpl');
-
-	$o = Renderer::replaceMacros($tpl,[
-		'$title' => DI::l10n()->t('Poke/Prod'),
-		'$desc' => DI::l10n()->t('poke, prod or do other things to somebody'),
-		'$clabel' => DI::l10n()->t('Recipient'),
-		'$choice' => DI::l10n()->t('Choose what you wish to do to recipient'),
-		'$verbs' => $shortlist,
-		'$parent' => $parent,
-		'$prv_desc' => DI::l10n()->t('Make this post private'),
-		'$submit' => DI::l10n()->t('Submit'),
-		'$name' => $name,
-		'$id' => $id
-	]);
-
-	return $o;
-}

mod/redir.php

@@ -31,6 +31,9 @@ use Friendica\Util\Network;
 use Friendica\Util\Strings;
 
 function redir_init(App $a) {
+	if (!Session::isAuthenticated()) {
+		throw new \Friendica\Network\HTTPException\ForbiddenException(DI::l10n()->t('Access denied.'));
+	}
 
 	$url = $_GET['url'] ?? '';
 	$quiet = !empty($_GET['quiet']) ? '&quiet=1' : '';
@@ -44,19 +47,21 @@ function redir_init(App $a) {
 	// Try magic auth before the legacy stuff
 	redir_magic($a, $cid, $url);
 
-	if (!empty($cid)) {
+	if (empty($cid)) {
+		throw new \Friendica\Network\HTTPException\BadRequestException(DI::l10n()->t('Bad Request.'));
+	}
+
 	$fields = ['id', 'uid', 'nurl', 'url', 'addr', 'name', 'network', 'poll', 'issued-id', 'dfrn-id', 'duplex', 'pending'];
 	$contact = DBA::selectFirst('contact', $fields, ['id' => $cid, 'uid' => [0, local_user()]]);
 	if (!DBA::isResult($contact)) {
-			notice(DI::l10n()->t('Contact not found.'));
+		throw new \Friendica\Network\HTTPException\NotFoundException(DI::l10n()->t('Contact not found.'));
-			DI::baseUrl()->redirect();
 	}
 
 	$contact_url = $contact['url'];
 
-		if (!Session::isAuthenticated() // Visitors (not logged in or not remotes) can't authenticate.
+	if (!empty($a->contact['id']) && $a->contact['id'] == $cid) {
-			|| (!empty($a->contact['id']) && $a->contact['id'] == $cid)) // Local user is already authenticated.
+		// Local user is already authenticated.
-		{
+		redir_check_url($contact_url, $url);
 		$a->redirect($url ?: $contact_url);
 	}
 
@@ -71,6 +76,7 @@ function redir_init(App $a) {
 
 		if (!empty($a->contact['id']) && $a->contact['id'] == $cid) {
 			// Local user is already authenticated.
+			redir_check_url($contact_url, $url);
 			$target_url = $url ?: $contact_url;
 			Logger::log($contact['name'] . " is already authenticated. Redirecting to " . $target_url, Logger::DEBUG);
 			$a->redirect($target_url);
@@ -87,6 +93,7 @@ function redir_init(App $a) {
 		// contact.
 		if (($host == $remotehost) && (Session::getRemoteContactID(Session::get('visitor_visiting')) == Session::get('visitor_id'))) {
 			// Remote user is already authenticated.
+			redir_check_url($contact_url, $url);
 			$target_url = $url ?: $contact_url;
 			Logger::log($contact['name'] . " is already authenticated. Redirecting to " . $target_url, Logger::DEBUG);
 			$a->redirect($target_url);
@@ -120,12 +127,12 @@ function redir_init(App $a) {
 			. '&dfrn_version=' . DFRN_PROTOCOL_VERSION . '&type=profile&sec=' . $sec . $dest . $quiet);
 	}
 
-		$url = $url ?: $contact_url;
+	if (empty($url)) {
+		throw new \Friendica\Network\HTTPException\BadRequestException(DI::l10n()->t('Bad Request.'));
 	}
 
 	// If we don't have a connected contact, redirect with
 	// the 'zrl' parameter.
-	if (!empty($url)) {
 	$my_profile = Profile::getMyURL();
 
 	if (!empty($my_profile) && !Strings::compareLink($my_profile, $url)) {
@@ -138,10 +145,6 @@ function redir_init(App $a) {
 	$a->redirect($url);
 }
 
-	notice(DI::l10n()->t('Contact not found.'));
-	DI::baseUrl()->redirect();
-}
-
 function redir_magic($a, $cid, $url)
 {
 	$visitor = Profile::getMyURL();
@@ -152,15 +155,10 @@ function redir_magic($a, $cid, $url)
 	$contact = DBA::selectFirst('contact', ['url'], ['id' => $cid]);
 	if (!DBA::isResult($contact)) {
 		Logger::info('Contact not found', ['id' => $cid]);
-		// Shouldn't happen under normal conditions
+		throw new \Friendica\Network\HTTPException\NotFoundException(DI::l10n()->t('Contact not found.'));
-		notice(DI::l10n()->t('Contact not found.'));
-		if (!empty($url)) {
-			System::externalRedirect($url);
-		} else {
-			DI::baseUrl()->redirect();
-		}
 	} else {
 		$contact_url = $contact['url'];
+		redir_check_url($contact_url, $url);
 		$target_url = $url ?: $contact_url;
 	}
 
@@ -184,3 +182,24 @@ function redir_magic($a, $cid, $url)
 		Logger::info('No magic for contact', ['contact' => $contact_url]);
 	}
 }
+
+function redir_check_url(string $contact_url, string $url)
+{
+	if (empty($contact_url) || empty($url)) {
+		return;
+	}
+
+	$url_host = parse_url($url, PHP_URL_HOST);
+	if (empty($url_host)) {
+		$url_host = parse_url(DI::baseUrl(), PHP_URL_HOST);
+	}
+
+	$contact_url_host = parse_url($contact_url, PHP_URL_HOST);
+
+	if ($url_host == $contact_url_host) {
+		return;
+	}
+
+	Logger::error('URL check host mismatch', ['contact' => $contact_url, 'url' => $url]);
+	throw new \Friendica\Network\HTTPException\ForbiddenException(DI::l10n()->t('Access denied.'));
+}

mod/repair_ostatus.php

@@ -70,7 +70,7 @@ function repair_ostatus_content(App $a) {
 
 	$o .= "<p>".DI::l10n()->t("Keep this window open until done.")."</p>";
 
-	Contact::createFromProbe($uid, $r[0]["url"], true);
+	Contact::createFromProbe($a->user, $r[0]["url"], true);
 
 	DI::page()['htmlhead'] = '<meta http-equiv="refresh" content="1; URL=' . DI::baseUrl() . '/repair_ostatus?counter='.$counter.'">';
 

mod/salmon.php

@@ -42,15 +42,11 @@ function salmon_post(App $a, $xml = '') {
 
 	$nick       = (($a->argc > 1) ? Strings::escapeTags(trim($a->argv[1])) : '');
 
-	$r = q("SELECT * FROM `user` WHERE `nickname` = '%s' AND `account_expired` = 0 AND `account_removed` = 0 LIMIT 1",
+	$importer = DBA::selectFirst('user', [], ['nickname' => $nick, 'account_expired' => false, 'account_removed' => false]);
-		DBA::escape($nick)
+	if (! DBA::isResult($importer)) {
-	);
-	if (! DBA::isResult($r)) {
 		throw new \Friendica\Network\HTTPException\InternalServerErrorException();
 	}
 
-	$importer = $r[0];
-
 	// parse the xml
 
 	$dom = simplexml_load_string($xml,'SimpleXMLElement',0, ActivityNamespace::SALMON_ME);
@@ -83,7 +79,7 @@ function salmon_post(App $a, $xml = '') {
 	// stash away some other stuff for later
 
 	$type = $base->data[0]->attributes()->type[0];
-	$keyhash = $base->sig[0]->attributes()->keyhash[0];
+	$keyhash = $base->sig[0]->attributes()->keyhash[0] ?? '';
 	$encoding = $base->encoding;
 	$alg = $base->alg;
 
@@ -124,7 +120,7 @@ function salmon_post(App $a, $xml = '') {
 	$m = Strings::base64UrlDecode($key_info[1]);
 	$e = Strings::base64UrlDecode($key_info[2]);
 
-	Logger::log('key details: ' . print_r($key_info,true), Logger::DEBUG);
+	Logger::info('key details', ['info' => $key_info]);
 
 	$pubkey = Crypto::meToPem($m, $e);
 
@@ -175,7 +171,7 @@ function salmon_post(App $a, $xml = '') {
 		Logger::log('Author ' . $author_link . ' unknown to user ' . $importer['uid'] . '.');
 
 		if (DI::pConfig()->get($importer['uid'], 'system', 'ostatus_autofriend')) {
-			$result = Contact::createFromProbe($importer['uid'], $author_link);
+			$result = Contact::createFromProbe($importer, $author_link);
 
 			if ($result['success']) {
 				$r = q("SELECT * FROM `contact` WHERE `network` = '%s' AND ( `url` = '%s' OR `alias` = '%s')

mod/settings.php

@@ -183,7 +183,7 @@ function settings_post(App $a)
 					intval($mail_pubmail),
 					intval(local_user())
 				);
-				Logger::log("mail: updating mailaccount. Response: ".print_r($r, true));
+				Logger::notice('updating mailaccount', ['response' => $r]);
 				$r = q("SELECT * FROM `mailacct` WHERE `uid` = %d LIMIT 1",
 					intval(local_user())
 				);
@@ -823,44 +823,11 @@ function settings_content(App $a)
 		]);
 	}
 
+	$net_pub_desc = '';
 	if (strlen(DI::config()->get('system', 'directory'))) {
 		$net_pub_desc = ' ' . DI::l10n()->t('Your profile will also be published in the global friendica directories (e.g. <a href="%s">%s</a>).', DI::config()->get('system', 'directory'), DI::config()->get('system', 'directory'));
-	} else {
-		$net_pub_desc = '';
 	}
 
-	$profile_in_net_dir = Renderer::replaceMacros($opt_tpl, [
-		'$field' => ['profile_in_netdirectory', DI::l10n()->t('Allow your profile to be searchable globally?'), $profile['net-publish'], DI::l10n()->t("Activate this setting if you want others to easily find and follow you. Your profile will be searchable on remote systems. This setting also determines whether Friendica will inform search engines that your profile should be indexed or not.") . $net_pub_desc]
-	]);
-
-	$hide_friends = Renderer::replaceMacros($opt_tpl, [
-		'$field' => ['hide-friends', DI::l10n()->t('Hide your contact/friend list from viewers of your profile?'), $profile['hide-friends'], DI::l10n()->t('A list of your contacts is displayed on your profile page. Activate this option to disable the display of your contact list.')],
-	]);
-
-	$hide_wall = Renderer::replaceMacros($opt_tpl, [
-		'$field' => ['hidewall', DI::l10n()->t('Hide your profile details from anonymous viewers?'), $a->user['hidewall'], DI::l10n()->t('Anonymous visitors will only see your profile picture, your display name and the nickname you are using on your profile page. Your public posts and replies will still be accessible by other means.')],
-	]);
-
-	$unlisted = Renderer::replaceMacros($opt_tpl, [
-		'$field' => ['unlisted', DI::l10n()->t('Make public posts unlisted'), DI::pConfig()->get(local_user(), 'system', 'unlisted'), DI::l10n()->t('Your public posts will not appear on the community pages or in search results, nor be sent to relay servers. However they can still appear on public feeds on remote servers.')],
-	]);
-
-	$accessiblephotos = Renderer::replaceMacros($opt_tpl, [
-		'$field' => ['accessible-photos', DI::l10n()->t('Make all posted pictures accessible'), DI::pConfig()->get(local_user(), 'system', 'accessible-photos'), DI::l10n()->t("This option makes every posted picture accessible via the direct link. This is a workaround for the problem that most other networks can't handle permissions on pictures. Non public pictures still won't be visible for the public on your photo albums though.")],
-	]);
-
-	$blockwall = Renderer::replaceMacros($opt_tpl, [
-		'$field' => ['blockwall', DI::l10n()->t('Allow friends to post to your profile page?'), (intval($a->user['blockwall']) ? '0' : '1'), DI::l10n()->t('Your contacts may write posts on your profile wall. These posts will be distributed to your contacts')],
-	]);
-
-	$blocktags = Renderer::replaceMacros($opt_tpl, [
-		'$field' => ['blocktags', DI::l10n()->t('Allow friends to tag your posts?'), (intval($a->user['blocktags']) ? '0' : '1'), DI::l10n()->t('Your contacts can add additional tags to your posts.')],
-	]);
-
-	$unkmail = Renderer::replaceMacros($opt_tpl, [
-		'$field' => ['unkmail', DI::l10n()->t('Permit unknown people to send you private mail?'), $unkmail, DI::l10n()->t('Friendica network users may send you private messages even if they are not in your contact list.')],
-	]);
-
 	$tpl_addr = Renderer::getMarkupTemplate('settings/nick_set.tpl');
 
 	$prof_addr = Renderer::replaceMacros($tpl_addr,[
@@ -870,18 +837,6 @@ function settings_content(App $a)
 
 	$stpl = Renderer::getMarkupTemplate('settings/settings.tpl');
 
-	$expire_arr = [
-		'days' => ['expire',  DI::l10n()->t("Automatically expire posts after this many days:"), $expire, DI::l10n()->t('If empty, posts will not expire. Expired posts will be deleted')],
-		'label' => DI::l10n()->t('Expiration settings'),
-		'items' => ['expire_items', DI::l10n()->t('Expire posts'), $expire_items, DI::l10n()->t('When activated, posts and comments will be expired.')],
-		'notes' => ['expire_notes', DI::l10n()->t('Expire personal notes'), $expire_notes, DI::l10n()->t('When activated, the personal notes on your profile page will be expired.')],
-		'starred' => ['expire_starred', DI::l10n()->t('Expire starred posts'), $expire_starred, DI::l10n()->t('Starring posts keeps them from being expired. That behaviour is overwritten by this setting.')],
-		'photos' => ['expire_photos', DI::l10n()->t('Expire photos'), $expire_photos, DI::l10n()->t('When activated, photos will be expired.')],
-		'network_only' => ['expire_network_only', DI::l10n()->t('Only expire posts by others'), $expire_network_only, DI::l10n()->t('When activated, your own posts never expire. Then the settings above are only valid for posts you received.')],
-	];
-
-	$group_select = Group::displayGroupSelection(local_user(), $a->user['def_gid']);
-
 	// Private/public post links for the non-JS ACL form
 	$private_post = 1;
 	if (!empty($_REQUEST['public']) && !$_REQUEST['public']) {
@@ -932,41 +887,32 @@ function settings_content(App $a)
 		'$defloc'	=> ['defloc', DI::l10n()->t('Default Post Location:'), $defloc, ''],
 		'$allowloc' => ['allow_location', DI::l10n()->t('Use Browser Location:'), ($a->user['allow_location'] == 1), ''],
 
-
 		'$h_prv' 	          => DI::l10n()->t('Security and Privacy Settings'),
-
-		'$maxreq' 	=> ['maxreq', DI::l10n()->t('Maximum Friend Requests/Day:'), $maxreq , DI::l10n()->t("\x28to prevent spam abuse\x29")],
-		'$permissions' => DI::l10n()->t('Default Post Permissions'),
-		'$permdesc' => DI::l10n()->t("\x28click to open/close\x29"),
 		'$visibility'         => $profile['net-publish'],
-		'$aclselect' => ACL::getFullSelectorHTML(DI::page(), $a->user),
+		'$maxreq' 	          => ['maxreq', DI::l10n()->t('Maximum Friend Requests/Day:'), $maxreq , DI::l10n()->t("\x28to prevent spam abuse\x29")],
-		'$blockwall'=> $blockwall, // array('blockwall', DI::l10n()->t('Allow friends to post to your profile page:'), !$blockwall, ''),
-		'$blocktags'=> $blocktags, // array('blocktags', DI::l10n()->t('Allow friends to tag your posts:'), !$blocktags, ''),
-
-		// ACL permissions box
-		'$group_perms' => DI::l10n()->t('Show to Groups'),
-		'$contact_perms' => DI::l10n()->t('Show to Contacts'),
-		'$private' => DI::l10n()->t('Default Private Post'),
-		'$public' => DI::l10n()->t('Default Public Post'),
-		'$is_private' => $private_post,
-		'$return_path' => $query_str,
-		'$public_link' => $public_post_link,
-		'$settings_perms' => DI::l10n()->t('Default Permissions for New Posts'),
-
-		'$group_select' => $group_select,
-
-
-		'$expire'	=> $expire_arr,
-
 		'$profile_in_dir'     => $profile_in_dir,
-		'$profile_in_net_dir' => $profile_in_net_dir,
+		'$profile_in_net_dir' => ['profile_in_netdirectory', DI::l10n()->t('Allow your profile to be searchable globally?'), $profile['net-publish'], DI::l10n()->t("Activate this setting if you want others to easily find and follow you. Your profile will be searchable on remote systems. This setting also determines whether Friendica will inform search engines that your profile should be indexed or not.") . $net_pub_desc],
-		'$hide_friends' => $hide_friends,
+		'$hide_friends'       => ['hide-friends', DI::l10n()->t('Hide your contact/friend list from viewers of your profile?'), $profile['hide-friends'], DI::l10n()->t('A list of your contacts is displayed on your profile page. Activate this option to disable the display of your contact list.')],
-		'$hide_wall' => $hide_wall,
+		'$hide_wall'          => ['hidewall', DI::l10n()->t('Hide your profile details from anonymous viewers?'), $a->user['hidewall'], DI::l10n()->t('Anonymous visitors will only see your profile picture, your display name and the nickname you are using on your profile page. Your public posts and replies will still be accessible by other means.')],
-		'$unlisted' => $unlisted,
+		'$unlisted'           => ['unlisted', DI::l10n()->t('Make public posts unlisted'), DI::pConfig()->get(local_user(), 'system', 'unlisted'), DI::l10n()->t('Your public posts will not appear on the community pages or in search results, nor be sent to relay servers. However they can still appear on public feeds on remote servers.')],
-		'$accessiblephotos' => $accessiblephotos,
+		'$accessiblephotos'   => ['accessible-photos', DI::l10n()->t('Make all posted pictures accessible'), DI::pConfig()->get(local_user(), 'system', 'accessible-photos'), DI::l10n()->t("This option makes every posted picture accessible via the direct link. This is a workaround for the problem that most other networks can't handle permissions on pictures. Non public pictures still won't be visible for the public on your photo albums though.")],
-		'$unkmail' => $unkmail,
+		'$blockwall'          => ['blockwall', DI::l10n()->t('Allow friends to post to your profile page?'), (intval($a->user['blockwall']) ? '0' : '1'), DI::l10n()->t('Your contacts may write posts on your profile wall. These posts will be distributed to your contacts')], // array('blockwall', DI::l10n()->t('Allow friends to post to your profile page:'), !$blockwall, ''),
+		'$blocktags'          => ['blocktags', DI::l10n()->t('Allow friends to tag your posts?'), (intval($a->user['blocktags']) ? '0' : '1'), DI::l10n()->t('Your contacts can add additional tags to your posts.')], // array('blocktags', DI::l10n()->t('Allow friends to tag your posts:'), !$blocktags, ''),
+		'$unkmail'            => ['unkmail', DI::l10n()->t('Permit unknown people to send you private mail?'), $unkmail, DI::l10n()->t('Friendica network users may send you private messages even if they are not in your contact list.')],
 		'$cntunkmail' 	      => ['cntunkmail', DI::l10n()->t('Maximum private messages per day from unknown people:'), $cntunkmail , DI::l10n()->t("\x28to prevent spam abuse\x29")],
+		'$group_select'       => Group::displayGroupSelection(local_user(), $a->user['def_gid']),
+		'$permissions'        => DI::l10n()->t('Default Post Permissions'),
+		'$aclselect'          => ACL::getFullSelectorHTML(DI::page(), $a->user),
 
+		'$expire' => [
+			'label'        => DI::l10n()->t('Expiration settings'),
+			'days'         => ['expire',  DI::l10n()->t("Automatically expire posts after this many days:"), $expire, DI::l10n()->t('If empty, posts will not expire. Expired posts will be deleted')],
+			'items'        => ['expire_items', DI::l10n()->t('Expire posts'), $expire_items, DI::l10n()->t('When activated, posts and comments will be expired.')],
+			'notes'        => ['expire_notes', DI::l10n()->t('Expire personal notes'), $expire_notes, DI::l10n()->t('When activated, the personal notes on your profile page will be expired.')],
+			'starred'      => ['expire_starred', DI::l10n()->t('Expire starred posts'), $expire_starred, DI::l10n()->t('Starring posts keeps them from being expired. That behaviour is overwritten by this setting.')],
+			'photos'       => ['expire_photos', DI::l10n()->t('Expire photos'), $expire_photos, DI::l10n()->t('When activated, photos will be expired.')],
+			'network_only' => ['expire_network_only', DI::l10n()->t('Only expire posts by others'), $expire_network_only, DI::l10n()->t('When activated, your own posts never expire. Then the settings above are only valid for posts you received.')],
+		],
 
 		'$h_not' 	=> DI::l10n()->t('Notification Settings'),
 		'$lbl_not' 	=> DI::l10n()->t('Send a notification email when:'),

mod/share.php

@@ -20,6 +20,7 @@
  */
 
 use Friendica\App;
+use Friendica\Content\Text\BBCode;
 use Friendica\Database\DBA;
 use Friendica\Model\Item;
 
@@ -42,7 +43,7 @@ function share_init(App $a) {
 		$pos = strpos($item['body'], "[share");
 		$o = substr($item['body'], $pos);
 	} else {
-		$o = share_header($item['author-name'], $item['author-link'], $item['author-avatar'], $item['guid'], $item['created'], $item['plink']);
+		$o = BBCode::getShareOpeningTag($item['author-name'], $item['author-link'], $item['author-avatar'], $item['plink'], $item['created'], $item['guid']);
 
 		if ($item['title']) {
 			$o .= '[h3]'.$item['title'].'[/h3]'."\n";
@@ -55,22 +56,3 @@ function share_init(App $a) {
 	echo $o;
 	exit();
 }
-
-/// @TODO Rewrite to handle over whole record array
-function share_header($author, $profile, $avatar, $guid, $posted, $link) {
-	$header = "[share author='" . str_replace(["'", "[", "]"], ["'", "[", "]"], $author).
-		"' profile='" . str_replace(["'", "[", "]"], ["'", "[", "]"], $profile).
-		"' avatar='" . str_replace(["'", "[", "]"], ["'", "[", "]"], $avatar);
-
-	if ($guid) {
-		$header .= "' guid='" . str_replace(["'", "[", "]"], ["'", "[", "]"], $guid);
-	}
-
-	if ($posted) {
-		$header .= "' posted='" . str_replace(["'", "[", "]"], ["'", "[", "]"], $posted);
-	}
-
-	$header .= "' link='" . str_replace(["'", "[", "]"], ["'", "[", "]"], $link)."']";
-
-	return $header;
-}

mod/tagger.php

@@ -28,6 +28,7 @@ use Friendica\Core\Worker;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Item;
+use Friendica\Model\Tag;
 use Friendica\Protocol\Activity;
 use Friendica\Util\Strings;
 use Friendica\Util\XML;
@@ -168,47 +169,7 @@ EOT;
 		Item::update(['visible' => true], ['id' => $item['id']]);
 	}
 
-	$term_objtype = ($item['resource-id'] ? TERM_OBJ_PHOTO : TERM_OBJ_POST);
+	Tag::store($item['uri-id'], Tag::HASHTAG, $term);
-
-	$t = q("SELECT count(tid) as tcount FROM term WHERE oid=%d AND term='%s'",
-		intval($item['id']),
-		DBA::escape($term)
-	);
-
-	if (!$blocktags && $t[0]['tcount'] == 0) {
-		q("INSERT INTO term (oid, otype, type, term, url, uid) VALUE (%d, %d, %d, '%s', '%s', %d)",
-		   intval($item['id']),
-		   $term_objtype,
-		   TERM_HASHTAG,
-		   DBA::escape($term),
-		   '',
-		   intval($owner_uid)
-		);
-	}
-
-	// if the original post is on this site, update it.
-	$original_item = Item::selectFirst(['tag', 'id', 'uid'], ['origin' => true, 'uri' => $item['uri']]);
-	if (DBA::isResult($original_item)) {
-		$x = q("SELECT `blocktags` FROM `user` WHERE `uid`=%d LIMIT 1",
-			intval($original_item['uid'])
-		);
-		$t = q("SELECT COUNT(`tid`) AS `tcount` FROM `term` WHERE `oid`=%d AND `term`='%s'",
-			intval($original_item['id']),
-			DBA::escape($term)
-		);
-
-		if (DBA::isResult($x) && !$x[0]['blocktags'] && $t[0]['tcount'] == 0){
-			q("INSERT INTO term (`oid`, `otype`, `type`, `term`, `url`, `uid`) VALUE (%d, %d, %d, '%s', '%s', %d)",
-				intval($original_item['id']),
-				$term_objtype,
-				TERM_HASHTAG,
-				DBA::escape($term),
-				'',
-				intval($owner_uid)
-			);
-		}
-	}
-
 
 	$arr['id'] = $post_id;
 

mod/tagrm.php

@@ -24,7 +24,7 @@ use Friendica\Content\Text\BBCode;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Item;
-use Friendica\Model\Term;
+use Friendica\Model\Tag;
 use Friendica\Util\Strings;
 
 function tagrm_post(App $a)
@@ -57,29 +57,24 @@ function tagrm_post(App $a)
  * @param $tags array
  * @throws Exception
  */
-function update_tags($item_id, $tags){
+function update_tags($item_id, $tags)
+{
 	if (empty($item_id) || empty($tags)) {
 		return;
 	}
 
-	$item = Item::selectFirst(['tag'], ['id' => $item_id, 'uid' => local_user()]);
+	$item = Item::selectFirst(['uri-id'], ['id' => $item_id, 'uid' => local_user()]);
 	if (!DBA::isResult($item)) {
 		return;
 	}
 
-	$old_tags = explode(',', $item['tag']);
-
 	foreach ($tags as $new_tag) {
-		foreach ($old_tags as $index => $old_tag) {
+		if (preg_match_all('/([#@!])\[url\=([^\[\]]*)\]([^\[\]]*)\[\/url\]/ism', $new_tag, $results, PREG_SET_ORDER)) {
-			if (strcmp($old_tag, $new_tag) == 0) {
+			foreach ($results as $tag) {
-				unset($old_tags[$index]);
+				Tag::removeByHash($item['uri-id'], $tag[1], $tag[3], $tag[2]);
-				break;
 			}
 		}
 	}
-
-	$tag_str = implode(',', $old_tags);
-	Term::insertFromTagFieldByItemId($item_id, $tag_str);
 }
 
 function tagrm_content(App $a)
@@ -102,15 +97,16 @@ function tagrm_content(App $a)
 		// NOTREACHED
 	}
 
-	$item = Item::selectFirst(['tag'], ['id' => $item_id, 'uid' => local_user()]);
+	$item = Item::selectFirst(['uri-id'], ['id' => $item_id, 'uid' => local_user()]);
 	if (!DBA::isResult($item)) {
 		DI::baseUrl()->redirect($_SESSION['photo_return']);
 	}
 
-	$arr = explode(',', $item['tag']);
+	$tag_text = Tag::getCSVByURIId($item['uri-id']);
 
+	$arr = explode(',', $tag_text);
 
-	if (empty($item['tag'])) {
+	if (empty($arr)) {
 		DI::baseUrl()->redirect($_SESSION['photo_return']);
 	}
 

mod/videos.php

@@ -67,7 +67,7 @@ function videos_init(App $a)
 			'$photo' => $profile['photo'],
 			'$addr' => $profile['addr'] ?? '',
 			'$account_type' => $account_type,
-			'$about' => BBCode::convert($profile['about'] ?? ''),
+			'$about' => BBCode::convert($profile['about']),
 		]);
 
 		// If not there, create 'aside' empty

mod/wall_upload.php

@@ -41,19 +41,13 @@ function wall_upload_post(App $a, $desktopmode = true)
 	Logger::log("wall upload: starting new upload", Logger::DEBUG);
 
 	$r_json = (!empty($_GET['response']) && $_GET['response'] == 'json');
-	$album = (!empty($_GET['album']) ? Strings::escapeTags(trim($_GET['album'])) : '');
+	$album = trim($_GET['album'] ?? '');
 
 	if ($a->argc > 1) {
 		if (empty($_FILES['media'])) {
 			$nick = $a->argv[1];			
-			$r = q("SELECT `user`.*, `contact`.`id` FROM `user`
+			$user = DBA::selectFirst('owner-view', ['id', 'uid', 'nickname', 'page-flags'], ['nickname' => $nick, 'blocked' => false]);
-				INNER JOIN `contact` on `user`.`uid` = `contact`.`uid`
+			if (!DBA::isResult($user)) {
-				WHERE `user`.`nickname` = '%s' AND `user`.`blocked` = 0
-				AND `contact`.`self` = 1 LIMIT 1",
-				DBA::escape($nick)
-			);
-
-			if (!DBA::isResult($r)) {
 				if ($r_json) {
 					echo json_encode(['error' => DI::l10n()->t('Invalid request.')]);
 					exit();
@@ -62,12 +56,7 @@ function wall_upload_post(App $a, $desktopmode = true)
 			}
 		} else {
 			$user_info = api_get_user($a);
-			$r = q("SELECT `user`.*, `contact`.`id` FROM `user`
+			$user = DBA::selectFirst('owner-view', ['id', 'uid', 'nickname', 'page-flags'], ['nickname' => $user_info['screen_name'], 'blocked' => false]);
-				INNER JOIN `contact` on `user`.`uid` = `contact`.`uid`
-				WHERE `user`.`nickname` = '%s' AND `user`.`blocked` = 0
-				AND `contact`.`self` = 1 LIMIT 1",
-				DBA::escape($user_info['screen_name'])
-			);
 		}
 	} else {
 		if ($r_json) {
@@ -83,10 +72,10 @@ function wall_upload_post(App $a, $desktopmode = true)
 	$can_post  = false;
 	$visitor   = 0;
 
-	$page_owner_uid   = $r[0]['uid'];
+	$page_owner_uid   = $user['uid'];
-	$default_cid      = $r[0]['id'];
+	$default_cid      = $user['id'];
-	$page_owner_nick  = $r[0]['nickname'];
+	$page_owner_nick  = $user['nickname'];
-	$community_page   = (($r[0]['page-flags'] == User::PAGE_FLAGS_COMMUNITY) ? true : false);
+	$community_page   = (($user['page-flags'] == User::PAGE_FLAGS_COMMUNITY) ? true : false);
 
 	if ((local_user()) && (local_user() == $page_owner_uid)) {
 		$can_post = true;
@@ -174,23 +163,7 @@ function wall_upload_post(App $a, $desktopmode = true)
 		exit();
 	}
 
-	// This is a special treatment for picture upload from Twidere
+	$filetype = Images::getMimeTypeBySource($src, $filename, $filetype);
-	if (($filename == "octet-stream") && ($filetype != "")) {
-		$filename = $filetype;
-		$filetype = "";
-	}
-
-	if ($filetype == "") {
-		$filetype = Images::guessType($filename);
-	}
-
-	// If there is a temp name, then do a manual check
-	// This is more reliable than the provided value
-
-	$imagedata = getimagesize($src);
-	if ($imagedata) {
-		$filetype = $imagedata['mime'];
-	}
 
 	Logger::log("File upload src: " . $src . " - filename: " . $filename .
 		" - size: " . $filesize . " - type: " . $filetype, Logger::DEBUG);

mods/.drone.yml

@@ -36,7 +36,7 @@ volumes:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -79,7 +79,7 @@ volumes:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -122,7 +122,7 @@ volumes:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -169,7 +169,7 @@ volumes:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -211,7 +211,7 @@ volumes:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -253,7 +253,7 @@ volumes:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -282,7 +282,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -306,7 +306,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -330,7 +330,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -360,7 +360,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -384,7 +384,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -408,7 +408,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -439,7 +439,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -463,7 +463,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:
@@ -487,7 +487,7 @@ services:
 
 trigger:
   branch:
-#    - master
+#    - stable
     - develop
 #    - "*-rc"
 #  event:

src/App/Page.php

@@ -276,7 +276,7 @@ class Page implements ArrayAccess
 		// If you're just visiting, let javascript take you home
 		if (!empty($_SESSION['visitor_home'])) {
 			$homebase = $_SESSION['visitor_home'];
-		} elseif (local_user()) {
+		} elseif (!empty($app->user['nickname'])) {
 			$homebase = 'profile/' . $app->user['nickname'];
 		}
 

src/BaseModule.php

@@ -96,11 +96,11 @@ abstract class BaseModule
 	 * Functions used to protect against Cross-Site Request Forgery
 	 * The security token has to base on at least one value that an attacker can't know - here it's the session ID and the private key.
 	 * In this implementation, a security token is reusable (if the user submits a form, goes back and resubmits the form, maybe with small changes;
-	 * or if the security token is used for ajax-calls that happen several times), but only valid for a certain amout of time (3hours).
+	 * or if the security token is used for ajax-calls that happen several times), but only valid for a certain amount of time (3hours).
-	 * The "typename" seperates the security tokens of different types of forms. This could be relevant in the following case:
+	 * The "typename" separates the security tokens of different types of forms. This could be relevant in the following case:
-	 *    A security token is used to protekt a link from CSRF (e.g. the "delete this profile"-link).
+	 *    A security token is used to protect a link from CSRF (e.g. the "delete this profile"-link).
 	 *    If the new page contains by any chance external elements, then the used security token is exposed by the referrer.
-	 *    Actually, important actions should not be triggered by Links / GET-Requests at all, but somethimes they still are,
+	 *    Actually, important actions should not be triggered by Links / GET-Requests at all, but sometimes they still are,
 	 *    so this mechanism brings in some damage control (the attacker would be able to forge a request to a form of this type, but not to forms of other types).
 	 */
 	public static function getFormSecurityToken($typename = '')
@@ -108,7 +108,7 @@ abstract class BaseModule
 		$a = DI::app();
 
 		$timestamp = time();
-		$sec_hash = hash('whirlpool', $a->user['guid'] . $a->user['prvkey'] . session_id() . $timestamp . $typename);
+		$sec_hash = hash('whirlpool', ($a->user['guid'] ?? '') . ($a->user['prvkey'] ?? '') . session_id() . $timestamp . $typename);
 
 		return $timestamp . '.' . $sec_hash;
 	}

src/Console/DatabaseStructure.php

@@ -54,7 +54,7 @@ Commands
 	dryrun   Show database update schema queries without running them
 	update   Update database schema
 	dumpsql  Dump database schema
-	toinnodb Convert all tables from MyISAM to InnoDB
+	toinnodb Convert all tables from MyISAM or InnoDB in the Antelope file format to InnoDB in the Barracuda file format
 
 Options
     -h|--help|-?       Show help information

src/Console/GlobalCommunitySilence.php

@@ -50,7 +50,7 @@ class GlobalCommunitySilence extends \Asika\SimpleConsole\Console
 	protected function getHelp()
 	{
 		$help = <<<HELP
-console globalcommunitysilence - Silence remote profile from global community page
+console globalcommunitysilence - Silence a profile from the global community page
 Usage
 	bin/console globalcommunitysilence <profile_url> [-h|--help|-?] [-v]
 

src/Console/User.php

@@ -59,7 +59,7 @@ console user - Modify user settings per console commands.
 Usage
 	bin/console user password <nickname> [<password>] [-h|--help|-?] [-v]
 	bin/console user add [<name> [<nickname> [<email> [<language>]]]] [-h|--help|-?] [-v]
-	bin/console user delete [<nickname>] [-q] [-h|--help|-?] [-v]
+	bin/console user delete [<nickname>] [-y] [-h|--help|-?] [-v]
 	bin/console user allow [<nickname>] [-h|--help|-?] [-v]
 	bin/console user deny [<nickname>] [-h|--help|-?] [-v]
 	bin/console user block [<nickname>] [-h|--help|-?] [-v]
@@ -78,8 +78,8 @@ Description
 
 Options
     -h|--help|-? Show help information
-    -v           Show more debug information.
+    -v           Show more debug information
-    -q           Quiet mode (don't ask for a command).
+    -y           Non-interactive mode, assume "yes" as answer to the user deletion prompt
 HELP;
 		return $help;
 	}
@@ -304,19 +304,24 @@ HELP;
 			}
 		}
 
-		$user = $this->dba->selectFirst('user', ['uid'], ['nickname' => $nick]);
+		$user = $this->dba->selectFirst('user', ['uid', 'account_removed'], ['nickname' => $nick]);
 		if (empty($user)) {
 			throw new RuntimeException($this->l10n->t('User not found'));
 		}
 
-		if (!$this->getOption('q')) {
+		if (!empty($user['account_removed'])) {
+			$this->out($this->l10n->t('User has already been marked for deletion.'));
+			return true;
+		}
+
+		if (!$this->getOption('y')) {
 			$this->out($this->l10n->t('Type "yes" to delete %s', $nick));
 			if (CliPrompt::prompt() !== 'yes') {
-				throw new RuntimeException('Delete abort.');
+				throw new RuntimeException($this->l10n->t('Deletion aborted.'));
 			}
 		}
 
-		return UserModel::remove($user['uid'] ?? -1);
+		return UserModel::remove($user['uid']);
 	}
 
 	/**
@@ -361,7 +366,7 @@ HELP;
 						$contact['email'],
 						Temporal::getRelativeDate($contact['created']),
 						Temporal::getRelativeDate($contact['login_date']),
-						Temporal::getRelativeDate($contact['lastitem_date']),
+						Temporal::getRelativeDate($contact['last-item']),
 					]);
 				}
 				$this->out($table->getTable());

src/Content/ForumManager.php

@@ -126,7 +126,7 @@ class ForumManager
 				$selected = (($cid == $contact['id']) ? ' forum-selected' : '');
 
 				$entry = [
-					'url' => 'network?cid=' . $contact['id'],
+					'url' => 'network?contactid=' . $contact['id'],
 					'external_url' => Contact::magicLink($contact['url']),
 					'name' => $contact['name'],
 					'cid' => $contact['id'],

src/Content/Item.php

@@ -21,7 +21,10 @@
 
 namespace Friendica\Content;
 
+use Friendica\Database\DBA;
+use Friendica\Model\Contact;
 use Friendica\Model\FileTag;
+use Friendica\Model\Tag;
 
 /**
  * A content helper class for displaying items
@@ -100,4 +103,136 @@ class Item
 
 		return [$categories, $folders];
 	}
+
+	/**
+	 * This function removes the tag $tag from the text $body and replaces it with
+	 * the appropriate link.
+	 *
+	 * @param string  $body        the text to replace the tag in
+	 * @param string  $inform      a comma-seperated string containing everybody to inform
+	 * @param integer $profile_uid the user id to replace the tag for (0 = anyone)
+	 * @param string  $tag         the tag to replace
+	 * @param string  $network     The network of the post
+	 *
+	 * @return array|bool ['replaced' => $replaced, 'contact' => $contact];
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	public static function replaceTag(&$body, &$inform, $profile_uid, $tag, $network = '')
+	{
+		$replaced = false;
+
+		//is it a person tag?
+		if (Tag::isType($tag, Tag::MENTION, Tag::IMPLICIT_MENTION, Tag::EXCLUSIVE_MENTION)) {
+			$tag_type = substr($tag, 0, 1);
+			//is it already replaced?
+			if (strpos($tag, '[url=')) {
+				// Checking for the alias that is used for OStatus
+				$pattern = '/[@!]\[url\=(.*?)\](.*?)\[\/url\]/ism';
+				if (preg_match($pattern, $tag, $matches)) {
+					$data = Contact::getDetailsByURL($matches[1]);
+
+					if ($data['alias'] != '') {
+						$newtag = '@[url=' . $data['alias'] . ']' . $data['nick'] . '[/url]';
+					}
+				}
+
+				return $replaced;
+			}
+
+			//get the person's name
+			$name = substr($tag, 1);
+
+			// Sometimes the tag detection doesn't seem to work right
+			// This is some workaround
+			$nameparts = explode(' ', $name);
+			$name = $nameparts[0];
+
+			// Try to detect the contact in various ways
+			if (strpos($name, 'http://')) {
+				// At first we have to ensure that the contact exists
+				Contact::getIdForURL($name);
+
+				// Now we should have something
+				$contact = Contact::getDetailsByURL($name, $profile_uid);
+			} elseif (strpos($name, '@')) {
+				// This function automatically probes when no entry was found
+				$contact = Contact::getDetailsByAddr($name, $profile_uid);
+			} else {
+				$contact = false;
+				$fields = ['id', 'url', 'nick', 'name', 'alias', 'network', 'forum', 'prv'];
+
+				if (strrpos($name, '+')) {
+					// Is it in format @nick+number?
+					$tagcid = intval(substr($name, strrpos($name, '+') + 1));
+					$contact = DBA::selectFirst('contact', $fields, ['id' => $tagcid, 'uid' => $profile_uid]);
+				}
+
+				// select someone by nick in the current network
+				if (!DBA::isResult($contact) && ($network != '')) {
+					$condition = ["`nick` = ? AND `network` = ? AND `uid` = ?",
+						$name, $network, $profile_uid];
+					$contact = DBA::selectFirst('contact', $fields, $condition);
+				}
+
+				// select someone by attag in the current network
+				if (!DBA::isResult($contact) && ($network != '')) {
+					$condition = ["`attag` = ? AND `network` = ? AND `uid` = ?",
+						$name, $network, $profile_uid];
+					$contact = DBA::selectFirst('contact', $fields, $condition);
+				}
+
+				//select someone by name in the current network
+				if (!DBA::isResult($contact) && ($network != '')) {
+					$condition = ['name' => $name, 'network' => $network, 'uid' => $profile_uid];
+					$contact = DBA::selectFirst('contact', $fields, $condition);
+				}
+
+				// select someone by nick in any network
+				if (!DBA::isResult($contact)) {
+					$condition = ["`nick` = ? AND `uid` = ?", $name, $profile_uid];
+					$contact = DBA::selectFirst('contact', $fields, $condition);
+				}
+
+				// select someone by attag in any network
+				if (!DBA::isResult($contact)) {
+					$condition = ["`attag` = ? AND `uid` = ?", $name, $profile_uid];
+					$contact = DBA::selectFirst('contact', $fields, $condition);
+				}
+
+				// select someone by name in any network
+				if (!DBA::isResult($contact)) {
+					$condition = ['name' => $name, 'uid' => $profile_uid];
+					$contact = DBA::selectFirst('contact', $fields, $condition);
+				}
+			}
+
+			// Check if $contact has been successfully loaded
+			if (DBA::isResult($contact)) {
+				if (strlen($inform) && (isset($contact['notify']) || isset($contact['id']))) {
+					$inform .= ',';
+				}
+
+				if (isset($contact['id'])) {
+					$inform .= 'cid:' . $contact['id'];
+				} elseif (isset($contact['notify'])) {
+					$inform  .= $contact['notify'];
+				}
+
+				$profile = $contact['url'];
+				$newname = ($contact['name'] ?? '') ?: $contact['nick'];
+			}
+
+			//if there is an url for this persons profile
+			if (isset($profile) && ($newname != '')) {
+				$replaced = true;
+				// create profile link
+				$profile = str_replace(',', '%2c', $profile);
+				$newtag = $tag_type.'[url=' . $profile . ']' . $newname . '[/url]';
+				$body = str_replace($tag_type . $name, $newtag, $body);
+			}
+		}
+
+		return ['replaced' => $replaced, 'contact' => $contact];
+	}
 }

src/Content/Nav.php

@@ -171,6 +171,7 @@ class Nav
 		}
 
 		if (local_user()) {
+			if (!empty($a->user)) {
 				// user menu
 				$nav['usermenu'][] = ['profile/' . $a->user['nickname'], DI::l10n()->t('Status'), '', DI::l10n()->t('Your posts and conversations')];
 				$nav['usermenu'][] = ['profile/' . $a->user['nickname'] . '/profile', DI::l10n()->t('Profile'), '', DI::l10n()->t('Your profile page')];
@@ -185,6 +186,9 @@ class Nav
 					'icon' => (DBA::isResult($contact) ? DI::baseUrl()->remove($contact['micro']) : 'images/person-48.jpg'),
 					'name' => $a->user['username'],
 				];
+			} else {
+				DI::logger()->warning('Empty $a->user for local user', ['local_user' => local_user(), '$a' => $a]);
+			}
 		}
 
 		// "Home" should also take you home from an authenticated remote profile connection
@@ -252,7 +256,7 @@ class Nav
 		}
 
 		// The following nav links are only show to logged in users
-		if (local_user()) {
+		if (local_user() && !empty($a->user)) {
 			$nav['network'] = ['network', DI::l10n()->t('Network'), '', DI::l10n()->t('Conversations from your friends')];
 
 			$nav['home'] = ['profile/' . $a->user['nickname'], DI::l10n()->t('Home'), '', DI::l10n()->t('Your posts and conversations')];

src/Content/PageInfo.php

@@ -0,0 +1,273 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Content;
+
+use Friendica\Core\Hook;
+use Friendica\Core\Logger;
+use Friendica\DI;
+use Friendica\Network\HTTPException;
+use Friendica\Util\ParseUrl;
+use Friendica\Util\Strings;
+
+/**
+ * Extracts trailing URLs from post bodies to transform them in enriched attachment tags through Site Info query
+ */
+class PageInfo
+{
+	/**
+	 * @param string $body
+	 * @param bool   $searchNakedUrls
+	 * @param bool   $no_photos
+	 * @return string
+	 * @throws HTTPException\InternalServerErrorException
+	 */
+	public static function appendToBody(string $body, bool $searchNakedUrls = false, bool $no_photos = false)
+	{
+		Logger::info('add_page_info_to_body: fetch page info for body', ['body' => $body]);
+
+		$url = self::getRelevantUrlFromBody($body, $searchNakedUrls);
+		if (!$url) {
+			return $body;
+		}
+
+		$footer = self::getFooterFromUrl($url, $no_photos);
+		if (!$footer) {
+			return $body;
+		}
+
+		$body = self::stripTrailingUrlFromBody($body, $url);
+
+		$body .= "\n" . $footer;
+
+		return $body;
+	}
+
+	/**
+	 * @param string $url
+	 * @param bool $no_photos
+	 * @param string $photo
+	 * @param bool $keywords
+	 * @param string $keyword_denylist
+	 * @return string
+	 * @throws HTTPException\InternalServerErrorException
+	 */
+	public static function getFooterFromUrl(string $url, bool $no_photos = false, string $photo = '', bool $keywords = false, string $keyword_denylist = '')
+	{
+		$data = self::queryUrl($url, $photo, $keywords, $keyword_denylist);
+
+		return self::getFooterFromData($data, $no_photos);
+	}
+
+	/**
+	 * @param array $data
+	 * @param bool  $no_photos
+	 * @return string
+	 * @throws HTTPException\InternalServerErrorException
+	 */
+	public static function getFooterFromData(array $data, bool $no_photos = false)
+	{
+		Hook::callAll('page_info_data', $data);
+
+		if (empty($data['type'])) {
+			return '';
+		}
+
+		// It maybe is a rich content, but if it does have everything that a link has,
+		// then treat it that way
+		if (($data['type'] == 'rich') && is_string($data['title']) &&
+			is_string($data['text']) && !empty($data['images'])) {
+			$data['type'] = 'link';
+		}
+
+		$data['title'] = $data['title'] ?? '';
+
+		if ((($data['type'] != 'link') && ($data['type'] != 'video') && ($data['type'] != 'photo')) || ($data['title'] == $data['url'])) {
+			return '';
+		}
+
+		if ($no_photos && ($data['type'] == 'photo')) {
+			return '';
+		}
+
+		// Escape some bad characters
+		$data['url'] = str_replace(['[', ']'], ['&#91;', '&#93;'], htmlentities($data['url'], ENT_QUOTES, 'UTF-8', false));
+		$data['title'] = str_replace(['[', ']'], ['&#91;', '&#93;'], htmlentities($data['title'], ENT_QUOTES, 'UTF-8', false));
+
+		$text = "[attachment type='" . $data['type'] . "'";
+
+		if (empty($data['text'])) {
+			$data['text'] = $data['title'];
+		}
+
+		if (empty($data['text'])) {
+			$data['text'] = $data['url'];
+		}
+
+		if (!empty($data['url'])) {
+			$text .= " url='" . $data['url'] . "'";
+		}
+
+		if (!empty($data['title'])) {
+			$text .= " title='" . $data['title'] . "'";
+		}
+
+		// Only embedd a picture link when it seems to be a valid picture ("width" is set)
+		if (!empty($data['images']) && !empty($data['images'][0]['width'])) {
+			$preview = str_replace(['[', ']'], ['&#91;', '&#93;'], htmlentities($data['images'][0]['src'], ENT_QUOTES, 'UTF-8', false));
+			// if the preview picture is larger than 500 pixels then show it in a larger mode
+			// But only, if the picture isn't higher than large (To prevent huge posts)
+			if (!DI::config()->get('system', 'always_show_preview') && ($data['images'][0]['width'] >= 500)
+				&& ($data['images'][0]['width'] >= $data['images'][0]['height'])) {
+				$text .= " image='" . $preview . "'";
+			} else {
+				$text .= " preview='" . $preview . "'";
+			}
+		}
+
+		$text .= ']' . $data['text'] . '[/attachment]';
+
+		$hashtags = '';
+		if (!empty($data['keywords'])) {
+			$hashtags = "\n";
+			foreach ($data['keywords'] as $keyword) {
+				/// @TODO make a positive list of allowed characters
+				$hashtag = str_replace([' ', '+', '/', '.', '#', '@', "'", '"', '’', '`', '(', ')', '„', '“'], '', $keyword);
+				$hashtags .= '#[url=' . DI::baseUrl() . '/search?tag=' . $hashtag . ']' . $hashtag . '[/url] ';
+			}
+		}
+
+		return $text . $hashtags;
+	}
+
+	/**
+	 * @param string  $url
+	 * @param string $photo
+	 * @param bool $keywords
+	 * @param string $keyword_denylist
+	 * @return array|bool
+	 * @throws HTTPException\InternalServerErrorException
+	 */
+	public static function queryUrl(string $url, string $photo = '', bool $keywords = false, string $keyword_denylist = '')
+	{
+		$data = ParseUrl::getSiteinfoCached($url, true);
+
+		if ($photo != '') {
+			$data['images'][0]['src'] = $photo;
+		}
+
+		if (!$keywords) {
+			unset($data['keywords']);
+		} elseif ($keyword_denylist && !empty($data['keywords'])) {
+			$list = explode(', ', $keyword_denylist);
+
+			foreach ($list as $keyword) {
+				$keyword = trim($keyword);
+
+				$index = array_search($keyword, $data['keywords']);
+				if ($index !== false) {
+					unset($data['keywords'][$index]);
+				}
+			}
+		}
+
+		Logger::info('fetch page info for URL', ['url' => $url, 'data' => $data]);
+
+		return $data;
+	}
+
+	/**
+	 * @param string $url
+	 * @param string $photo
+	 * @param string $keyword_denylist
+	 * @return array
+	 * @throws HTTPException\InternalServerErrorException
+	 */
+	public static function getTagsFromUrl(string $url, string $photo = '', string $keyword_denylist = '')
+	{
+		$data = self::queryUrl($url, $photo, true, $keyword_denylist);
+
+		if (empty($data['keywords'])) {
+			return [];
+		}
+
+		$taglist = [];
+		foreach ($data['keywords'] as $keyword) {
+			$hashtag = str_replace([' ', '+', '/', '.', '#', "'"],
+				['', '', '', '', '', ''], $keyword);
+
+			$taglist[] = $hashtag;
+		}
+
+		return $taglist;
+	}
+
+	/**
+	 * Picks a non-hashtag, non-mention, schemeful URL at the end of the provided body string to be converted into Page Info.
+	 *
+	 * @param string $body
+	 * @param bool   $searchNakedUrls Whether we should pick a naked URL (outside of BBCode tags) as a last resort
+	 * @return string|null
+	 */
+	protected static function getRelevantUrlFromBody(string $body, bool $searchNakedUrls = false)
+	{
+		$URLSearchString = 'https?://[^\[\]]*';
+
+		// Fix for Mastodon where the mentions are in a different format
+		$body = preg_replace("~\[url=($URLSearchString)]([#!@])(.*?)\[/url]~is", '$2[url=$1]$3[/url]', $body);
+
+		preg_match("~(?<![!#@])\[url]($URLSearchString)\[/url]$~is", $body, $matches);
+
+		if (!$matches) {
+			preg_match("~(?<![!#@])\[url=($URLSearchString)].*\[/url]$~is", $body, $matches);
+		}
+
+		if (!$matches && $searchNakedUrls) {
+			preg_match('~(?<=\W|^)(?<![=\]])(https?://.+)$~is', $body, $matches);
+			if ($matches && !Strings::endsWith($body, $matches[1])) {
+				unset($matches);
+			}
+		}
+
+		return $matches[1] ?? null;
+	}
+
+	/**
+	 * Remove the provided URL from the body if it is at the end of it.
+	 * Keep the link label if it isn't the full URL.
+	 *
+	 * @param string $body
+	 * @param string $url
+	 * @return string|string[]|null
+	 */
+	protected static function stripTrailingUrlFromBody(string $body, string $url)
+	{
+		$quotedUrl = preg_quote($url, '#');
+		$body = preg_replace("#(?:
+			\[url]$quotedUrl\[/url]|
+			\[url=$quotedUrl]$quotedUrl\[/url]|
+			\[url=$quotedUrl]([^[]*?)\[/url]|
+			$quotedUrl
+		)$#isx", '$1', $body);
+
+		return $body;
+	}
+}

src/Content/Text/BBCode.php

@@ -24,6 +24,8 @@ namespace Friendica\Content\Text;
 use DOMDocument;
 use DOMXPath;
 use Exception;
+use Friendica\Content\ContactSelector;
+use Friendica\Content\Item;
 use Friendica\Content\OEmbed;
 use Friendica\Content\Smilies;
 use Friendica\Core\Hook;
@@ -35,6 +37,7 @@ use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Model\Event;
 use Friendica\Model\Photo;
+use Friendica\Model\Tag;
 use Friendica\Network\Probe;
 use Friendica\Object\Image;
 use Friendica\Protocol\Activity;
@@ -48,6 +51,15 @@ use Friendica\Util\XML;
 
 class BBCode
 {
+	const INTERNAL = 0;
+	const API = 2;
+	const DIASPORA = 3;
+	const CONNECTORS = 4;
+	const OSTATUS = 7;
+	const TWITTER = 8;
+	const BACKLINK = 8;
+	const ACTIVITYPUB = 9;
+
 	/**
 	 * Fetches attachment data that were generated the old way
 	 *
@@ -434,25 +446,36 @@ class BBCode
 	 */
 	public static function toPlaintext($text, $keep_urls = true)
 	{
-		$naked_text = HTML::toPlaintext(BBCode::convert($text, false, 0, true), 0, !$keep_urls);
+		$naked_text = HTML::toPlaintext(self::convert($text, false, 0, true), 0, !$keep_urls);
 
 		return $naked_text;
 	}
 
-	private static function proxyUrl($image, $simplehtml = false)
+	private static function proxyUrl($image, $simplehtml = self::INTERNAL)
 	{
 		// Only send proxied pictures to API and for internal display
-		if (in_array($simplehtml, [false, 2])) {
+		if (in_array($simplehtml, [self::INTERNAL, self::API])) {
 			return ProxyUtils::proxifyUrl($image);
 		} else {
 			return $image;
 		}
 	}
 
-	public static function scaleExternalImages($srctext)
+	/**
+	 * This function changing the visual size (not the real size) of images.
+	 * The function does not work for pictures with an alternate text description.
+	 * This could only be changed by using some new "img" BBCode format.
+	 *
+	 * @param string $srctext The body with images
+	 * @return string The body with possibly scaled images
+	 */
+	public static function scaleExternalImages(string $srctext)
 	{
 		$s = $srctext;
 
+		// Simplify image links
+		$s = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '[img]$3[/img]', $s);
+
 		$matches = null;
 		$c = preg_match_all('/\[img.*?\](.*?)\[\/img\]/ism', $s, $matches, PREG_SET_ORDER);
 		if ($c) {
@@ -464,13 +487,14 @@ class BBCode
 					continue;
 				}
 
-				$i = Network::fetchUrl($mtch[1]);
+				$curlResult = Network::curl($mtch[1], true);
-				if (!$i) {
+				if (!$curlResult->isSuccess()) {
-					return $srctext;
+					continue;
 				}
 
-				// guess mimetype from headers or filename
+				$i = $curlResult->getBody();
-				$type = Images::guessType($mtch[1], true);
+				$type = $curlResult->getContentType();
+				$type = Images::getMimeTypeByData($i, $mtch[1], $type);
 
 				if ($i) {
 					$Image = new Image($i, $type);
@@ -482,14 +506,14 @@ class BBCode
 							$Image->scaleDown(640);
 							$new_width = $Image->getWidth();
 							$new_height = $Image->getHeight();
-							Logger::log('scale_external_images: ' . $orig_width . '->' . $new_width . 'w ' . $orig_height . '->' . $new_height . 'h' . ' match: ' . $mtch[0], Logger::DEBUG);
+							Logger::info('External images scaled', ['orig_width' => $orig_width, 'new_width' => $new_width, 'orig_height' => $orig_height, 'new_height' => $new_height, 'match' => $mtch[0]]);
 							$s = str_replace(
 								$mtch[0],
 								'[img=' . $new_width . 'x' . $new_height. ']' . $mtch[1] . '[/img]'
 								. "\n",
 								$s
 							);
-							Logger::log('scale_external_images: new string: ' . $s, Logger::DEBUG);
+							Logger::info('New string', ['image' => $s]);
 						}
 					}
 				}
@@ -517,7 +541,7 @@ class BBCode
 		// than the maximum, then don't waste time looking for the images
 		if ($maxlen && (strlen($body) > $maxlen)) {
 
-			Logger::log('the total body length exceeds the limit', Logger::DEBUG);
+			Logger::info('the total body length exceeds the limit', ['maxlen' => $maxlen, 'body_len' => strlen($body)]);
 
 			$orig_body = $body;
 			$new_body = '';
@@ -537,7 +561,7 @@ class BBCode
 
 					if (($textlen + $img_start) > $maxlen) {
 						if ($textlen < $maxlen) {
-							Logger::log('the limit happens before an embedded image', Logger::DEBUG);
+							Logger::info('the limit happens before an embedded image');
 							$new_body = $new_body . substr($orig_body, 0, $maxlen - $textlen);
 							$textlen = $maxlen;
 						}
@@ -551,7 +575,7 @@ class BBCode
 
 					if (($textlen + $img_end) > $maxlen) {
 						if ($textlen < $maxlen) {
-							Logger::log('the limit happens before the end of a non-embedded image', Logger::DEBUG);
+							Logger::info('the limit happens before the end of a non-embedded image');
 							$new_body = $new_body . substr($orig_body, 0, $maxlen - $textlen);
 							$textlen = $maxlen;
 						}
@@ -574,11 +598,11 @@ class BBCode
 
 			if (($textlen + strlen($orig_body)) > $maxlen) {
 				if ($textlen < $maxlen) {
-					Logger::log('the limit happens after the end of the last image', Logger::DEBUG);
+					Logger::info('the limit happens after the end of the last image');
 					$new_body = $new_body . substr($orig_body, 0, $maxlen - $textlen);
 				}
 			} else {
-				Logger::log('the text size with embedded images extracted did not violate the limit', Logger::DEBUG);
+				Logger::info('the text size with embedded images extracted did not violate the limit');
 				$new_body = $new_body . $orig_body;
 			}
 
@@ -594,12 +618,12 @@ class BBCode
 	 * Note: Can produce a [bookmark] tag in the returned string
 	 *
 	 * @param string  $text
-	 * @param bool|int $simplehtml
+	 * @param integer $simplehtml
 	 * @param bool    $tryoembed
 	 * @return string
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 */
-	private static function convertAttachment($text, $simplehtml = false, $tryoembed = true)
+	private static function convertAttachment($text, $simplehtml = self::INTERNAL, $tryoembed = true)
 	{
 		$data = self::getAttachmentData($text);
 		if (empty($data) || empty($data['url'])) {
@@ -628,7 +652,7 @@ class BBCode
 		} catch (Exception $e) {
 			$data['title'] = ($data['title'] ?? '') ?: $data['url'];
 
-			if ($simplehtml != 4) {
+			if ($simplehtml != self::CONNECTORS) {
 				$return = sprintf('<div class="type-%s">', $data['type']);
 			}
 
@@ -655,7 +679,7 @@ class BBCode
 				$return .= sprintf('<sup><a href="%s">%s</a></sup>', $data['url'], parse_url($data['url'], PHP_URL_HOST));
 			}
 
-			if ($simplehtml != 4) {
+			if ($simplehtml != self::CONNECTORS) {
 				$return .= '</div>';
 			}
 		}
@@ -954,27 +978,12 @@ class BBCode
 			function ($match) use ($callback) {
 				$attribute_string = $match[2];
 				$attributes = [];
-				foreach (['author', 'profile', 'avatar', 'link', 'posted'] as $field) {
+				foreach (['author', 'profile', 'avatar', 'link', 'posted', 'guid'] as $field) {
 					preg_match("/$field=(['\"])(.+?)\\1/ism", $attribute_string, $matches);
 					$attributes[$field] = html_entity_decode($matches[2] ?? '', ENT_QUOTES, 'UTF-8');
 				}
 
-				// We only call this so that a previously unknown contact can be added.
+				$author_contact = Contact::getByURL($attributes['profile'], 0, ['url', 'addr', 'name', 'micro'], false);
-				// This is important for the function "Model\Contact::getDetailsByURL()".
-				// This function then can fetch an entry from the contact table.
-				$default['url'] = $attributes['profile'];
-
-				if (!empty($attributes['author'])) {
-					$default['name'] = $attributes['author'];
-				}
-
-				if (!empty($attributes['avatar'])) {
-					$default['photo'] = $attributes['avatar'];
-				}
-
-				Contact::getIdForURL($attributes['profile'], 0, true, $default);
-
-				$author_contact = Contact::getDetailsByURL($attributes['profile']);
 				$author_contact['url'] = ($author_contact['url'] ?? $attributes['profile']);
 				$author_contact['addr'] = ($author_contact['addr'] ?? '') ?: Protocol::getAddrFromProfileUrl($attributes['profile']);
 
@@ -1013,13 +1022,10 @@ class BBCode
 		$mention = Protocol::formatMention($attributes['profile'], $attributes['author']);
 
 		switch ($simplehtml) {
-			case 1:
+			case self::API:
-				$text = ($is_quote_share? '<br />' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' <a href="' . $attributes['profile'] . '">' . $mention . '</a>: </p>' . "\n" . '«' . $content . '»';
-				break;
-			case 2:
 				$text = ($is_quote_share? '<br />' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' ' . $author_contact['addr'] . ': </p>' . "\n" . $content;
 				break;
-			case 3: // Diaspora
+			case self::DIASPORA:
 				if (stripos(Strings::normaliseLink($attributes['link']), 'http://twitter.com/') === 0) {
 					$text = ($is_quote_share? '<hr />' : '') . '<p><a href="' . $attributes['link'] . '">' . $attributes['link'] . '</a></p>' . "\n";
 				} else {
@@ -1037,7 +1043,7 @@ class BBCode
 				}
 
 				break;
-			case 4:
+			case self::CONNECTORS:
 				$headline = '<p><b>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8');
 				$headline .= DI::l10n()->t('<a href="%1$s" target="_blank" rel="noopener noreferrer">%2$s</a> %3$s', $attributes['link'], $mention, $attributes['posted']);
 				$headline .= ':</b></p>' . "\n";
@@ -1045,37 +1051,32 @@ class BBCode
 				$text = ($is_quote_share? '<hr />' : '') . $headline . '<blockquote class="shared_content">' . trim($content) . '</blockquote>' . "\n";
 
 				break;
-			case 5:
+			case self::OSTATUS:
-				$text = ($is_quote_share? '<br />' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' ' . $author_contact['addr'] . ': </p>' . "\n" . $content;
-				break;
-			case 7: // statusnet/GNU Social
 				$text = ($is_quote_share? '<br />' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' @' . $author_contact['addr'] . ': ' . $content . '</p>' . "\n";
 				break;
-			case 9: // ActivityPub
+			case self::ACTIVITYPUB:
 				$author = '@<span class="vcard"><a href="' . $author_contact['url'] . '" class="url u-url mention" title="' . $author_contact['addr'] . '"><span class="fn nickname mention">' . $author_contact['addr'] . '</span></a>:</span>';
 				$text = '<div><a href="' . $attributes['link'] . '">' . html_entity_decode('&#x2672;', ENT_QUOTES, 'UTF-8') . '</a> ' . $author . '<blockquote>' . $content . '</blockquote></div>' . "\n";
 				break;
 			default:
-				// Transforms quoted tweets in rich attachments to avoid nested tweets
-				if (stripos(Strings::normaliseLink($attributes['link']), 'http://twitter.com/') === 0 && OEmbed::isAllowedURL($attributes['link'])) {
-					try {
-						$text = ($is_quote_share? '<br />' : '') . OEmbed::getHTML($attributes['link']);
-					} catch (Exception $e) {
-						$text = ($is_quote_share? '<br />' : '') . sprintf('[bookmark=%s]%s[/bookmark]', $attributes['link'], $content);
-					}
-				} else {
 				$text = ($is_quote_share? "\n" : '');
 
+				$contact = Contact::getByURL($attributes['profile'], 0, ['network'], false);
+				$network = $contact['network'] ?? Protocol::PHANTOM;
+
 				$tpl = Renderer::getMarkupTemplate('shared_content.tpl');
 				$text .= Renderer::replaceMacros($tpl, [
 					'$profile'      => $attributes['profile'],
 					'$avatar'       => $attributes['avatar'],
 					'$author'       => $attributes['author'],
 					'$link'         => $attributes['link'],
+					'$link_title'   => DI::l10n()->t('link to source'),
 					'$posted'       => $attributes['posted'],
-						'$content' => trim($content)
+					'$guid'         => $attributes['guid'],
+					'$network_name' => ContactSelector::networkToName($network, $attributes['profile']),
+					'$network_icon' => ContactSelector::networkToIcon($network, $attributes['profile']),
+					'$content'      => self::setMentions(trim($content), 0, $network),
 				]);
-				}
 				break;
 		}
 
@@ -1246,10 +1247,17 @@ class BBCode
 	 * @return string
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 */
-	public static function convert($text, $try_oembed = true, $simple_html = 0, $for_plaintext = false)
+	public static function convert(string $text = null, $try_oembed = true, $simple_html = self::INTERNAL, $for_plaintext = false)
 	{
+		// Accounting for null default column values
+		if (is_null($text) || $text === '') {
+			return '';
+		}
+
 		$a = DI::app();
 
+		$text = self::performWithEscapedTags($text, ['code'], function ($text) use ($try_oembed, $simple_html, $for_plaintext, $a) {
+			$text = self::performWithEscapedTags($text, ['noparse', 'nobb', 'pre'], function ($text) use ($try_oembed, $simple_html, $for_plaintext, $a) {
 				/*
 				 * preg_match_callback function to replace potential Oembed tags with Oembed content
 				 *
@@ -1271,36 +1279,14 @@ class BBCode
 					return $return;
 				};
 
-		// Extracting code blocks before the whitespace processing and the autolinker
-		$codeblocks = [];
 
-		$text = preg_replace_callback("#\[code(?:=([^\]]*))?\](.*?)\[\/code\]#ism",
-			function ($matches) use (&$codeblocks) {
-				$return = '#codeblock-' . count($codeblocks) . '#';
-				if (strpos($matches[2], "\n") !== false) {
-					$codeblocks[] = '<pre><code class="language-' . trim($matches[1]) . '">' . htmlspecialchars(trim($matches[2], "\n\r"), ENT_NOQUOTES, 'UTF-8') . '</code></pre>';
-				} else {
-					$codeblocks[] = '<code>' . htmlspecialchars($matches[2], ENT_NOQUOTES, 'UTF-8') . '</code>';
-				}
-
-				return $return;
-			},
-			$text
-		);
-
-		// Hide all [noparse] contained bbtags by spacefying them
-		// POSSIBLE BUG --> Will the 'preg' functions crash if there's an embedded image?
-
-		$text = preg_replace_callback("/\[noparse\](.*?)\[\/noparse\]/ism", 'self::escapeNoparseCallback', $text);
-		$text = preg_replace_callback("/\[nobb\](.*?)\[\/nobb\]/ism", 'self::escapeNoparseCallback', $text);
-		$text = preg_replace_callback("/\[pre\](.*?)\[\/pre\]/ism", 'self::escapeNoparseCallback', $text);
 
 				// Remove the abstract element. It is a non visible element.
 				$text = self::stripAbstract($text);
 
-		// Move all spaces out of the tags
+				// Move new lines outside of tags
-		$text = preg_replace("/\[(\w*)\](\s*)/ism", '$2[$1]', $text);
+				$text = preg_replace("#\[(\w*)](\n*)#ism", '$2[$1]', $text);
-		$text = preg_replace("/(\s*)\[\/(\w*)\]/ism", '[/$2]$1', $text);
+				$text = preg_replace("#(\n*)\[/(\w*)]#ism", '[/$2]$1', $text);
 
 				// Extract the private images which use data urls since preg has issues with
 				// large data sizes. Stash them away while we do bbcode conversion, and then put them back
@@ -1374,9 +1360,9 @@ class BBCode
 
 				/// @todo Have a closer look at the different html modes
 				// Handle attached links or videos
-		if (in_array($simple_html, [9])) {
+				if ($simple_html == self::ACTIVITYPUB) {
 					$text = self::removeAttachment($text);
-		} elseif (!in_array($simple_html, [0, 4])) {
+				} elseif (!in_array($simple_html, [self::INTERNAL, self::CONNECTORS])) {
 					$text = self::removeAttachment($text, true);
 				} else {
 					$text = self::convertAttachment($text, $simple_html, $try_oembed);
@@ -1439,7 +1425,7 @@ class BBCode
 
 				// Check for sized text
 				// [size=50] --> font-size: 50px (with the unit).
-		if ($simple_html != 3) {
+				if ($simple_html != self::DIASPORA) {
 					$text = preg_replace("(\[size=(\d*?)\](.*?)\[\/size\])ism", "<span style=\"font-size: $1px; line-height: initial;\">$2</span>", $text);
 					$text = preg_replace("(\[size=(.*?)\](.*?)\[\/size\])ism", "<span style=\"font-size: $1; line-height: initial;\">$2</span>", $text);
 				} else {
@@ -1712,8 +1698,16 @@ class BBCode
 					$text = Smilies::replace($text);
 				}
 
+				if (!$for_plaintext && DI::config()->get('system', 'big_emojis') && ($simple_html != self::DIASPORA)) {
+					$conv = html_entity_decode(str_replace([' ', "\n", "\r"], '', $text));
+					// Emojis are always 4 byte Unicode characters
+					if (!empty($conv) && (strlen($conv) / mb_strlen($conv) == 4)) {
+						$text = '<span style="font-size: xx-large; line-height: initial;">' . $text . '</span>';
+					}
+				}
+
 				if (!$for_plaintext) {
-			if (in_array($simple_html, [7, 9])) {
+					if (in_array($simple_html, [self::OSTATUS, self::ACTIVITYPUB])) {
 						$text = preg_replace_callback("/\[url\](.*?)\[\/url\]/ism", 'self::convertUrlForActivityPubCallback', $text);
 						$text = preg_replace_callback("/\[url\=(.*?)\](.*?)\[\/url\]/ism", 'self::convertUrlForActivityPubCallback', $text);
 					}
@@ -1725,14 +1719,14 @@ class BBCode
 				$text = str_replace(["\r","\n"], ['<br />', '<br />'], $text);
 
 				// Remove all hashtag addresses
-		if ($simple_html && !in_array($simple_html, [3, 7, 9])) {
+				if ($simple_html && !in_array($simple_html, [self::DIASPORA, self::OSTATUS, self::ACTIVITYPUB])) {
 					$text = preg_replace("/([#@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '$1$3', $text);
-		} elseif ($simple_html == 3) {
+				} elseif ($simple_html == self::DIASPORA) {
 					// The ! is converted to @ since Diaspora only understands the @
 					$text = preg_replace("/([@!])\[url\=(.*?)\](.*?)\[\/url\]/ism",
 						'@<a href="$2">$3</a>',
 						$text);
-		} elseif (in_array($simple_html, [7, 9])) {
+				} elseif (in_array($simple_html, [self::OSTATUS, self::ACTIVITYPUB])) {
 					$text = preg_replace("/([@!])\[url\=(.*?)\](.*?)\[\/url\]/ism",
 						'$1<span class="vcard"><a href="$2" class="url u-url mention" title="$3"><span class="fn nickname mention">$3</span></a></span>',
 						$text);
@@ -1748,26 +1742,18 @@ class BBCode
 				$text = preg_replace("/#\[url\=.*?\]\^\[\/url\]\[url\=(.*?)\](.*?)\[\/url\]/i",
 							"[bookmark=$1]$2[/bookmark]", $text);
 
-		if (in_array($simple_html, [2, 6, 7, 8])) {
+				if (in_array($simple_html, [self::API, self::OSTATUS, self::TWITTER])) {
 					$text = preg_replace_callback("/([^#@!])\[url\=([^\]]*)\](.*?)\[\/url\]/ism", "self::expandLinksCallback", $text);
 					//$Text = preg_replace("/[^#@!]\[url\=([^\]]*)\](.*?)\[\/url\]/ism", ' $2 [url]$1[/url]', $Text);
 					$text = preg_replace("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism", ' $2 [url]$1[/url]',$text);
 				}
 
-		if ($simple_html == 5) {
-			$text = preg_replace("/[^#@!]\[url\=(.*?)\](.*?)\[\/url\]/ism", '[url]$1[/url]', $text);
-		}
-
 				// Perform URL Search
 				if ($try_oembed) {
 					$text = preg_replace_callback("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism", $try_oembed_callback, $text);
 				}
 
-		if ($simple_html == 5) {
-			$text = preg_replace("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism", '[url]$1[/url]', $text);
-		} else {
 				$text = preg_replace("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism", '[url=$1]$2[/url]', $text);
-		}
 
 				// Handle Diaspora posts
 				$text = preg_replace_callback(
@@ -1794,12 +1780,16 @@ class BBCode
 				 * - #[url=<anything>]<term>[/url]
 				 * - [url=<anything>]#<term>[/url]
 				 */
-		$text = preg_replace_callback("/(?:#\[url\=[^\[\]]*\]|\[url\=[^\[\]]*\]#)(.*?)\[\/url\]/ism", function($matches) {
+				$text = preg_replace_callback("/(?:#\[url\=[^\[\]]*\]|\[url\=[^\[\]]*\]#)(.*?)\[\/url\]/ism", function($matches) use ($simple_html) {
-			return '#<a href="'
+					if ($simple_html == BBCode::ACTIVITYPUB) {
-				. DI::baseUrl()	. '/search?tag=' . rawurlencode($matches[1])
+						return '<a href="' . DI::baseUrl() . '/search?tag=' . rawurlencode($matches[1])
+							. '" data-tag="' . XML::escape($matches[1]) . '" rel="tag ugc">#'
+							. XML::escape($matches[1]) . '</a>';
+					} else {
+						return '#<a href="' . DI::baseUrl() . '/search?tag=' . rawurlencode($matches[1])
 							. '" class="tag" rel="tag" title="' . XML::escape($matches[1]) . '">'
-				. XML::escape($matches[1])
+							. XML::escape($matches[1]) . '</a>';
-				. '</a>';
+					}
 				}, $text);
 
 				// We need no target="_blank" rel="noopener noreferrer" for local links
@@ -1824,13 +1814,6 @@ class BBCode
 				$text = preg_replace("/\[mail\](.*?)\[\/mail\]/", '<a href="mailto:$1">$1</a>', $text);
 				$text = preg_replace("/\[mail\=(.*?)\](.*?)\[\/mail\]/", '<a href="mailto:$1">$2</a>', $text);
 
-		// Unhide all [noparse] contained bbtags unspacefying them
-		// and triming the [noparse] tag.
-
-		$text = preg_replace_callback("/\[noparse\](.*?)\[\/noparse\]/ism", 'self::unescapeNoparseCallback', $text);
-		$text = preg_replace_callback("/\[nobb\](.*?)\[\/nobb\]/ism", 'self::unescapeNoparseCallback', $text);
-		$text = preg_replace_callback("/\[pre\](.*?)\[\/pre\]/ism", 'self::unescapeNoparseCallback', $text);
-
 				/// @todo What is the meaning of these lines?
 				$text = preg_replace('/\[\&amp\;([#a-z0-9]+)\;\]/', '&$1;', $text);
 				$text = preg_replace('/\&\#039\;/', '\'', $text);
@@ -1849,7 +1832,7 @@ class BBCode
 				$text = preg_replace('#<([^>]*?)(src)="(?!' . implode('|', $allowed_src_protocols) . ')(.*?)"(.*?)>#ism',
 							 '<$1$2=""$4 data-original-src="$3" class="invalid-src" title="' . DI::l10n()->t('Invalid source protocol') . '">', $text);
 
-		// sanitize href attributes (only whitelisted protocols URLs)
+				// sanitize href attributes (only allowlisted protocols URLs)
 				// default value for backward compatibility
 				$allowed_link_protocols = DI::config()->get('system', 'allowed_link_protocols', []);
 
@@ -1872,17 +1855,31 @@ class BBCode
 					}
 				);
 
-		if ($saved_image) {
 				$text = self::interpolateSavedImagesIntoItemBody($text, $saved_image);
+
+				return $text;
+			}); // Escaped noparse, nobb, pre
+
+			// Remove escaping tags
+			$text = preg_replace("/\[noparse\](.*?)\[\/noparse\]/ism", '\1', $text);
+			$text = preg_replace("/\[nobb\](.*?)\[\/nobb\]/ism", '\1', $text);
+
+			// Additionally, [pre] tags preserve spaces
+			$text = preg_replace_callback("/\[pre\](.*?)\[\/pre\]/ism", function ($match) {
+				return str_replace(' ', '&nbsp;', $match[1]);
+			}, $text);
+
+			return $text;
+		}); // Escaped code
+
+		$text = preg_replace_callback("#\[code(?:=([^\]]*))?\](.*?)\[\/code\]#ism",
+			function ($matches) {
+				if (strpos($matches[2], "\n") !== false) {
+					$return = '<pre><code class="language-' . trim($matches[1]) . '">' . htmlspecialchars(trim($matches[2], "\n\r"), ENT_NOQUOTES, 'UTF-8') . '</code></pre>';
+				} else {
+					$return = '<code>' . htmlspecialchars($matches[2], ENT_NOQUOTES, 'UTF-8') . '</code>';
 				}
 
-		// Restore code blocks
-		$text = preg_replace_callback('/#codeblock-([0-9]+)#/iU',
-			function ($matches) use ($codeblocks) {
-				$return = $matches[0];
-				if (isset($codeblocks[intval($matches[1])])) {
-					$return = $codeblocks[$matches[1]];
-				}
 				return $return;
 			},
 			$text
@@ -2028,7 +2025,7 @@ class BBCode
 
 		// Convert it to HTML - don't try oembed
 		if ($for_diaspora) {
-			$text = self::convert($text, false, 3);
+			$text = self::convert($text, false, self::DIASPORA);
 
 			// Add all tags that maybe were removed
 			if (preg_match_all("/#\[url\=([$url_search_string]*)\](.*?)\[\/url\]/ism", $original_text, $tags)) {
@@ -2042,7 +2039,7 @@ class BBCode
 				$text = $text . " " . $tagline;
 			}
 		} else {
-			$text = self::convert($text, false, 4);
+			$text = self::convert($text, false, self::CONNECTORS);
 		}
 
 		// If a link is followed by a quote then there should be a newline before it
@@ -2094,12 +2091,10 @@ class BBCode
 	{
 		$ret = [];
 
+		BBCode::performWithEscapedTags($string, ['noparse', 'pre', 'code'], function ($string) use (&$ret) {
 			// Convert hashtag links to hashtags
 			$string = preg_replace('/#\[url\=([^\[\]]*)\](.*?)\[\/url\]/ism', '#$2 ', $string);
 
-		// ignore anything in a code block
-		$string = preg_replace('/\[code.*?\].*?\[\/code\]/sm', '', $string);
-
 			// Force line feeds at bbtags
 			$string = str_replace(['[', ']'], ["\n[", "]\n"], $string);
 
@@ -2127,17 +2122,13 @@ class BBCode
 			// Otherwise pull out single word tags. These can be @nickname, @first_last
 			// and #hash tags.
 
-		if (preg_match_all('/([!#@][^\^ \x0D\x0A,;:?]+)([ \x0D\x0A,;:?]|$)/', $string, $matches)) {
+			if (preg_match_all('/([!#@][^\^ \x0D\x0A,;:?\']*[^\^ \x0D\x0A,;:?!\'.])/', $string, $matches)) {
 				foreach ($matches[1] as $match) {
 					if (strstr($match, ']')) {
 						// we might be inside a bbcode color tag - leave it alone
 						continue;
 					}
 
-				if (substr($match, -1, 1) === '.') {
-					$match = substr($match,0,-1);
-				}
-
 					// ignore strictly numeric tags like #1
 					if ((strpos($match, '#') === 0) && ctype_digit(substr($match, 1))) {
 						continue;
@@ -2147,10 +2138,105 @@ class BBCode
 					if (strpos($string, $match) && preg_match('/[a-zA-z0-9\/]/', substr($string, strpos($string, $match) - 1, 1))) {
 						continue;
 					}
+
 					$ret[] = $match;
 				}
 			}
+		});
 
-		return $ret;
+		return array_unique($ret);
+	}
+
+	/**
+	 * Perform a custom function on a text after having escaped blocks enclosed in the provided tag list.
+	 *
+	 * @param string   $text
+	 * @param array    $tagList A list of tag names, e.g ['noparse', 'nobb', 'pre']
+	 * @param callable $callback
+	 * @return string
+	 * @throws Exception
+	 *@see Strings::performWithEscapedBlocks
+	 *
+	 */
+	public static function performWithEscapedTags(string $text, array $tagList, callable $callback)
+	{
+		$tagList = array_map('preg_quote', $tagList);
+
+		return Strings::performWithEscapedBlocks($text, '#\[(?:' . implode('|', $tagList) . ').*?\[/(?:' . implode('|', $tagList) . ')]#ism', $callback);
+	}
+
+	/**
+	 * Replaces mentions in the provided message body for the provided user and network if any
+	 *
+	 * @param $body
+	 * @param $profile_uid
+	 * @param $network
+	 * @return string
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	public static function setMentions($body, $profile_uid = 0, $network = '')
+	{
+		BBCode::performWithEscapedTags($body, ['noparse', 'pre', 'code', 'img'], function ($body) use ($profile_uid, $network) {
+			$tags = BBCode::getTags($body);
+
+			$tagged = [];
+			$inform = '';
+
+			foreach ($tags as $tag) {
+				$tag_type = substr($tag, 0, 1);
+
+				if ($tag_type == Tag::TAG_CHARACTER[Tag::HASHTAG]) {
+					continue;
+				}
+
+				/*
+				 * If we already tagged 'Robert Johnson', don't try and tag 'Robert'.
+				 * Robert Johnson should be first in the $tags array
+				 */
+				foreach ($tagged as $nextTag) {
+					if (stristr($nextTag, $tag . ' ')) {
+						continue 2;
+					}
+				}
+
+				$success = Item::replaceTag($body, $inform, $profile_uid, $tag, $network);
+
+				if ($success['replaced']) {
+					$tagged[] = $tag;
+				}
+			}
+
+			return $body;
+		});
+
+		return $body;
+	}
+
+	/**
+	 * @param string      $author  Author display name
+	 * @param string      $profile Author profile URL
+	 * @param string      $avatar  Author profile picture URL
+	 * @param string      $link    Post source URL
+	 * @param string      $posted  Post created date
+	 * @param string|null $guid    Post guid (if any)
+	 * @return string
+	 * @TODO Rewrite to handle over whole record array
+	 */
+	public static function getShareOpeningTag(string $author, string $profile, string $avatar, string $link, string $posted, string $guid = null)
+	{
+		$header = "[share author='" . str_replace(["'", "[", "]"], ["&#x27;", "&#x5B;", "&#x5D;"], $author) .
+			"' profile='" . str_replace(["'", "[", "]"], ["&#x27;", "&#x5B;", "&#x5D;"], $profile) .
+			"' avatar='" . str_replace(["'", "[", "]"], ["&#x27;", "&#x5B;", "&#x5D;"], $avatar) .
+			"' link='" . str_replace(["'", "[", "]"], ["&#x27;", "&#x5B;", "&#x5D;"], $link) .
+			"' posted='" . str_replace(["'", "[", "]"], ["&#x27;", "&#x5B;", "&#x5D;"], $posted);
+
+		if ($guid) {
+			$header .= "' guid='" . str_replace(["'", "[", "]"], ["&#x27;", "&#x5B;", "&#x5D;"], $guid);
+		}
+
+		$header  .= "']";
+
+		return $header;
 	}
 }

src/Content/Text/HTML.php

@@ -26,6 +26,7 @@ use DOMXPath;
 use Friendica\Content\Widget\ContactBlock;
 use Friendica\Core\Hook;
 use Friendica\Core\Renderer;
+use Friendica\Core\Search;
 use Friendica\DI;
 use Friendica\Model\Contact;
 use Friendica\Util\Network;
@@ -166,24 +167,7 @@ class HTML
 	{
 		$message = str_replace("\r", "", $message);
 
-		// Removing code blocks before the whitespace removal processing below
+		$message = Strings::performWithEscapedBlocks($message, '#<pre><code.*</code></pre>#iUs', function ($message) {
-		$codeblocks = [];
-		$message = preg_replace_callback(
-			'#<pre><code(?: class="language-([^"]*)")?>(.*)</code></pre>#iUs',
-			function ($matches) use (&$codeblocks) {
-				$return = '[codeblock-' . count($codeblocks) . ']';
-
-				$prefix = '[code]';
-				if ($matches[1] != '') {
-					$prefix = '[code=' . $matches[1] . ']';
-				}
-
-				$codeblocks[] = $prefix . PHP_EOL . trim($matches[2]) . PHP_EOL . '[/code]';
-				return $return;
-			},
-			$message
-		);
-
 			$message = str_replace(
 				[
 					"<li><p>",
@@ -403,15 +387,18 @@ class HTML
 			// Handling Yahoo style of mails
 			$message = str_replace('[hr][b]From:[/b]', '[quote][b]From:[/b]', $message);
 
-		// Restore code blocks
+			return $message;
+		});
+
 		$message = preg_replace_callback(
-			'#\[codeblock-([0-9]+)\]#iU',
+			'#<pre><code(?: class="language-([^"]*)")?>(.*)</code></pre>#iUs',
-			function ($matches) use ($codeblocks) {
+			function ($matches) {
-				$return = '';
+				$prefix = '[code]';
-				if (isset($codeblocks[intval($matches[1])])) {
+				if ($matches[1] != '') {
-					$return = $codeblocks[$matches[1]];
+					$prefix = '[code=' . $matches[1] . ']';
 				}
-				return $return;
+
+				return $prefix . PHP_EOL . trim($matches[2]) . PHP_EOL . '[/code]';
 			},
 			$message
 		);
@@ -917,7 +904,7 @@ class HTML
 			'$save_label'   => $save_label,
 			'$search_hint'  => DI::l10n()->t('@name, !forum, #tags, content'),
 			'$mode'         => $mode,
-			'$return_url'   => urlencode('search?q=' . urlencode($s)),
+			'$return_url'   => urlencode(Search::getSearchPath($s)),
 		];
 
 		if (!$aside) {

src/Content/Text/Markdown.php

@@ -35,20 +35,20 @@ class Markdown
 	 * compatibility with Diaspora in spite of the Markdown standard.
 	 *
 	 * @param string $text
-	 * @param bool   $hardwrap
+	 * @param bool   $hardwrap Enables line breaks on \n without two trailing spaces
+	 * @param string $baseuri  Optional. Prepend anchor links with this URL
 	 * @return string
-	 * @throws \Exception
 	 */
-	public static function convert($text, $hardwrap = true) {
+	public static function convert($text, $hardwrap = true, $baseuri = null) {
 		$stamp1 = microtime(true);
 
 		$MarkdownParser = new MarkdownParser();
 		$MarkdownParser->code_class_prefix  = 'language-';
 		$MarkdownParser->hard_wrap          = $hardwrap;
 		$MarkdownParser->hashtag_protection = true;
-		$MarkdownParser->url_filter_func    = function ($url) {
+		$MarkdownParser->url_filter_func    = function ($url) use ($baseuri) {
-			if (strpos($url, '#') === 0) {
+			if (!empty($baseuri) && strpos($url, '#') === 0) {
-				$url = ltrim($_SERVER['REQUEST_URI'], '/') . $url;
+				$url = ltrim($baseuri, '/') . $url;
 			}
 			return  $url;
 		};
@@ -122,9 +122,6 @@ class Markdown
 		// protect the recycle symbol from turning into a tag, but without unescaping angles and naked ampersands
 		$s = str_replace('♲', html_entity_decode('♲', ENT_QUOTES, 'UTF-8'), $s);
 
-		// Convert everything that looks like a link to a link
-		$s = preg_replace('/([^\]=]|^)(https?\:\/\/)([a-zA-Z0-9:\/\-?&;.=_~#%$!+,@]+(?<!,))/ism', '$1[url=$2$3]$2$3[/url]', $s);
-
 		//$s = preg_replace("/([^\]\=]|^)(https?\:\/\/)(vimeo|youtu|www\.youtube|soundcloud)([a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", '$1[url=$2$3$4]$2$3$4[/url]',$s);
 		$s = BBCode::pregReplaceInTag('/\[url\=?(.*?)\]https?:\/\/www.youtube.com\/watch\?v\=(.*?)\[\/url\]/ism', '[youtube]$2[/youtube]', 'url', $s);
 		$s = BBCode::pregReplaceInTag('/\[url\=https?:\/\/www.youtube.com\/watch\?v\=(.*?)\].*?\[\/url\]/ism'   , '[youtube]$1[/youtube]', 'url', $s);

src/Content/Widget/SavedSearches.php

@@ -22,6 +22,7 @@
 namespace Friendica\Content\Widget;
 
 use Friendica\Core\Renderer;
+use Friendica\Core\Search;
 use Friendica\Database\DBA;
 use Friendica\DI;
 
@@ -35,24 +36,27 @@ class SavedSearches
 	 */
 	public static function getHTML($return_url, $search = '')
 	{
-		$o = '';
-
-		$saved_searches = DBA::select('search', ['id', 'term'], ['uid' => local_user()]);
-		if (DBA::isResult($saved_searches)) {
 		$saved = [];
-			foreach ($saved_searches as $saved_search) {
+		$saved_searches = DBA::select('search', ['id', 'term'], ['uid' => local_user()]);
+		while ($saved_search = DBA::fetch($saved_searches)) {
 			$saved[] = [
 				'id'          => $saved_search['id'],
 				'term'        => $saved_search['term'],
 				'encodedterm' => urlencode($saved_search['term']),
+				'searchpath'  => Search::getSearchPath($saved_search['term']),
 				'delete'      => DI::l10n()->t('Remove term'),
 				'selected'    => $search == $saved_search['term'],
 			];
 		}
+		DBA::close($saved_searches);
+
+		if (empty($saved)) {
+			return '';
+		}
 
 		$tpl = Renderer::getMarkupTemplate('widget/saved_searches.tpl');
 
-			$o = Renderer::replaceMacros($tpl, [
+		return Renderer::replaceMacros($tpl, [
 			'$title'      => DI::l10n()->t('Saved Searches'),
 			'$add'        => '',
 			'$searchbox'  => '',
@@ -60,7 +64,4 @@ class SavedSearches
 			'$return_url' => urlencode($return_url),
 		]);
 	}
-
-		return $o;
-	}
 }

src/Content/Widget/TagCloud.php

@@ -25,6 +25,7 @@ use Friendica\Core\Renderer;
 use Friendica\Database\DBA;
 use Friendica\DI;
 use Friendica\Model\Item;
+use Friendica\Model\Tag;
 
 /**
  * TagCloud widget
@@ -45,7 +46,7 @@ class TagCloud
 	 * @return string       HTML formatted output.
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 */
-	public static function getHTML($uid, $count = 0, $owner_id = 0, $flags = '', $type = TERM_HASHTAG)
+	public static function getHTML($uid, $count = 0, $owner_id = 0, $flags = '', $type = Tag::HASHTAG)
 	{
 		$o = '';
 		$r = self::tagadelic($uid, $count, $owner_id, $flags, $type);
@@ -84,7 +85,7 @@ class TagCloud
 	 * @return array        Alphabetical sorted array of used tags of an user.
 	 * @throws \Exception
 	 */
-	private static function tagadelic($uid, $count = 0, $owner_id = 0, $flags = '', $type = TERM_HASHTAG)
+	private static function tagadelic($uid, $count = 0, $owner_id = 0, $flags = '', $type = Tag::HASHTAG)
 	{
 		$sql_options = Item::getPermissionsSQLByUserId($uid);
 		$limit = $count ? sprintf('LIMIT %d', intval($count)) : '';
@@ -100,16 +101,13 @@ class TagCloud
 		}
 
 		// Fetch tags
-		$tag_stmt = DBA::p("SELECT `term`, COUNT(`term`) AS `total` FROM `term`
+		$tag_stmt = DBA::p("SELECT `name`, COUNT(`name`) AS `total` FROM `tag-search-view`
-			LEFT JOIN `item` ON `term`.`oid` = `item`.`id`
+			LEFT JOIN `item` ON `tag-search-view`.`uri-id` = `item`.`uri-id`
-			WHERE `term`.`uid` = ? AND `term`.`type` = ?
+			WHERE `tag-search-view`.`uid` = ?
-			AND `term`.`otype` = ?
 			AND `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
 			$sql_options
-			GROUP BY `term` ORDER BY `total` DESC $limit",
+			GROUP BY `name` ORDER BY `total` DESC $limit",
-			$uid,
+			$uid
-			$type,
-			TERM_OBJ_POST
 		);
 		if (!DBA::isResult($tag_stmt)) {
 			return [];
@@ -138,7 +136,7 @@ class TagCloud
 		}
 
 		foreach ($arr as $rr) {
-			$tags[$x][0] = $rr['term'];
+			$tags[$x][0] = $rr['name'];
 			$tags[$x][1] = log($rr['total']);
 			$tags[$x][2] = 0;
 			$min = min($min, $tags[$x][1]);

src/Content/Widget/TrendingTags.php

@@ -23,7 +23,7 @@ namespace Friendica\Content\Widget;
 
 use Friendica\Core\Renderer;
 use Friendica\DI;
-use Friendica\Model\Term;
+use Friendica\Model\Tag;
 
 /**
  * Trending tags aside widget for the community pages, handles both local and global scopes
@@ -41,9 +41,9 @@ class TrendingTags
 	public static function getHTML($content = 'global', int $period = 24)
 	{
 		if ($content == 'local') {
-			$tags = Term::getLocalTrendingHashtags($period, 20);
+			$tags = Tag::getLocalTrendingHashtags($period, 20);
 		} else {
-			$tags = Term::getGlobalTrendingHashtags($period, 20);
+			$tags = Tag::getGlobalTrendingHashtags($period, 20);
 		}
 
 		$tpl = Renderer::getMarkupTemplate('widget/trending_tags.tpl');

src/Core/Console.php

@@ -51,7 +51,7 @@ Commands:
 	docbloxerrorchecker    Check the file tree for DocBlox errors
 	extract                Generate translation string file for the Friendica project (deprecated)
 	globalcommunityblock   Block remote profile from interacting with this node
-	globalcommunitysilence Silence remote profile from global community page
+	globalcommunitysilence Silence a profile from the global community page
 	archivecontact         Archive a contact when you know that it isn't existing anymore
 	help                   Show help about a command, e.g (bin/console help config)
 	autoinstall            Starts automatic installation of friendica based on values from htconfig.php

src/Core/Installer.php

@@ -259,7 +259,7 @@ class Installer
 		$help = "";
 		if (!$passed) {
 			$help .= DI::l10n()->t('Could not find a command line version of PHP in the web server PATH.') . EOL;
-			$help .= DI::l10n()->t("If you don't have a command line version of PHP installed on your server, you will not be able to run the background processing. See <a href='https://github.com/friendica/friendica/blob/master/doc/Install.md#set-up-the-worker'>'Setup the worker'</a>") . EOL;
+			$help .= DI::l10n()->t("If you don't have a command line version of PHP installed on your server, you will not be able to run the background processing. See <a href='https://github.com/friendica/friendica/blob/stable/doc/Install.md#set-up-the-worker'>'Setup the worker'</a>") . EOL;
 			$help .= EOL . EOL;
 			$tpl = Renderer::getMarkupTemplate('field_input.tpl');
 			/// @todo Separate backend Installer class and presentation layer/view

src/Core/Renderer.php

@@ -23,8 +23,8 @@ namespace Friendica\Core;
 
 use Exception;
 use Friendica\DI;
-use Friendica\Render\FriendicaSmarty;
+use Friendica\Network\HTTPException\InternalServerErrorException;
-use Friendica\Render\ITemplateEngine;
+use Friendica\Render\TemplateEngine;
 
 /**
  * This class handles Renderer related functions.
@@ -66,28 +66,30 @@ class Renderer
 	];
 
 	/**
-	 * This is our template processor
+	 * Returns the rendered template output from the template string and variables
 	 *
-	 * @param string|FriendicaSmarty $s    The string requiring macro substitution or an instance of FriendicaSmarty
+	 * @param string $template
-	 * @param array                  $vars Key value pairs (search => replace)
+	 * @param array  $vars
-	 *
+	 * @return string
-	 * @return string substituted string
+	 * @throws InternalServerErrorException
-	 * @throws Exception
 	 */
-	public static function replaceMacros($s, array $vars = [])
+	public static function replaceMacros(string $template, array $vars = [])
 	{
 		$stamp1 = microtime(true);
 
 		// pass $baseurl to all templates if it isn't set
-		$vars = array_merge(['$baseurl' => DI::baseUrl()->get()], $vars);
+		$vars = array_merge(['$baseurl' => DI::baseUrl()->get(), '$APP' => DI::app()], $vars);
 
 		$t = self::getTemplateEngine();
 
 		try {
-			$output = $t->replaceMacros($s, $vars);
+			$output = $t->replaceMacros($template, $vars);
 		} catch (Exception $e) {
-			echo "<pre><b>" . __FUNCTION__ . "</b>: " . $e->getMessage() . "</pre>";
+			DI::logger()->critical($e->getMessage(), ['template' => $template, 'vars' => $vars]);
-			exit();
+			$message = is_site_admin() ?
+				$e->getMessage() :
+				DI::l10n()->t('Friendica can\'t display this page at the moment, please contact the administrator.');
+			throw new InternalServerErrorException($message);
 		}
 
 		DI::profiler()->saveTimestamp($stamp1, "rendering", System::callstack());
@@ -98,23 +100,25 @@ class Renderer
 	/**
 	 * Load a given template $s
 	 *
-	 * @param string $s    Template to load.
+	 * @param string $file   Template to load.
-	 * @param string $root Optional.
+	 * @param string $subDir Subdirectory (Optional)
 	 *
 	 * @return string template.
-	 * @throws Exception
+	 * @throws InternalServerErrorException
 	 */
-	public static function getMarkupTemplate($s, $root = '')
+	public static function getMarkupTemplate($file, $subDir = '')
 	{
 		$stamp1 = microtime(true);
-		$a = DI::app();
 		$t = self::getTemplateEngine();
 
 		try {
-			$template = $t->getTemplateFile($s, $root);
+			$template = $t->getTemplateFile($file, $subDir);
 		} catch (Exception $e) {
-			echo "<pre><b>" . __FUNCTION__ . "</b>: " . $e->getMessage() . "</pre>";
+			DI::logger()->critical($e->getMessage(), ['file' => $file, 'subDir' => $subDir]);
-			exit();
+			$message = is_site_admin() ?
+				$e->getMessage() :
+				DI::l10n()->t('Friendica can\'t display this page at the moment, please contact the administrator.');
+			throw new InternalServerErrorException($message);
 		}
 
 		DI::profiler()->saveTimestamp($stamp1, "file", System::callstack());
@@ -126,18 +130,22 @@ class Renderer
 	 * Register template engine class
 	 *
 	 * @param string $class
+	 * @throws InternalServerErrorException
 	 */
 	public static function registerTemplateEngine($class)
 	{
 		$v = get_class_vars($class);
 
-		if (!empty($v['name']))
+		if (!empty($v['name'])) {
-		{
 			$name = $v['name'];
 			self::$template_engines[$name] = $class;
 		} else {
-			echo "template engine <tt>$class</tt> cannot be registered without a name.\n";
+			$admin_message = DI::l10n()->t('template engine cannot be registered without a name.');
-			die();
+			DI::logger()->critical($admin_message, ['class' => $class]);
+			$message = is_site_admin() ?
+				$admin_message :
+				DI::l10n()->t('Friendica can\'t display this page at the moment, please contact the administrator.');
+			throw new InternalServerErrorException($message);
 		}
 	}
 
@@ -147,7 +155,8 @@ class Renderer
 	 * If $name is not defined, return engine defined by theme,
 	 * or default
 	 *
-	 * @return ITemplateEngine Template Engine instance
+	 * @return TemplateEngine Template Engine instance
+	 * @throws InternalServerErrorException
 	 */
 	public static function getTemplateEngine()
 	{
@@ -157,15 +166,20 @@ class Renderer
 			if (isset(self::$template_engine_instance[$template_engine])) {
 				return self::$template_engine_instance[$template_engine];
 			} else {
+				$a = DI::app();
 				$class = self::$template_engines[$template_engine];
-				$obj = new $class;
+				$obj = new $class($a->getCurrentTheme(), $a->theme_info);
 				self::$template_engine_instance[$template_engine] = $obj;
 				return $obj;
 			}
 		}
 
-		echo "template engine <tt>$template_engine</tt> is not registered!\n";
+		$admin_message = DI::l10n()->t('template engine is not registered!');
-		exit();
+		DI::logger()->critical($admin_message, ['template_engine' => $template_engine]);
+		$message = is_site_admin() ?
+			$admin_message :
+			DI::l10n()->t('Friendica can\'t display this page at the moment, please contact the administrator.');
+		throw new InternalServerErrorException($message);
 	}
 
 	/**

src/Core/Search.php

@@ -100,7 +100,7 @@ class Search
 	/**
 	 * Search in the global directory for occurrences of the search string
 	 *
-	 * @see https://github.com/friendica/friendica-directory/blob/master/docs/Protocol.md#search
+	 * @see https://github.com/friendica/friendica-directory/blob/stable/docs/Protocol.md#search
 	 *
 	 * @param string $search
 	 * @param int    $type specific type of searching
@@ -142,7 +142,7 @@ class Search
 		$profiles = $results['profiles'] ?? [];
 
 		foreach ($profiles as $profile) {
-			$profile_url = $profile['profile_url'] ?? '';
+			$profile_url = $profile['url'] ?? '';
 			$contactDetails = Contact::getDetailsByURL($profile_url, local_user());
 
 			$result = new ContactResult(
@@ -311,4 +311,19 @@ class Search
 	{
 		return DI::config()->get('system', 'directory', self::DEFAULT_DIRECTORY);
 	}
+
+	/**
+	 * Return the search path (either fulltext search or tag search)
+	 *
+	 * @param string $search
+	 * @return string search path
+	 */
+	public static function getSearchPath(string $search)
+	{
+		if (substr($search, 0, 1) == '#') {
+			return 'search?tag=' . urlencode(substr($search, 1));
+		} else {
+			return 'search?q=' . urlencode($search);
+		}
+	}
 }

src/Core/System.php

@@ -45,20 +45,22 @@ class System
 		array_shift($trace);
 
 		$callstack = [];
-		$previous = ['class' => '', 'function' => ''];
+		$previous = ['class' => '', 'function' => '', 'database' => false];
 
 		// The ignore list contains all functions that are only wrapper functions
 		$ignore = ['fetchUrl', 'call_user_func_array'];
 
 		while ($func = array_pop($trace)) {
 			if (!empty($func['class'])) {
-				// Don't show multiple calls from the "dba" class to show the essential parts of the callstack
+				// Don't show multiple calls from the Database classes to show the essential parts of the callstack
-				if ((($previous['class'] != $func['class']) || ($func['class'] != 'Friendica\Database\DBA')) && ($previous['function'] != 'q')) {
+				$func['database'] = in_array($func['class'], ['Friendica\Database\DBA', 'Friendica\Database\Database']);
+				if (!$previous['database'] || !$func['database']) {	
 					$classparts = explode("\\", $func['class']);
 					$callstack[] = array_pop($classparts).'::'.$func['function'];
 					$previous = $func;
 				}
 			} elseif (!in_array($func['function'], $ignore)) {
+				$func['database'] = ($func['function'] == 'q');
 				$callstack[] = $func['function'];
 				$func['class'] = '';
 				$previous = $func;

src/Core/Worker.php

@@ -66,7 +66,7 @@ class Worker
 
 		// At first check the maximum load. We shouldn't continue with a high load
 		if (DI::process()->isMaxLoadReached()) {
-			Logger::log('Pre check: maximum load reached, quitting.', Logger::DEBUG);
+			Logger::info('Pre check: maximum load reached, quitting.');
 			return;
 		}
 
@@ -82,25 +82,25 @@ class Worker
 
 		// Count active workers and compare them with a maximum value that depends on the load
 		if (self::tooMuchWorkers()) {
-			Logger::log('Pre check: Active worker limit reached, quitting.', Logger::DEBUG);
+			Logger::info('Pre check: Active worker limit reached, quitting.');
 			return;
 		}
 
 		// Do we have too few memory?
 		if (DI::process()->isMinMemoryReached()) {
-			Logger::log('Pre check: Memory limit reached, quitting.', Logger::DEBUG);
+			Logger::info('Pre check: Memory limit reached, quitting.');
 			return;
 		}
 
 		// Possibly there are too much database connections
 		if (self::maxConnectionsReached()) {
-			Logger::log('Pre check: maximum connections reached, quitting.', Logger::DEBUG);
+			Logger::info('Pre check: maximum connections reached, quitting.');
 			return;
 		}
 
 		// Possibly there are too much database processes that block the system
 		if (DI::process()->isMaxProcessesReached()) {
-			Logger::log('Pre check: maximum processes reached, quitting.', Logger::DEBUG);
+			Logger::info('Pre check: maximum processes reached, quitting.');
 			return;
 		}
 
@@ -121,7 +121,7 @@ class Worker
 
 				// The work will be done
 				if (!self::execute($entry)) {
-					Logger::log('Process execution failed, quitting.', Logger::DEBUG);
+					Logger::info('Process execution failed, quitting.');
 					return;
 				}
 
@@ -143,14 +143,14 @@ class Worker
 				if (DI::lock()->acquire('worker', 0)) {
 				// Count active workers and compare them with a maximum value that depends on the load
 					if (self::tooMuchWorkers()) {
-						Logger::log('Active worker limit reached, quitting.', Logger::DEBUG);
+						Logger::info('Active worker limit reached, quitting.');
 						DI::lock()->release('worker');
 						return;
 					}
 
 					// Check free memory
 					if (DI::process()->isMinMemoryReached()) {
-						Logger::log('Memory limit reached, quitting.', Logger::DEBUG);
+						Logger::info('Memory limit reached, quitting.');
 						DI::lock()->release('worker');
 						return;
 					}
@@ -170,7 +170,7 @@ class Worker
 		if (DI::config()->get('system', 'worker_daemon_mode', false)) {
 			self::IPCSetJobState(false);
 		}
-		Logger::log("Couldn't select a workerqueue entry, quitting process " . getmypid() . ".", Logger::DEBUG);
+		Logger::info("Couldn't select a workerqueue entry, quitting process", ['pid' => getmypid()]);
 	}
 
 	/**
@@ -264,23 +264,27 @@ class Worker
 
 		// Quit when in maintenance
 		if (DI::config()->get('system', 'maintenance', false, true)) {
-			Logger::log("Maintenance mode - quit process ".$mypid, Logger::DEBUG);
+			Logger::info("Maintenance mode - quit process", ['pid' => $mypid]);
 			return false;
 		}
 
 		// Constantly check the number of parallel database processes
 		if (DI::process()->isMaxProcessesReached()) {
-			Logger::log("Max processes reached for process ".$mypid, Logger::DEBUG);
+			Logger::info("Max processes reached for process", ['pid' => $mypid]);
 			return false;
 		}
 
 		// Constantly check the number of available database connections to let the frontend be accessible at any time
 		if (self::maxConnectionsReached()) {
-			Logger::log("Max connection reached for process ".$mypid, Logger::DEBUG);
+			Logger::info("Max connection reached for process", ['pid' => $mypid]);
 			return false;
 		}
 
 		$argv = json_decode($queue["parameter"], true);
+		if (empty($argv)) {
+			Logger::error('Parameter is empty', ['queue' => $queue]);
+			return false;
+		}
 
 		// Check for existance and validity of the include file
 		$include = $argv[0];
@@ -383,8 +387,6 @@ class Worker
 	{
 		$a = DI::app();
 
-		$argc = count($argv);
-
 		Logger::enableWorker($funcname);
 
 		Logger::info("Process start.", ['priority' => $queue["priority"], 'id' => $queue["id"]]);
@@ -406,7 +408,7 @@ class Worker
 		if ($method_call) {
 			call_user_func_array(sprintf('Friendica\Worker\%s::execute', $funcname), $argv);
 		} else {
-			$funcname($argv, $argc);
+			$funcname($argv, count($argv));
 		}
 
 		Logger::disableWorker();
@@ -504,7 +506,7 @@ class Worker
 			$used = DBA::numRows($r);
 			DBA::close($r);
 
-			Logger::log("Connection usage (user values): ".$used."/".$max, Logger::DEBUG);
+			Logger::info("Connection usage (user values)", ['usage' => $used, 'max' => $max]);
 
 			$level = ($used / $max) * 100;
 
@@ -532,7 +534,7 @@ class Worker
 		if ($used == 0) {
 			return false;
 		}
-		Logger::log("Connection usage (system values): ".$used."/".$max, Logger::DEBUG);
+		Logger::info("Connection usage (system values)", ['used' => $used, 'max' => $max]);
 
 		$level = $used / $max * 100;
 
@@ -582,6 +584,10 @@ class Worker
 				$max_duration = $max_duration_defaults[$entry["priority"]];
 
 				$argv = json_decode($entry["parameter"], true);
+				if (empty($argv)) {
+					return;
+				}
+
 				$argv[0] = basename($argv[0]);
 
 				// How long is the process already running?
@@ -610,10 +616,11 @@ class Worker
 					self::$db_duration += (microtime(true) - $stamp);
 					self::$db_duration_write += (microtime(true) - $stamp);
 				} else {
-					Logger::log("Worker process ".$entry["pid"]." (".substr(json_encode($argv), 0, 50).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", Logger::DEBUG);
+					Logger::info('Process runtime is okay', ['pid' => $entry["pid"], 'duration' => $duration, 'max' => $max_duration, 'command' => substr(json_encode($argv), 0, 50)]);
 				}
 			}
 		}
+		DBA::close($entries);
 	}
 
 	/**
@@ -684,7 +691,7 @@ class Worker
 				self::$db_duration_stat += (microtime(true) - $stamp);
 				while ($entry = DBA::fetch($jobs)) {
 					$stamp = (float)microtime(true);
-					$processes = DBA::p("SELECT COUNT(*) AS `running` FROM `process` INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` WHERE NOT `done` AND `priority` = ?", $entry["priority"]);
+					$processes = DBA::p("SELECT COUNT(*) AS `running` FROM `workerqueue-view` WHERE `priority` = ?", $entry["priority"]);
 					self::$db_duration += (microtime(true) - $stamp);
 					self::$db_duration_stat += (microtime(true) - $stamp);
 					if ($process = DBA::fetch($processes)) {
@@ -698,7 +705,7 @@ class Worker
 			} else {
 				$waiting_processes =  self::totalEntries();
 				$stamp = (float)microtime(true);
-				$jobs = DBA::p("SELECT COUNT(*) AS `running`, `priority` FROM `process` INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid` AND NOT `done` GROUP BY `priority` ORDER BY `priority`");
+				$jobs = DBA::p("SELECT COUNT(*) AS `running`, `priority` FROM `workerqueue-view` GROUP BY `priority` ORDER BY `priority`");
 				self::$db_duration += (microtime(true) - $stamp);
 				self::$db_duration_stat += (microtime(true) - $stamp);
 
@@ -720,7 +727,7 @@ class Worker
 				$high_running = self::processWithPriorityActive($top_priority);
 
 				if (!$high_running && ($top_priority > PRIORITY_UNDEFINED) && ($top_priority < PRIORITY_NEGLIGIBLE)) {
-					Logger::log("There are jobs with priority ".$top_priority." waiting but none is executed. Open a fastlane.", Logger::DEBUG);
+					Logger::info("Jobs with a higher priority are waiting but none is executed. Open a fastlane.", ['priority' => $top_priority]);
 					$queues = $active + 1;
 				}
 			}
@@ -729,7 +736,7 @@ class Worker
 
 			// Are there fewer workers running as possible? Then fork a new one.
 			if (!DI::config()->get("system", "worker_dont_fork", false) && ($queues > ($active + 1)) && self::entriesExists()) {
-				Logger::log("Active workers: ".$active."/".$queues." Fork a new worker.", Logger::DEBUG);
+				Logger::info("There are fewer workers as possible, fork a new worker.", ['active' => $active, 'queues' => $queues]);
 				if (DI::config()->get('system', 'worker_daemon_mode', false)) {
 					self::IPCSetJobState(true);
 				} else {
@@ -839,9 +846,7 @@ class Worker
 		$running = [];
 		$running_total = 0;
 		$stamp = (float)microtime(true);
-		$processes = DBA::p("SELECT COUNT(DISTINCT(`process`.`pid`)) AS `running`, `priority` FROM `process`
+		$processes = DBA::p("SELECT COUNT(DISTINCT(`pid`)) AS `running`, `priority` FROM `workerqueue-view` GROUP BY `priority`");
-			INNER JOIN `workerqueue` ON `workerqueue`.`pid` = `process`.`pid`
-			WHERE NOT `done` GROUP BY `priority`");
 		self::$db_duration += (microtime(true) - $stamp);
 		while ($process = DBA::fetch($processes)) {
 			$running[$process['priority']] = $process['running'];
@@ -933,7 +938,7 @@ class Worker
 	/**
 	 * Returns the next worker process
 	 *
-	 * @return string SQL statement
+	 * @return array worker processes
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 */
 	public static function workerProcess()
@@ -1028,7 +1033,7 @@ class Worker
 
 			self::runCron();
 
-			Logger::log('Call worker', Logger::DEBUG);
+			Logger::info('Call worker');
 			self::spawnWorker();
 			return;
 		}
@@ -1074,7 +1079,7 @@ class Worker
 	 */
 	private static function runCron()
 	{
-		Logger::log('Add cron entries', Logger::DEBUG);
+		Logger::info('Add cron entries');
 
 		// Check for spooled items
 		self::add(['priority' => PRIORITY_HIGH, 'force_priority' => true], 'SpoolPost');

src/DI.php

@@ -279,6 +279,14 @@ abstract class DI
 		return self::$dice->create(Factory\Api\Mastodon\Relationship::class);
 	}
 
+	/**
+	 * @return Factory\Api\Twitter\User
+	 */
+	public static function twitterUser()
+	{
+		return self::$dice->create(Factory\Api\Twitter\User::class);
+	}
+
 	/**
 	 * @return Factory\Notification\Notification
 	 */
@@ -383,6 +391,14 @@ abstract class DI
 		return self::$dice->create(Util\ACLFormatter::class);
 	}
 
+	/**
+	 * @return string
+	 */
+	public static function basePath()
+	{
+		return self::$dice->create('$basepath');
+	}
+
 	/**
 	 * @return Util\DateTimeFormat
 	 */

src/Database/DBA.php

@@ -648,6 +648,20 @@ class DBA
 	/**
 	 * Returns the SQL parameter string built from the provided parameter array
 	 *
+	 * Expected format for each key:
+	 *
+	 * group_by:
+	 *  - list of column names
+	 *
+	 * order:
+	 *  - numeric keyed column name => ASC
+	 *  - associative element with boolean value => DESC (true), ASC (false)
+	 *  - associative element with string value => 'ASC' or 'DESC' literally
+	 *
+	 * limit:
+	 *  - single numeric value => count
+	 *  - list with two numeric values => offset, count
+	 *
 	 * @param array $params
 	 * @return string
 	 */
@@ -665,7 +679,11 @@ class DBA
 				if ($order === 'RAND()') {
 					$order_string .= "RAND(), ";
 				} elseif (!is_int($fields)) {
-					$order_string .= self::quoteIdentifier($fields) . " " . ($order ? "DESC" : "ASC") . ", ";
+					if ($order !== 'DESC' && $order !== 'ASC') {
+						$order = $order ? 'DESC' : 'ASC';
+					}
+
+					$order_string .= self::quoteIdentifier($fields) . " " . $order . ", ";
 				} else {
 					$order_string .= self::quoteIdentifier($order) . ", ";
 				}
@@ -741,6 +759,17 @@ class DBA
 		return DI::dba()->processlist();
 	}
 
+	/**
+	 * Fetch a database variable
+	 *
+	 * @param string $name
+	 * @return string content
+	 */
+	public static function getVariable(string $name)
+	{
+		return DI::dba()->getVariable($name);
+	}
+
 	/**
 	 * Checks if $array is a filled array with at least one entry.
 	 *

src/Database/DBStructure.php

@@ -25,10 +25,10 @@ use Exception;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
 use Friendica\DI;
+use Friendica\Model\Item;
+use Friendica\Model\User;
 use Friendica\Util\DateTimeFormat;
 
-require_once __DIR__ . '/../../include/dba.php';
-
 /**
  * This class contains functions that doesn't need to know if pdo, mysqli or whatever is used.
  */
@@ -49,7 +49,7 @@ class DBStructure
 	private static $definition = [];
 
 	/**
-	 * Converts all tables from MyISAM to InnoDB
+	 * Converts all tables from MyISAM/InnoDB Antelope to InnoDB Barracuda
 	 */
 	public static function convertToInnoDB()
 	{
@@ -59,13 +59,19 @@ class DBStructure
 			['engine' => 'MyISAM', 'table_schema' => DBA::databaseName()]
 		);
 
+		$tables = array_merge($tables, DBA::selectToArray(
+			['information_schema' => 'tables'],
+			['table_name'],
+			['engine' => 'InnoDB', 'ROW_FORMAT' => ['COMPACT', 'REDUNDANT'], 'table_schema' => DBA::databaseName()]
+		));
+
 		if (!DBA::isResult($tables)) {
-			echo DI::l10n()->t('There are no tables on MyISAM.') . "\n";
+			echo DI::l10n()->t('There are no tables on MyISAM or InnoDB with the Antelope file format.') . "\n";
 			return;
 		}
 
 		foreach ($tables AS $table) {
-			$sql = "ALTER TABLE " . DBA::quoteIdentifier($table['table_name']) . " engine=InnoDB;";
+			$sql = "ALTER TABLE " . DBA::quoteIdentifier($table['table_name']) . " ENGINE=InnoDB ROW_FORMAT=DYNAMIC;";
 			echo $sql . "\n";
 
 			$result = DBA::e($sql);
@@ -106,10 +112,12 @@ class DBStructure
 
 			echo "\n";
 		}
+
+		View::printStructure($basePath);
 	}
 
 	/**
-	 * Loads the database structure definition from the config/dbstructure.config.php file.
+	 * Loads the database structure definition from the static/dbstructure.config.php file.
 	 * On first pass, defines DB_UPDATE_VERSION constant.
 	 *
 	 * @see static/dbstructure.config.php
@@ -154,11 +162,16 @@ class DBStructure
 		$comment = "";
 		$sql_rows = [];
 		$primary_keys = [];
+		$foreign_keys = [];
+
 		foreach ($structure["fields"] AS $fieldname => $field) {
 			$sql_rows[] = "`" . DBA::escape($fieldname) . "` " . self::FieldCommand($field);
 			if (!empty($field['primary'])) {
 				$primary_keys[] = $fieldname;
 			}
+			if (!empty($field['foreign'])) {
+				$foreign_keys[$fieldname] = $field;
+			}
 		}
 
 		if (!empty($structure["indexes"])) {
@@ -170,6 +183,10 @@ class DBStructure
 			}
 		}
 
+		foreach ($foreign_keys AS $fieldname => $parameters) {
+			$sql_rows[] = self::foreignCommand($name, $fieldname, $parameters);
+		}
+
 		if (isset($structure["engine"])) {
 			$engine = " ENGINE=" . $structure["engine"];
 		}
@@ -275,10 +292,18 @@ class DBStructure
 	public static function update($basePath, $verbose, $action, $install = false, array $tables = null, array $definition = null)
 	{
 		if ($action && !$install) {
+			if (self::isUpdating()) {
+				return DI::l10n()->t('Another database update is currently running.');
+			}
+
 			DI::config()->set('system', 'maintenance', 1);
 			DI::config()->set('system', 'maintenance_reason', DI::l10n()->t('%s: Database update', DateTimeFormat::utcNow() . ' ' . date('e')));
 		}
 
+		// ensure that all initial values exist. This test has to be done prior and after the structure check.
+		// Prior is needed if the specific tables already exists - after is needed when they had been created.
+		self::checkInitialValues();
+
 		$errors = '';
 
 		Logger::log('updating structure', Logger::DEBUG);
@@ -287,7 +312,7 @@ class DBStructure
 		$database = [];
 
 		if (is_null($tables)) {
-			$tables = q("SHOW TABLES");
+			$tables = DBA::toArray(DBA::p("SHOW TABLES"));
 		}
 
 		if (DBA::isResult($tables)) {
@@ -379,6 +404,7 @@ class DBStructure
 
 						// Remove the relation data that is used for the referential integrity
 						unset($parameters['relation']);
+						unset($parameters['foreign']);
 
 						// We change the collation after the indexes had been changed.
 						// This is done to avoid index length problems.
@@ -433,9 +459,43 @@ class DBStructure
 					}
 				}
 
-				if (isset($database[$name]["table_status"]["Comment"])) {
+				$existing_foreign_keys = $database[$name]['foreign_keys'];
+
+				// Foreign keys
+				// Compare the field structure field by field
+				foreach ($structure["fields"] AS $fieldname => $parameters) {
+					if (empty($parameters['foreign'])) {
+						continue;
+					}
+
+					$constraint = self::getConstraintName($name, $fieldname, $parameters);
+
+					unset($existing_foreign_keys[$constraint]);
+
+					if (empty($database[$name]['foreign_keys'][$constraint])) {
+						$sql2 = self::addForeignKey($name, $fieldname, $parameters);
+
+						if ($sql3 == "") {
+							$sql3 = "ALTER" . $ignore . " TABLE `" . $temp_name . "` " . $sql2;
+						} else {
+							$sql3 .= ", " . $sql2;
+						}
+					}
+				}
+
+				foreach ($existing_foreign_keys as $param) {
+					$sql2 = self::dropForeignKey($param['CONSTRAINT_NAME']);
+
+					if ($sql3 == "") {
+						$sql3 = "ALTER" . $ignore . " TABLE `" . $temp_name . "` " . $sql2;
+					} else {
+						$sql3 .= ", " . $sql2;
+					}
+				}
+
+				if (isset($database[$name]["table_status"]["TABLE_COMMENT"])) {
 					$structurecomment = $structure["comment"] ?? '';
-					if ($database[$name]["table_status"]["Comment"] != $structurecomment) {
+					if ($database[$name]["table_status"]["TABLE_COMMENT"] != $structurecomment) {
 						$sql2 = "COMMENT = '" . DBA::escape($structurecomment) . "'";
 
 						if ($sql3 == "") {
@@ -446,8 +506,8 @@ class DBStructure
 					}
 				}
 
-				if (isset($database[$name]["table_status"]["Engine"]) && isset($structure['engine'])) {
+				if (isset($database[$name]["table_status"]["ENGINE"]) && isset($structure['engine'])) {
-					if ($database[$name]["table_status"]["Engine"] != $structure['engine']) {
+					if ($database[$name]["table_status"]["ENGINE"] != $structure['engine']) {
 						$sql2 = "ENGINE = '" . DBA::escape($structure['engine']) . "'";
 
 						if ($sql3 == "") {
@@ -458,8 +518,8 @@ class DBStructure
 					}
 				}
 
-				if (isset($database[$name]["table_status"]["Collation"])) {
+				if (isset($database[$name]["table_status"]["TABLE_COLLATION"])) {
-					if ($database[$name]["table_status"]["Collation"] != 'utf8mb4_general_ci') {
+					if ($database[$name]["table_status"]["TABLE_COLLATION"] != 'utf8mb4_general_ci') {
 						$sql2 = "DEFAULT COLLATE utf8mb4_general_ci";
 
 						if ($sql3 == "") {
@@ -588,6 +648,10 @@ class DBStructure
 			}
 		}
 
+		View::create(false, $action);
+
+		self::checkInitialValues();
+
 		if ($action && !$install) {
 			DI::config()->set('system', 'maintenance', 0);
 			DI::config()->set('system', 'maintenance_reason', '');
@@ -604,22 +668,36 @@ class DBStructure
 
 	private static function tableStructure($table)
 	{
-		$structures = q("DESCRIBE `%s`", $table);
+		// This query doesn't seem to be executable as a prepared statement
+		$indexes = DBA::toArray(DBA::p("SHOW INDEX FROM " . DBA::quoteIdentifier($table)));
 
-		$full_columns = q("SHOW FULL COLUMNS FROM `%s`", $table);
+		$fields = DBA::selectToArray(['INFORMATION_SCHEMA' => 'COLUMNS'],
+			['COLUMN_NAME', 'COLUMN_TYPE', 'IS_NULLABLE', 'COLUMN_DEFAULT', 'EXTRA',
+			'COLUMN_KEY', 'COLLATION_NAME', 'COLUMN_COMMENT'],
+			["`TABLE_SCHEMA` = ? AND `TABLE_NAME` = ?",
+			DBA::databaseName(), $table]);
 
-		$indexes = q("SHOW INDEX FROM `%s`", $table);
+		$foreign_keys = DBA::selectToArray(['INFORMATION_SCHEMA' => 'KEY_COLUMN_USAGE'],
+			['COLUMN_NAME', 'CONSTRAINT_NAME', 'REFERENCED_TABLE_NAME', 'REFERENCED_COLUMN_NAME'],
+			["`TABLE_SCHEMA` = ? AND `TABLE_NAME` = ? AND `REFERENCED_TABLE_SCHEMA` IS NOT NULL",
+			DBA::databaseName(), $table]);
 
-		$table_status = q("SHOW TABLE STATUS WHERE `name` = '%s'", $table);
+		$table_status = DBA::selectFirst(['INFORMATION_SCHEMA' => 'TABLES'],
-
+			['ENGINE', 'TABLE_COLLATION', 'TABLE_COMMENT'],
-		if (DBA::isResult($table_status)) {
+			["`TABLE_SCHEMA` = ? AND `TABLE_NAME` = ?",
-			$table_status = $table_status[0];
+			DBA::databaseName(), $table]);
-		} else {
-			$table_status = [];
-		}
 
 		$fielddata = [];
 		$indexdata = [];
+		$foreigndata = [];
+
+		if (DBA::isResult($foreign_keys)) {
+			foreach ($foreign_keys as $foreign_key) {
+				$parameters = ['foreign' => [$foreign_key['REFERENCED_TABLE_NAME'] => $foreign_key['REFERENCED_COLUMN_NAME']]];
+				$constraint = self::getConstraintName($table, $foreign_key['COLUMN_NAME'], $parameters);
+				$foreigndata[$constraint] = $foreign_key;
+			}
+		}
 
 		if (DBA::isResult($indexes)) {
 			foreach ($indexes AS $index) {
@@ -640,39 +718,39 @@ class DBStructure
 				$indexdata[$index["Key_name"]][] = $column;
 			}
 		}
-		if (DBA::isResult($structures)) {
+
-			foreach ($structures AS $field) {
+		$fielddata = [];
-				// Replace the default size values so that we don't have to define them
+		if (DBA::isResult($fields)) {
+			foreach ($fields AS $field) {
 				$search = ['tinyint(1)', 'tinyint(3) unsigned', 'tinyint(4)', 'smallint(5) unsigned', 'smallint(6)', 'mediumint(8) unsigned', 'mediumint(9)', 'bigint(20)', 'int(10) unsigned', 'int(11)'];
 				$replace = ['boolean', 'tinyint unsigned', 'tinyint', 'smallint unsigned', 'smallint', 'mediumint unsigned', 'mediumint', 'bigint', 'int unsigned', 'int'];
-				$field["Type"] = str_replace($search, $replace, $field["Type"]);
+				$field['COLUMN_TYPE'] = str_replace($search, $replace, $field['COLUMN_TYPE']);
 
-				$fielddata[$field["Field"]]["type"] = $field["Type"];
+				$fielddata[$field['COLUMN_NAME']]['type'] = $field['COLUMN_TYPE'];
-				if ($field["Null"] == "NO") {
+
-					$fielddata[$field["Field"]]["not null"] = true;
+				if ($field['IS_NULLABLE'] == 'NO') {
+					$fielddata[$field['COLUMN_NAME']]['not null'] = true;
 				}
 
-				if (isset($field["Default"])) {
+				if (isset($field['COLUMN_DEFAULT']) && ($field['COLUMN_DEFAULT'] != 'NULL')) {
-					$fielddata[$field["Field"]]["default"] = $field["Default"];
+					$fielddata[$field['COLUMN_NAME']]['default'] = trim($field['COLUMN_DEFAULT'], "'");
 				}
 
-				if ($field["Extra"] != "") {
+				if (!empty($field['EXTRA'])) {
-					$fielddata[$field["Field"]]["extra"] = $field["Extra"];
+					$fielddata[$field['COLUMN_NAME']]['extra'] = $field['EXTRA'];
 				}
 
-				if ($field["Key"] == "PRI") {
+				if ($field['COLUMN_KEY'] == 'PRI') {
-					$fielddata[$field["Field"]]["primary"] = true;
+					$fielddata[$field['COLUMN_NAME']]['primary'] = true;
 				}
-			}
+
-		}
+				$fielddata[$field['COLUMN_NAME']]['Collation'] = $field['COLLATION_NAME'];
-		if (DBA::isResult($full_columns)) {
+				$fielddata[$field['COLUMN_NAME']]['comment'] = $field['COLUMN_COMMENT'];
-			foreach ($full_columns AS $column) {
-				$fielddata[$column["Field"]]["Collation"] = $column["Collation"];
-				$fielddata[$column["Field"]]["comment"] = $column["Comment"];
 			}
 		}
 
-		return ["fields" => $fielddata, "indexes" => $indexdata, "table_status" => $table_status];
+		return ["fields" => $fielddata, "indexes" => $indexdata,
+			"foreign_keys" => $foreigndata, "table_status" => $table_status];
 	}
 
 	private static function dropIndex($indexname)
@@ -693,6 +771,45 @@ class DBStructure
 		return ($sql);
 	}
 
+	private static function getConstraintName(string $tablename, string $fieldname, array $parameters)
+	{
+		$foreign_table = array_keys($parameters['foreign'])[0];
+		$foreign_field = array_values($parameters['foreign'])[0];
+
+		return $tablename . "-" . $fieldname. "-" . $foreign_table. "-" . $foreign_field;
+	}
+
+	private static function foreignCommand(string $tablename, string $fieldname, array $parameters) {
+		$foreign_table = array_keys($parameters['foreign'])[0];
+		$foreign_field = array_values($parameters['foreign'])[0];
+
+		$sql = "FOREIGN KEY (`" . $fieldname . "`) REFERENCES `" . $foreign_table . "` (`" . $foreign_field . "`)";
+
+		if (!empty($parameters['foreign']['on update'])) {
+			$sql .= " ON UPDATE " . strtoupper($parameters['foreign']['on update']);
+		} else {
+			$sql .= " ON UPDATE RESTRICT";
+		}
+
+		if (!empty($parameters['foreign']['on delete'])) {
+			$sql .= " ON DELETE " . strtoupper($parameters['foreign']['on delete']);
+		} else {
+			$sql .= " ON DELETE CASCADE";
+		}
+
+		return $sql;
+	}
+
+	private static function addForeignKey(string $tablename, string $fieldname, array $parameters)
+	{
+		return sprintf("ADD %s", self::foreignCommand($tablename, $fieldname, $parameters));
+	}
+
+	private static function dropForeignKey(string $constraint)
+	{
+		return sprintf("DROP FOREIGN KEY `%s`", $constraint);
+	}
+
 	/**
 	 * Constructs a GROUP BY clause from a UNIQUE index definition.
 	 *
@@ -830,6 +947,19 @@ class DBStructure
 		return true;
 	}
 
+	/**
+	 * Check if a foreign key exists for the given table field
+	 *
+	 * @param string $table
+	 * @param string $field
+	 * @return boolean
+	 */
+	public static function existsForeignKeyForField(string $table, string $field)
+	{
+		return DBA::exists(['INFORMATION_SCHEMA' => 'KEY_COLUMN_USAGE'],
+			["`TABLE_SCHEMA` = ? AND `TABLE_NAME` = ? AND `COLUMN_NAME` = ? AND `REFERENCED_TABLE_SCHEMA` IS NOT NULL",
+			DBA::databaseName(), $table, $field]);
+	}
 	/**
 	 *    Check if a table exists
 	 *
@@ -868,4 +998,97 @@ class DBStructure
 		$stmtColumns = DBA::p("SHOW COLUMNS FROM `" . $table . "`");
 		return DBA::toArray($stmtColumns);
 	}
+
+	/**
+	 * Check if initial database values do exist - or create them
+	 */
+	public static function checkInitialValues()
+	{
+		if (self::existsTable('verb') && !DBA::exists('verb', ['id' => 1])) {
+			foreach (Item::ACTIVITIES as $index => $activity) {
+				DBA::insert('verb', ['id' => $index + 1, 'name' => $activity], true);
+			}
+		}
+
+		if (self::existsTable('contact') && !DBA::exists('contact', ['id' => 0])) {
+			DBA::insert('contact', ['nurl' => '']);
+			$lastid = DBA::lastInsertId();
+			if ($lastid != 0) {
+				DBA::update('contact', ['id' => 0], ['id' => $lastid]);
+			}		
+		}
+
+		if (self::existsTable('permissionset')) {
+			if (!DBA::exists('permissionset', ['id' => 0])) {
+				DBA::insert('permissionset', ['allow_cid' => '', 'allow_gid' => '', 'deny_cid' => '', 'deny_gid' => '']);	
+				$lastid = DBA::lastInsertId();
+				if ($lastid != 0) {
+					DBA::update('permissionset', ['id' => 0], ['id' => $lastid]);
+				}
+			}
+			if (!self::existsForeignKeyForField('item', 'psid')) {
+				$sets = DBA::p("SELECT `psid`, `item`.`uid`, `item`.`private` FROM `item`
+					LEFT JOIN `permissionset` ON `permissionset`.`id` = `item`.`psid`
+					WHERE `permissionset`.`id` IS NULL AND NOT `psid` IS NULL");
+				while ($set = DBA::fetch($sets)) {
+					if (($set['private'] == Item::PRIVATE) && ($set['uid'] != 0)) {
+						$owner = User::getOwnerDataById($set['uid']);
+						if ($owner) {
+							$permission = '<' . $owner['id'] . '>';
+						} else {
+							$permission = '<>';
+						}
+					} else {
+						$permission = '';
+					}
+					$fields = ['id' => $set['psid'], 'uid' => $set['uid'], 'allow_cid' => $permission,
+						'allow_gid' => '', 'deny_cid' => '', 'deny_gid' => ''];
+					DBA::insert('permissionset', $fields);
+				}
+				DBA::close($sets);
+			}
+		}
+	
+		if (self::existsTable('tag') && !DBA::exists('tag', ['id' => 0])) {
+			DBA::insert('tag', ['name' => '']);
+			$lastid = DBA::lastInsertId();
+			if ($lastid != 0) {
+				DBA::update('tag', ['id' => 0], ['id' => $lastid]);
+			}
+		}
+
+		if (!self::existsForeignKeyForField('tokens', 'client_id')) {
+			$tokens = DBA::p("SELECT `tokens`.`id` FROM `tokens`
+				LEFT JOIN `clients` ON `clients`.`client_id` = `tokens`.`client_id`
+				WHERE `clients`.`client_id` IS NULL");
+			while ($token = DBA::fetch($tokens)) {
+				DBA::delete('tokens', ['id' => $token['id']]);
+			}
+			DBA::close($tokens);
+		}
+	}
+
+	/**
+	 * Checks if a database update is currently running
+	 *
+	 * @return boolean
+	 */
+	private static function isUpdating()
+	{
+		$isUpdate = false;
+
+		$processes = DBA::select(['information_schema' => 'processlist'], ['info'],
+			['db' => DBA::databaseName(), 'command' => ['Query', 'Execute']]);
+
+		while ($process = DBA::fetch($processes)) {
+			$parts = explode(' ', $process['info']);
+			if (in_array(strtolower(array_shift($parts)), ['alter', 'create', 'drop', 'rename'])) {
+				$isUpdate = true;
+			}
+		}
+
+		DBA::close($processes);
+
+		return $isUpdate;
+	}
 }

src/Database/Database.php

@@ -21,8 +21,10 @@
 
 namespace Friendica\Database;
 
+use Exception;
 use Friendica\Core\Config\Cache;
 use Friendica\Core\System;
+use Friendica\DI;
 use Friendica\Network\HTTPException\InternalServerErrorException;
 use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Profiler;
@@ -57,11 +59,13 @@ class Database
 	/** @var PDO|mysqli */
 	protected $connection;
 	protected $driver;
+	private $emulate_prepares = false;
 	private $error          = false;
 	private $errorno        = 0;
 	private $affected_rows  = 0;
 	protected $in_transaction = false;
 	protected $in_retrial     = false;
+	protected $testmode       = false;
 	private $relation       = [];
 
 	public function __construct(Cache $configCache, Profiler $profiler, LoggerInterface $logger, array $server = [])
@@ -130,7 +134,10 @@ class Database
 			return false;
 		}
 
-		if (class_exists('\PDO') && in_array('mysql', PDO::getAvailableDrivers())) {
+		$this->emulate_prepares = (bool)$this->configCache->get('database', 'emulate_prepares');
+		$this->pdo_emulate_prepares = (bool)$this->configCache->get('database', 'pdo_emulate_prepares');
+
+		if (!$this->configCache->get('database', 'disable_pdo') && class_exists('\PDO') && in_array('mysql', PDO::getAvailableDrivers())) {
 			$this->driver = 'pdo';
 			$connect      = "mysql:host=" . $server . ";dbname=" . $db;
 
@@ -144,7 +151,7 @@ class Database
 
 			try {
 				$this->connection = @new PDO($connect, $user, $pass);
-				$this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false);
+				$this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, $this->pdo_emulate_prepares);
 				$this->connected = true;
 			} catch (PDOException $e) {
 				$this->connected = false;
@@ -178,6 +185,10 @@ class Database
 		return $this->connected;
 	}
 
+	public function setTestmode(bool $test)
+	{
+		$this->testmode = $test;
+	}
 	/**
 	 * Sets the logger for DBA
 	 *
@@ -308,7 +319,7 @@ class Database
 		}
 
 		$watchlist = explode(',', $this->configCache->get('system', 'db_log_index_watch'));
-		$blacklist = explode(',', $this->configCache->get('system', 'db_log_index_blacklist'));
+		$denylist = explode(',', $this->configCache->get('system', 'db_log_index_denylist'));
 
 		while ($row = $this->fetch($r)) {
 			if ((intval($this->configCache->get('system', 'db_loglimit_index')) > 0)) {
@@ -322,7 +333,7 @@ class Database
 				$log = true;
 			}
 
-			if (in_array($row['key'], $blacklist) || ($row['key'] == "")) {
+			if (in_array($row['key'], $denylist) || ($row['key'] == "")) {
 				$log = false;
 			}
 
@@ -338,7 +349,7 @@ class Database
 	}
 
 	/**
-	 * Removes every not whitelisted character from the identifier string
+	 * Removes every not allowlisted character from the identifier string
 	 *
 	 * @param string $identifier
 	 *
@@ -428,8 +439,10 @@ class Database
 	{
 		$offset = 0;
 		foreach ($args AS $param => $value) {
-			if (is_int($args[$param]) || is_float($args[$param])) {
+			if (is_int($args[$param]) || is_float($args[$param]) || is_bool($args[$param])) {
 				$replace = intval($args[$param]);
+			} elseif (is_null($args[$param])) {
+				$replace = 'NULL';
 			} else {
 				$replace = "'" . $this->escape($args[$param]) . "'";
 			}
@@ -492,6 +505,7 @@ class Database
 			$sql = "/*" . System::callstack() . " */ " . $sql;
 		}
 
+		$is_error            = false;
 		$this->error         = '';
 		$this->errorno       = 0;
 		$this->affected_rows = 0;
@@ -515,12 +529,13 @@ class Database
 		switch ($this->driver) {
 			case 'pdo':
 				// If there are no arguments we use "query"
-				if (count($args) == 0) {
+				if ($this->emulate_prepares || count($args) == 0) {
-					if (!$retval = $this->connection->query($sql)) {
+					if (!$retval = $this->connection->query($this->replaceParameters($sql, $args))) {
 						$errorInfo     = $this->connection->errorInfo();
 						$this->error   = $errorInfo[2];
 						$this->errorno = $errorInfo[1];
 						$retval        = false;
+						$is_error      = true;
 						break;
 					}
 					$this->affected_rows = $retval->rowCount();
@@ -533,6 +548,7 @@ class Database
 					$this->error   = $errorInfo[2];
 					$this->errorno = $errorInfo[1];
 					$retval        = false;
+					$is_error      = true;
 					break;
 				}
 
@@ -550,6 +566,7 @@ class Database
 					$this->error   = $errorInfo[2];
 					$this->errorno = $errorInfo[1];
 					$retval        = false;
+					$is_error      = true;
 				} else {
 					$retval              = $stmt;
 					$this->affected_rows = $retval->rowCount();
@@ -562,12 +579,13 @@ class Database
 				$can_be_prepared = in_array($command, ['select', 'update', 'insert', 'delete']);
 
 				// The fallback routine is called as well when there are no arguments
-				if (!$can_be_prepared || (count($args) == 0)) {
+				if ($this->emulate_prepares || !$can_be_prepared || (count($args) == 0)) {
 					$retval = $this->connection->query($this->replaceParameters($sql, $args));
 					if ($this->connection->errno) {
 						$this->error   = $this->connection->error;
 						$this->errorno = $this->connection->errno;
 						$retval        = false;
+						$is_error      = true;
 					} else {
 						if (isset($retval->num_rows)) {
 							$this->affected_rows = $retval->num_rows;
@@ -584,6 +602,7 @@ class Database
 					$this->error   = $stmt->error;
 					$this->errorno = $stmt->errno;
 					$retval        = false;
+					$is_error      = true;
 					break;
 				}
 
@@ -611,6 +630,7 @@ class Database
 					$this->error   = $this->connection->error;
 					$this->errorno = $this->connection->errno;
 					$retval        = false;
+					$is_error      = true;
 				} else {
 					$stmt->store_result();
 					$retval              = $stmt;
@@ -619,15 +639,29 @@ class Database
 				break;
 		}
 
+		// See issue https://github.com/friendica/friendica/issues/8572
+		// Ensure that we always get an error message on an error.
+		if ($is_error && empty($this->errorno)) {
+			$this->errorno = -1;
+		}
+
+		if ($is_error && empty($this->error)) {
+			$this->error = 'Unknown database error';
+		}
+
 		// We are having an own error logging in the function "e"
 		if (($this->errorno != 0) && !$called_from_e) {
 			// We have to preserve the error code, somewhere in the logging it get lost
 			$error   = $this->error;
 			$errorno = $this->errorno;
 
+			if ($this->testmode) {
+				throw new Exception(DI::l10n()->t('Database error %d "%s" at "%s"', $errorno, $error, $this->replaceParameters($sql, $args)));
+			}
+
 			$this->logger->error('DB Error', [
-				'code'      => $this->errorno,
+				'code'      => $errorno,
-				'error'     => $this->error,
+				'error'     => $error,
 				'callstack' => System::callstack(8),
 				'params'    => $this->replaceParameters($sql, $args),
 			]);
@@ -638,21 +672,21 @@ class Database
 					// It doesn't make sense to continue when the database connection was lost
 					if ($this->in_retrial) {
 						$this->logger->notice('Giving up retrial because of database error', [
-							'code'  => $this->errorno,
+							'code'  => $errorno,
-							'error' => $this->error,
+							'error' => $error,
 						]);
 					} else {
 						$this->logger->notice('Couldn\'t reconnect after database error', [
-							'code'  => $this->errorno,
+							'code'  => $errorno,
-							'error' => $this->error,
+							'error' => $error,
 						]);
 					}
 					exit(1);
 				} else {
 					// We try it again
 					$this->logger->notice('Reconnected after database error', [
-						'code'  => $this->errorno,
+						'code'  => $errorno,
-						'error' => $this->error,
+						'error' => $error,
 					]);
 					$this->in_retrial = true;
 					$ret              = $this->p($sql, $args);
@@ -724,9 +758,13 @@ class Database
 			$error   = $this->error;
 			$errorno = $this->errorno;
 
+			if ($this->testmode) {
+				throw new Exception(DI::l10n()->t('Database error %d "%s" at "%s"', $errorno, $error, $this->replaceParameters($sql, $params)));
+			}
+
 			$this->logger->error('DB Error', [
-				'code'      => $this->errorno,
+				'code'      => $errorno,
-				'error'     => $this->error,
+				'error'     => $error,
 				'callstack' => System::callstack(8),
 				'params'    => $this->replaceParameters($sql, $params),
 			]);
@@ -735,8 +773,8 @@ class Database
 			// A reconnect like in $this->p could be dangerous with modifications
 			if ($errorno == 2006) {
 				$this->logger->notice('Giving up because of database error', [
-					'code'  => $this->errorno,
+					'code'  => $errorno,
-					'error' => $this->error,
+					'error' => $error,
 				]);
 				exit(1);
 			}
@@ -941,7 +979,7 @@ class Database
 	 * @return boolean was the insert successful?
 	 * @throws \Exception
 	 */
-	public function insert($table, $param, $on_duplicate_update = false)
+	public function insert($table, array $param, bool $on_duplicate_update = false)
 	{
 		if (empty($table) || empty($param)) {
 			$this->logger->info('Table and fields have to be set');
@@ -1009,7 +1047,7 @@ class Database
 		$success = $this->e("LOCK TABLES " . DBA::buildTableString($table) . " WRITE");
 
 		if ($this->driver == 'pdo') {
-			$this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false);
+			$this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, $this->pdo_emulate_prepares);
 		}
 
 		if (!$success) {
@@ -1042,7 +1080,7 @@ class Database
 		$success = $this->e("UNLOCK TABLES");
 
 		if ($this->driver == 'pdo') {
-			$this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false);
+			$this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, $this->pdo_emulate_prepares);
 			$this->e("SET autocommit=1");
 		} else {
 			$this->connection->autocommit(true);
@@ -1429,24 +1467,30 @@ class Database
 	/**
 	 * Select rows from a table
 	 *
+	 *
+	 * Example:
+	 * $table = 'item';
+	 * or:
+	 * $table = ['schema' => 'table'];
+	 * @see DBA::buildTableString()
+	 *
+	 * $fields = ['id', 'uri', 'uid', 'network'];
+	 *
+	 * $condition = ['uid' => 1, 'network' => 'dspr', 'blocked' => true];
+	 * or:
+	 * $condition = ['`uid` = ? AND `network` IN (?, ?)', 1, 'dfrn', 'dspr'];
+	 * @see DBA::buildCondition()
+	 *
+	 * $params = ['order' => ['id', 'received' => true, 'created' => 'ASC'), 'limit' => 10];
+	 * @see DBA::buildParameter()
+	 *
+	 * $data = DBA::select($table, $fields, $condition, $params);
+	 *
 	 * @param string|array $table     Table name or array [schema => table]
 	 * @param array        $fields    Array of selected fields, empty for all
 	 * @param array        $condition Array of fields for condition
 	 * @param array        $params    Array of several parameters
-	 *
 	 * @return boolean|object
-	 *
-	 * Example:
-	 * $table = "item";
-	 * $fields = array("id", "uri", "uid", "network");
-	 *
-	 * $condition = array("uid" => 1, "network" => 'dspr');
-	 * or:
-	 * $condition = array("`uid` = ? AND `network` IN (?, ?)", 1, 'dfrn', 'dspr');
-	 *
-	 * $params = array("order" => array("id", "received" => true), "limit" => 10);
-	 *
-	 * $data = DBA::select($table, $fields, $condition, $params);
 	 * @throws \Exception
 	 */
 	public function select($table, array $fields = [], array $condition = [], array $params = [])
@@ -1640,6 +1684,18 @@ class Database
 		return (["list" => $statelist, "amount" => $processes]);
 	}
 
+	/**
+	 * Fetch a database variable
+	 *
+	 * @param string $name
+	 * @return string content
+	 */
+	public function getVariable(string $name)
+	{
+		$result = $this->fetchFirst("SHOW GLOBAL VARIABLES WHERE `Variable_name` = ?", $name);
+		return $result['Value'] ?? null;
+	}
+
 	/**
 	 * Checks if $array is a filled array with at least one entry.
 	 *

src/Database/PostUpdate.php

@@ -25,10 +25,15 @@ use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
 use Friendica\DI;
 use Friendica\Model\Contact;
+use Friendica\Model\GServer;
 use Friendica\Model\Item;
 use Friendica\Model\ItemURI;
 use Friendica\Model\PermissionSet;
+use Friendica\Model\Post\Category;
+use Friendica\Model\Tag;
 use Friendica\Model\UserItem;
+use Friendica\Model\Verb;
+use Friendica\Util\Strings;
 
 /**
  * These database-intensive post update routines are meant to be executed in the background by the cronjob.
@@ -38,6 +43,9 @@ use Friendica\Model\UserItem;
  */
 class PostUpdate
 {
+	// Needed for the helper function to read from the legacy term table
+	const OBJECT_TYPE_POST  = 1;
+
 	/**
 	 * Calls the post update functions
 	 */
@@ -64,6 +72,30 @@ class PostUpdate
 		if (!self::update1329()) {
 			return false;
 		}
+		if (!self::update1341()) {
+			return false;
+		}
+		if (!self::update1342()) {
+			return false;
+		}
+		if (!self::update1345()) {
+			return false;
+		}
+		if (!self::update1346()) {
+			return false;
+		}
+		if (!self::update1347()) {
+			return false;
+		}
+		if (!self::update1348()) {
+			return false;
+		}
+		if (!self::update1349()) {
+			return false;
+		}
+		if (!self::update1350()) {
+			return false;
+		}
 
 		return true;
 	}
@@ -533,4 +565,489 @@ class PostUpdate
 
 		return false;
 	}
+
+	/**
+	 * Fill the "tag" table with tags and mentions from the body
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 */
+	private static function update1341()
+	{
+		// Was the script completed?
+		if (DI::config()->get('system', 'post_update_version') >= 1341) {
+			return true;
+		}
+
+		$id = DI::config()->get('system', 'post_update_version_1341_id', 0);
+
+		Logger::info('Start', ['item' => $id]);
+
+		$rows = 0;
+
+		$items = DBA::p("SELECT `uri-id`,`body` FROM `item-content` WHERE
+			(`body` LIKE ? OR `body` LIKE ? OR `body` LIKE ?) AND `uri-id` >= ?
+			ORDER BY `uri-id` LIMIT 100000", '%#%', '%@%', '%!%', $id);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($item = DBA::fetch($items)) {
+			Tag::storeFromBody($item['uri-id'], $item['body'], '#!@', false);
+			$id = $item['uri-id'];
+			++$rows;
+			if ($rows % 1000 == 0) {
+				DI::config()->set('system', 'post_update_version_1341_id', $id);
+			}
+		}
+		DBA::close($items);
+
+		DI::config()->set('system', 'post_update_version_1341_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		// When there are less than 1,000 items processed this means that we reached the end
+		// The other entries will then be processed with the regular functionality
+		if ($rows < 1000) {
+			DI::config()->set('system', 'post_update_version', 1341);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * Fill the "tag" table with tags and mentions from the "term" table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 */
+	private static function update1342()
+	{
+		// Was the script completed?
+		if (DI::config()->get('system', 'post_update_version') >= 1342) {
+			return true;
+		}
+
+		$id = DI::config()->get('system', 'post_update_version_1342_id', 0);
+
+		Logger::info('Start', ['item' => $id]);
+
+		$rows = 0;
+
+		$terms = DBA::p("SELECT `term`.`tid`, `item`.`uri-id`, `term`.`type`, `term`.`term`, `term`.`url`, `item-content`.`body`
+			FROM `term`
+			INNER JOIN `item` ON `item`.`id` = `term`.`oid`
+			INNER JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
+			WHERE term.type IN (?, ?, ?, ?) AND `tid` >= ? ORDER BY `tid` LIMIT 100000",
+			Tag::HASHTAG, Tag::MENTION, Tag::EXCLUSIVE_MENTION, Tag::IMPLICIT_MENTION, $id);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($term = DBA::fetch($terms)) {
+			if (($term['type'] == Tag::MENTION) && !empty($term['url']) && !strstr($term['body'], $term['url'])) {
+                $condition = ['nurl' => Strings::normaliseLink($term['url']), 'uid' => 0, 'deleted' => false];
+                $contact = DBA::selectFirst('contact', ['url', 'alias'], $condition, ['order' => ['id']]);
+                if (!DBA::isResult($contact)) {
+                        $ssl_url = str_replace('http://', 'https://', $term['url']);
+                        $condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $term['url'], Strings::normaliseLink($term['url']), $ssl_url, 0];
+                        $contact = DBA::selectFirst('contact', ['url', 'alias'], $condition, ['order' => ['id']]);
+                }
+
+                if (DBA::isResult($contact) && (!strstr($term['body'], $contact['url']) && (empty($contact['alias']) || !strstr($term['body'], $contact['alias'])))) {
+                        $term['type'] = Tag::IMPLICIT_MENTION;
+                }
+			}
+
+			Tag::store($term['uri-id'], $term['type'], $term['term'], $term['url'], false);
+
+			$id = $term['tid'];
+			++$rows;
+			if ($rows % 1000 == 0) {
+				DI::config()->set('system', 'post_update_version_1342_id', $id);
+			}
+		}
+		DBA::close($terms);
+
+		DI::config()->set('system', 'post_update_version_1342_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		// When there are less than 1,000 items processed this means that we reached the end
+		// The other entries will then be processed with the regular functionality
+		if ($rows < 1000) {
+			DI::config()->set('system', 'post_update_version', 1342);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * Fill the "post-delivery-data" table with data from the "item-delivery-data" table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 */
+	private static function update1345()
+	{
+		// Was the script completed?
+		if (DI::config()->get('system', 'post_update_version') >= 1345) {
+			return true;
+		}
+
+		$id = DI::config()->get('system', 'post_update_version_1345_id', 0);
+
+		Logger::info('Start', ['item' => $id]);
+
+		$rows = 0;
+
+		$deliveries = DBA::p("SELECT `uri-id`, `iid`, `item-delivery-data`.`postopts`, `item-delivery-data`.`inform`,
+			`queue_count`, `queue_done`, `activitypub`, `dfrn`, `diaspora`, `ostatus`, `legacy_dfrn`, `queue_failed`
+			FROM `item-delivery-data`
+			INNER JOIN `item` ON `item`.`id` = `item-delivery-data`.`iid`
+			WHERE `iid` >= ? ORDER BY `iid` LIMIT 10000", $id);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($delivery = DBA::fetch($deliveries)) {
+			$id = $delivery['iid'];
+			unset($delivery['iid']);
+			DBA::insert('post-delivery-data', $delivery, true);
+			++$rows;
+		}
+		DBA::close($deliveries);
+
+		DI::config()->set('system', 'post_update_version_1345_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		// When there are less than 100 items processed this means that we reached the end
+		// The other entries will then be processed with the regular functionality
+		if ($rows < 100) {
+			DI::config()->set('system', 'post_update_version', 1345);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * Generates the legacy item.file field string from an item ID.
+	 * Includes only file and category terms.
+	 *
+	 * @param int $item_id
+	 * @return string
+	 * @throws \Exception
+	 */
+	private static function fileTextFromItemId($item_id)
+	{
+		$file_text = '';
+
+		$condition = ['otype' => self::OBJECT_TYPE_POST, 'oid' => $item_id, 'type' => [Category::FILE, Category::CATEGORY]];
+		$tags = DBA::selectToArray('term', ['type', 'term', 'url'], $condition);
+		foreach ($tags as $tag) {
+			if ($tag['type'] == Category::CATEGORY) {
+				$file_text .= '<' . $tag['term'] . '>';
+			} else {
+				$file_text .= '[' . $tag['term'] . ']';
+			}
+		}
+
+		return $file_text;
+	}
+
+	/**
+	 * Fill the "tag" table with tags and mentions from the "term" table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 */
+	private static function update1346()
+	{
+		// Was the script completed?
+		if (DI::config()->get('system', 'post_update_version') >= 1346) {
+			return true;
+		}
+
+		$id = DI::config()->get('system', 'post_update_version_1346_id', 0);
+
+		Logger::info('Start', ['item' => $id]);
+
+		$rows = 0;
+
+		$terms = DBA::select('term', ['oid'],
+			["`type` IN (?, ?) AND `oid` >= ?", Category::CATEGORY, Category::FILE, $id],
+			['order' => ['oid'], 'limit' => 1000, 'group_by' => ['oid']]);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($term = DBA::fetch($terms)) {
+			$item = Item::selectFirst(['uri-id', 'uid'], ['id' => $term['oid']]);
+			if (!DBA::isResult($item)) {
+				continue;
+			}
+
+			$file = self::fileTextFromItemId($term['oid']);
+			if (!empty($file)) {
+				Category::storeTextByURIId($item['uri-id'], $item['uid'], $file);
+			}
+
+			$id = $term['oid'];
+			++$rows;
+			if ($rows % 100 == 0) {
+				DI::config()->set('system', 'post_update_version_1346_id', $id);
+			}
+		}
+		DBA::close($terms);
+
+		DI::config()->set('system', 'post_update_version_1346_id', $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		// When there are less than 10 items processed this means that we reached the end
+		// The other entries will then be processed with the regular functionality
+		if ($rows < 10) {
+			DI::config()->set('system', 'post_update_version', 1346);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * update the "vid" (verb) field in the item table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	private static function update1347()
+	{
+		// Was the script completed?
+		if (DI::config()->get("system", "post_update_version") >= 1347) {
+			return true;
+		}
+
+		$id = DI::config()->get("system", "post_update_version_1347_id", 0);
+
+		Logger::info('Start', ['item' => $id]);
+
+		$start_id = $id;
+		$rows = 0;
+
+		$items = DBA::p("SELECT `item`.`id`, `item`.`verb` AS `item-verb`, `item-content`.`verb`, `item-activity`.`activity`
+			FROM `item` LEFT JOIN `item-content` ON `item-content`.`uri-id` = `item`.`uri-id`
+			LEFT JOIN `item-activity` ON `item-activity`.`uri-id` = `item`.`uri-id` AND `item`.`gravity` = ?
+			WHERE `item`.`id` >= ? AND `item`.`vid` IS NULL ORDER BY `item`.`id` LIMIT 10000", GRAVITY_ACTIVITY, $id);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($item = DBA::fetch($items)) {
+			$id = $item['id'];
+			$verb = $item['item-verb'];
+			if (empty($verb)) {
+				$verb = $item['verb'];
+			}
+			if (empty($verb) && is_int($item['activity'])) {
+				$verb = Item::ACTIVITIES[$item['activity']];
+			}
+			if (empty($verb)) {
+				continue;
+			}
+
+			DBA::update('item', ['vid' => Verb::getID($verb)], ['id' => $item['id']]);
+			++$rows;
+		}
+		DBA::close($items);
+
+		DI::config()->set("system", "post_update_version_1347_id", $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		if ($start_id == $id) {
+			DI::config()->set("system", "post_update_version", 1347);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * update the "gsid" (global server id) field in the contact table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	private static function update1348()
+	{
+		// Was the script completed?
+		if (DI::config()->get("system", "post_update_version") >= 1348) {
+			return true;
+		}
+
+		$id = DI::config()->get("system", "post_update_version_1348_id", 0);
+
+		Logger::info('Start', ['contact' => $id]);
+
+		$start_id = $id;
+		$rows = 0;
+		$condition = ["`id` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id];
+		$params = ['order' => ['id'], 'limit' => 10000];
+		$contacts = DBA::select('contact', ['id', 'baseurl'], $condition, $params);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($contact = DBA::fetch($contacts)) {
+			$id = $contact['id'];
+
+			DBA::update('contact',
+				['gsid' => GServer::getID($contact['baseurl'], true), 'baseurl' => GServer::cleanURL($contact['baseurl'])],
+				['id' => $contact['id']]);
+
+			++$rows;
+		}
+		DBA::close($contacts);
+
+		DI::config()->set("system", "post_update_version_1348_id", $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		if ($start_id == $id) {
+			DI::config()->set("system", "post_update_version", 1348);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * update the "gsid" (global server id) field in the apcontact table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	private static function update1349()
+	{
+		// Was the script completed?
+		if (DI::config()->get("system", "post_update_version") >= 1349) {
+			return true;
+		}
+
+		$id = DI::config()->get("system", "post_update_version_1349_id", '');
+
+		Logger::info('Start', ['apcontact' => $id]);
+
+		$start_id = $id;
+		$rows = 0;
+		$condition = ["`url` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id];
+		$params = ['order' => ['url'], 'limit' => 10000];
+		$apcontacts = DBA::select('apcontact', ['url', 'baseurl'], $condition, $params);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($apcontact = DBA::fetch($apcontacts)) {
+			$id = $apcontact['url'];
+
+			DBA::update('apcontact',
+				['gsid' => GServer::getID($apcontact['baseurl'], true), 'baseurl' => GServer::cleanURL($apcontact['baseurl'])],
+				['url' => $apcontact['url']]);
+
+			++$rows;
+		}
+		DBA::close($apcontacts);
+
+		DI::config()->set("system", "post_update_version_1349_id", $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		if ($start_id == $id) {
+			DI::config()->set("system", "post_update_version", 1349);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * update the "gsid" (global server id) field in the gcontact table
+	 *
+	 * @return bool "true" when the job is done
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	private static function update1350()
+	{
+		// Was the script completed?
+		if (DI::config()->get("system", "post_update_version") >= 1350) {
+			return true;
+		}
+
+		$id = DI::config()->get("system", "post_update_version_1350_id", 0);
+
+		Logger::info('Start', ['gcontact' => $id]);
+
+		$start_id = $id;
+		$rows = 0;
+		$condition = ["`id` > ? AND `gsid` IS NULL AND `server_url` != '' AND NOT `server_url` IS NULL", $id];
+		$params = ['order' => ['id'], 'limit' => 10000];
+		$gcontacts = DBA::select('gcontact', ['id', 'server_url'], $condition, $params);
+
+		if (DBA::errorNo() != 0) {
+			Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]);
+			return false;
+		}
+
+		while ($gcontact = DBA::fetch($gcontacts)) {
+			$id = $gcontact['id'];
+
+			DBA::update('gcontact',
+				['gsid' => GServer::getID($gcontact['server_url'], true), 'server_url' => GServer::cleanURL($gcontact['server_url'])],
+				['id' => $gcontact['id']]);
+
+			++$rows;
+		}
+		DBA::close($gcontacts);
+
+		DI::config()->set("system", "post_update_version_1350_id", $id);
+
+		Logger::info('Processed', ['rows' => $rows, 'last' => $id]);
+
+		if ($start_id == $id) {
+			DI::config()->set("system", "post_update_version", 1350);
+			Logger::info('Done');
+			return true;
+		}
+
+		return false;
+	}
 }

src/Database/View.php

@@ -0,0 +1,137 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Database;
+
+use Exception;
+use Friendica\Core\Hook;
+use Friendica\DI;
+
+class View
+{
+	/**
+	 * view definition loaded from static/dbview.config.php
+	 *
+	 * @var array
+	 */
+	private static $definition = [];
+
+	/**
+	 * Loads the database structure definition from the static/dbview.config.php file.
+	 * On first pass, defines DB_UPDATE_VERSION constant.
+	 *
+	 * @see static/dbview.config.php
+	 * @param boolean $with_addons_structure Whether to tack on addons additional tables
+	 * @param string  $basePath              The base path of this application
+	 * @return array
+	 * @throws Exception
+	 */
+	public static function definition($basePath = '', $with_addons_structure = true)
+	{
+		if (!self::$definition) {
+			if (empty($basePath)) {
+				$basePath = DI::app()->getBasePath();
+			}
+
+			$filename = $basePath . '/static/dbview.config.php';
+
+			if (!is_readable($filename)) {
+				throw new Exception('Missing database view config file static/dbview.config.php');
+			}
+
+			$definition = require $filename;
+
+			if (!$definition) {
+				throw new Exception('Corrupted database view config file static/dbview.config.php');
+			}
+
+			self::$definition = $definition;
+		} else {
+			$definition = self::$definition;
+		}
+
+		if ($with_addons_structure) {
+			Hook::callAll('dbview_definition', $definition);
+		}
+
+		return $definition;
+	}
+
+	public static function create(bool $verbose, bool $action)
+	{
+		$definition = self::definition();
+
+		foreach ($definition as $name => $structure) {
+			self::createview($name, $structure, $verbose, $action);
+		}
+	}
+
+	public static function printStructure($basePath)
+	{
+		$database = self::definition($basePath, false);
+
+		foreach ($database AS $name => $structure) {
+			echo "--\n";
+			echo "-- VIEW $name\n";
+			echo "--\n";
+			self::createView($name, $structure, true, false);
+
+			echo "\n";
+		}
+	}
+
+	private static function createview($name, $structure, $verbose, $action)
+	{
+		$r = true;
+
+		$sql_rows = [];
+		foreach ($structure["fields"] AS $fieldname => $origin) {
+			if (is_string($origin)) {
+				$sql_rows[] = $origin . " AS `" . DBA::escape($fieldname) . "`";
+			} elseif (is_array($origin) && (sizeof($origin) == 2)) {
+				$sql_rows[] = "`" . DBA::escape($origin[0]) . "`.`" . DBA::escape($origin[1]) . "` AS `" . DBA::escape($fieldname) . "`";
+			}
+		}
+
+		$sql = sprintf("DROP VIEW IF EXISTS `%s`", DBA::escape($name));
+
+		if ($verbose) {
+			echo $sql . ";\n";
+		}
+
+		if ($action) {
+			DBA::e($sql);
+		}
+
+		$sql = sprintf("CREATE VIEW `%s` AS SELECT \n\t", DBA::escape($name)) .
+			implode(",\n\t", $sql_rows) . "\n\t" . $structure['query'];
+	
+		if ($verbose) {
+			echo $sql . ";\n";
+		}
+
+		if ($action) {
+			$r = DBA::e($sql);
+		}
+
+		return $r;
+	}
+}

src/Factory/Api/Mastodon/Field.php

@@ -37,7 +37,7 @@ class Field extends BaseFactory
 	 */
 	public function createFromProfileField(ProfileField $profileField)
 	{
-		return new \Friendica\Api\Entity\Mastodon\Field($profileField->label, BBCode::convert($profileField->value, false, 9));
+		return new \Friendica\Api\Entity\Mastodon\Field($profileField->label, BBCode::convert($profileField->value, false, BBCode::ACTIVITYPUB));
 	}
 
 	/**

src/Factory/Api/Twitter/User.php

@@ -0,0 +1,55 @@
+<?php
+/**
+ * @copyright Copyright (C) 2020, Friendica
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace Friendica\Factory\Api\Twitter;
+
+use Friendica\BaseFactory;
+use Friendica\Model\APContact;
+use Friendica\Model\Contact;
+use Friendica\Network\HTTPException;
+
+class User extends BaseFactory
+{
+	/**
+	 * @param int  $contactId
+	 * @param int  $uid Public contact (=0) or owner user id
+	 * @param bool $skip_status
+	 * @param bool $include_user_entities
+	 * @return \Friendica\Object\Api\Twitter\User
+	 * @throws HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	public function createFromContactId(int $contactId, $uid = 0, $skip_status = false, $include_user_entities = true)
+	{
+		$cdata = Contact::getPublicAndUserContacID($contactId, $uid);
+		if (!empty($cdata)) {
+			$publicContact = Contact::getById($cdata['public']);
+			$userContact = Contact::getById($cdata['user']);
+		} else {
+			$publicContact = Contact::getById($contactId);
+			$userContact = [];
+		}
+
+		$apcontact = APContact::getByURL($publicContact['url'], false);
+
+		return new \Friendica\Object\Api\Twitter\User($publicContact, $apcontact, $userContact, $skip_status, $include_user_entities);
+	}
+}

src/Factory/Notification/Notification.php

@@ -95,11 +95,11 @@ class Notification extends BaseFactory
 			$item['author-avatar'] = $item['contact-avatar'];
 		}
 
-		$item['label'] = (($item['id'] == $item['parent']) ? 'post' : 'comment');
+		$item['label'] = (($item['gravity'] == GRAVITY_PARENT) ? 'post' : 'comment');
 		$item['link']  = $this->baseUrl->get(true) . '/display/' . $item['parent-guid'];
 		$item['image'] = Proxy::proxifyUrl($item['author-avatar'], false, Proxy::SIZE_MICRO);
 		$item['url']   = $item['author-link'];
-		$item['text']  = (($item['id'] == $item['parent'])
+		$item['text']  = (($item['gravity'] == GRAVITY_PARENT)
 			? $this->l10n->t("%s created a new post", $item['author-name'])
 			: $this->l10n->t("%s commented on %s's post", $item['author-name'], $item['parent-author-name']));
 		$item['when']  = DateTimeFormat::local($item['created'], 'r');
@@ -272,7 +272,7 @@ class Notification extends BaseFactory
 		}
 
 		$fields = ['id', 'parent', 'verb', 'author-name', 'unseen', 'author-link', 'author-avatar', 'contact-avatar',
-			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid'];
+			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid', 'gravity'];
 		$params = ['order' => ['received' => true], 'limit' => [$start, $limit]];
 
 		$formattedNotifications = [];
@@ -313,7 +313,7 @@ class Notification extends BaseFactory
 		}
 
 		$fields = ['id', 'parent', 'verb', 'author-name', 'unseen', 'author-link', 'author-avatar', 'contact-avatar',
-			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid'];
+			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid', 'gravity'];
 		$params = ['order' => ['received' => true], 'limit' => [$start, $limit]];
 
 		$formattedNotifications = [];
@@ -350,7 +350,7 @@ class Notification extends BaseFactory
 		}
 
 		$fields = ['id', 'parent', 'verb', 'author-name', 'unseen', 'author-link', 'author-avatar', 'contact-avatar',
-			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid'];
+			'network', 'created', 'object', 'parent-author-name', 'parent-author-link', 'parent-guid', 'gravity'];
 		$params = ['order' => ['received' => true], 'limit' => [$start, $limit]];
 
 		$formattedNotifications = [];

src/Model/APContact.php

@@ -25,6 +25,8 @@ use Friendica\Content\Text\HTML;
 use Friendica\Core\Logger;
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Network\Probe;
+use Friendica\Protocol\ActivityNamespace;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Util\Crypto;
 use Friendica\Util\Network;
@@ -35,56 +37,55 @@ use Friendica\Util\Strings;
 class APContact
 {
 	/**
-	 * Resolves the profile url from the address by using webfinger
+	 * Fetch webfinger data
 	 *
-	 * @param string $addr profile address (user@domain.tld)
+	 * @param string $addr Address
-	 * @param string $url profile URL. When set then we return "true" when this profile url can be found at the address
+	 * @return array webfinger data
-	 * @return string|boolean url
-	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 */
-	private static function addrToUrl($addr, $url = null)
+	public static function fetchWebfingerData(string $addr)
 	{
 		$addr_parts = explode('@', $addr);
 		if (count($addr_parts) != 2) {
-			return false;
+			return [];
 		}
 
-		$xrd_timeout = DI::config()->get('system', 'xrd_timeout');
+		$data = ['addr' => $addr];
-
+		$template = 'https://' . $addr_parts[1] . '/.well-known/webfinger?resource=acct:' . urlencode($addr);
-		$webfinger = 'https://' . $addr_parts[1] . '/.well-known/webfinger?resource=acct:' . urlencode($addr);
+		$webfinger = Probe::webfinger(str_replace('{uri}', urlencode($addr), $template), 'application/jrd+json');
-
+		if (empty($webfinger['links'])) {
-		$curlResult = Network::curl($webfinger, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/jrd+json,application/json']);
+			$template = 'http://' . $addr_parts[1] . '/.well-known/webfinger?resource=acct:' . urlencode($addr);
-		if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
+			$webfinger = Probe::webfinger(str_replace('{uri}', urlencode($addr), $template), 'application/jrd+json');
-			$webfinger = Strings::normaliseLink($webfinger);
+			if (empty($webfinger['links'])) {
-
+				return [];
-			$curlResult = Network::curl($webfinger, false, ['timeout' => $xrd_timeout, 'accept_content' => 'application/jrd+json,application/json']);
-
-			if (!$curlResult->isSuccess() || empty($curlResult->getBody())) {
-				return false;
 			}
+			$data['baseurl'] = 'http://' . $addr_parts[1];
+		} else {
+			$data['baseurl'] = 'https://' . $addr_parts[1];
 		}
 
-		$data = json_decode($curlResult->getBody(), true);
+		foreach ($webfinger['links'] as $link) {
-
+			if (empty($link['rel'])) {
-		if (empty($data['links'])) {
-			return false;
-		}
-
-		foreach ($data['links'] as $link) {
-			if (!empty($url) && !empty($link['href']) && ($link['href'] == $url)) {
-				return true;
-			}
-
-			if (empty($link['href']) || empty($link['rel']) || empty($link['type'])) {
 				continue;
 			}
 
-			if (empty($url) && ($link['rel'] == 'self') && ($link['type'] == 'application/activity+json')) {
+			if (!empty($link['template']) && ($link['rel'] == ActivityNamespace::OSTATUSSUB)) {
-				return $link['href'];
+				$data['subscribe'] = $link['template'];
+			}
+
+			if (!empty($link['href']) && !empty($link['type']) && ($link['rel'] == 'self') && ($link['type'] == 'application/activity+json')) {
+				$data['url'] = $link['href'];
+			}
+
+			if (!empty($link['href']) && !empty($link['type']) && ($link['rel'] == 'http://webfinger.net/rel/profile-page') && ($link['type'] == 'text/html')) {
+				$data['alias'] = $link['href'];
 			}
 		}
 
-		return false;
+		if (!empty($data['url']) && !empty($data['alias']) && ($data['url'] == $data['alias'])) {
+			unset($data['alias']);
+		}
+
+		return $data;
 	}
 
 	/**
@@ -133,11 +134,15 @@ class APContact
 			}
 		}
 
-		if (empty(parse_url($url, PHP_URL_SCHEME))) {
+		$apcontact = [];
-			$url = self::addrToUrl($url);
+
-			if (empty($url)) {
+		$webfinger = empty(parse_url($url, PHP_URL_SCHEME));
+		if ($webfinger) {
+			$apcontact = self::fetchWebfingerData($url);
+			if (empty($apcontact['url'])) {
 				return $fetched_contact;
 			}
+			$url = $apcontact['url'];
 		}
 
 		$data = ActivityPub::fetchContent($url);
@@ -151,7 +156,6 @@ class APContact
 			return $fetched_contact;
 		}
 
-		$apcontact = [];
 		$apcontact['url'] = $compacted['@id'];
 		$apcontact['uuid'] = JsonLD::fetchElement($compacted, 'diaspora:guid', '@value');
 		$apcontact['type'] = str_replace('as:', '', JsonLD::fetchElement($compacted, '@type'));
@@ -182,10 +186,12 @@ class APContact
 			$apcontact['photo'] = JsonLD::fetchElement($compacted['as:icon'], 'as:url', '@id');
 		}
 
+		if (empty($apcontact['alias'])) {
 			$apcontact['alias'] = JsonLD::fetchElement($compacted, 'as:url', '@id');
 			if (is_array($apcontact['alias'])) {
 				$apcontact['alias'] = JsonLD::fetchElement($compacted['as:url'], 'as:href', '@id');
 			}
+		}
 
 		// Quit if none of the basic values are set
 		if (empty($apcontact['url']) || empty($apcontact['inbox']) || empty($apcontact['type'])) {
@@ -201,11 +207,13 @@ class APContact
 		unset($parts['scheme']);
 		unset($parts['path']);
 
+		if (empty($apcontact['addr'])) {
 			if (!empty($apcontact['nick'])) {
 				$apcontact['addr'] = $apcontact['nick'] . '@' . str_replace('//', '', Network::unparseURL($parts));
 			} else {
 				$apcontact['addr'] = '';
 			}
+		}
 
 		$apcontact['pubkey'] = null;
 		if (!empty($compacted['w3id:publicKey'])) {
@@ -276,21 +284,38 @@ class APContact
 			}
 		}
 
-		$parts = parse_url($apcontact['url']);
+		if (!$webfinger && !empty($apcontact['addr'])) {
-		unset($parts['path']);
+			$data = self::fetchWebfingerData($apcontact['addr']);
-		$baseurl = Network::unparseURL($parts);
+			if (!empty($data)) {
+				$apcontact['baseurl'] = $data['baseurl'];
 
-		// Check if the address is resolvable or the profile url is identical with the base url of the system
+				if (empty($apcontact['alias']) && !empty($data['alias'])) {
-		if (self::addrToUrl($apcontact['addr'], $apcontact['url']) || Strings::compareLink($apcontact['url'], $baseurl)) {
+					$apcontact['alias'] = $data['alias'];
-			$apcontact['baseurl'] = $baseurl;
+				}
+				if (!empty($data['subscribe'])) {
+					$apcontact['subscribe'] = $data['subscribe'];
+				}
 			} else {
 				$apcontact['addr'] = null;
 			}
+		}
 
 		if (empty($apcontact['baseurl'])) {
 			$apcontact['baseurl'] = null;
 		}
 
+		if (empty($apcontact['subscribe'])) {
+			$apcontact['subscribe'] = null;
+		}		
+
+		if (!empty($apcontact['baseurl']) && empty($fetched_contact['gsid'])) {
+			$apcontact['gsid'] = GServer::getID($apcontact['baseurl']);
+		} elseif (!empty($fetched_contact['gsid'])) {
+			$apcontact['gsid'] = $fetched_contact['gsid'];
+		} else {
+			$apcontact['gsid'] = null;
+		}
+
 		if ($apcontact['url'] == $apcontact['alias']) {
 			$apcontact['alias'] = null;
 		}
@@ -304,7 +329,7 @@ class APContact
 			DBA::delete('apcontact', ['url' => $url]);
 		}
 
-		Logger::log('Updated profile for ' . $url, Logger::DEBUG);
+		Logger::info('Updated profile', ['url' => $url]);
 
 		return $apcontact;
 	}

src/Model/Attach.php

@@ -159,7 +159,7 @@ class Attach
 	 */
 	public static function getData($item)
 	{
-		$backendClass = DI::storageManager()->getByName($photo['backend-class'] ?? '');
+		$backendClass = DI::storageManager()->getByName($item['backend-class'] ?? '');
 		if ($backendClass === null) {
 			// legacy data storage in 'data' column
 			$i = self::selectFirst(['data'], ['id' => $item['id']]);

src/Model/Contact.php

@@ -190,6 +190,44 @@ class Contact
 		return DBA::selectFirst('contact', $fields, ['id' => $id]);
 	}
 
+	/**
+	 * Fetches a contact by a given url
+	 *
+	 * @param string  $url    profile url
+	 * @param integer $uid    User ID of the contact
+	 * @param array   $fields Field list
+	 * @param boolean $update true = always update, false = never update, null = update when not found or outdated
+	 * @return array contact array
+	 */
+	public static function getByURL(string $url, int $uid = 0, array $fields = [], $update = null)
+	{
+		if ($update || is_null($update)) {
+			$cid = self::getIdForURL($url, $uid, !($update ?? false));
+			if (empty($cid)) {
+				return [];
+			}
+			return self::getById($cid, $fields);
+		}
+
+		// We first try the nurl (http://server.tld/nick), most common case
+		$options = ['order' => ['id']];
+		$contact = DBA::selectFirst('contact', $fields, ['nurl' => Strings::normaliseLink($url), 'uid' => $uid, 'deleted' => false], $options);
+
+		// Then the addr (nick@server.tld)
+		if (!DBA::isResult($contact)) {
+			$contact = DBA::selectFirst('contact', $fields, ['addr' => str_replace('acct:', '', $url), 'uid' => $uid, 'deleted' => false], $options);
+		}
+
+		// Then the alias (which could be anything)
+		if (!DBA::isResult($contact)) {
+			// The link could be provided as http although we stored it as https
+			$ssl_url = str_replace('http://', 'https://', $url);
+			$condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $url, Strings::normaliseLink($url), $ssl_url, $uid];
+			$contact = DBA::selectFirst('contact', $fields, $condition, $options);
+		}
+		return $contact;
+	}
+
 	/**
 	 * Tests if the given contact is a follower
 	 *
@@ -843,11 +881,12 @@ class Contact
 			// create an unfollow slap
 			$item = [];
 			$item['verb'] = Activity::O_UNFOLLOW;
+			$item['gravity'] = GRAVITY_ACTIVITY;
 			$item['follow'] = $contact["url"];
 			$item['body'] = '';
 			$item['title'] = '';
 			$item['guid'] = '';
-			$item['tag'] = '';
+			$item['uri-id'] = 0;
 			$item['attach'] = '';
 			$slap = OStatus::salmon($item, $user);
 
@@ -887,10 +926,10 @@ class Contact
 				return;
 			}
 		} elseif (!isset($contact['url'])) {
-			Logger::log('Empty contact: ' . json_encode($contact) . ' - ' . System::callstack(20), Logger::DEBUG);
+			Logger::info('Empty contact', ['contact' => $contact, 'callstack' => System::callstack(20)]);
 		}
 
-		Logger::log('Contact '.$contact['id'].' is marked for archival', Logger::DEBUG);
+		Logger::info('Contact is marked for archival', ['id' => $contact['id']]);
 
 		// Contact already archived or "self" contact? => nothing to do
 		if ($contact['archive'] || $contact['self']) {
@@ -949,7 +988,7 @@ class Contact
 			return;
 		}
 
-		Logger::log('Contact '.$contact['id'].' is marked as vital again', Logger::DEBUG);
+		Logger::info('Contact is marked as vital again', ['id' => $contact['id']]);
 
 		if (!isset($contact['url']) && !empty($contact['id'])) {
 			$fields = ['id', 'url', 'batch'];
@@ -1038,7 +1077,6 @@ class Contact
 		}
 
 		if (DBA::isResult($r)) {
-			$authoritativeResult = true;
 			// If there is more than one entry we filter out the connector networks
 			if (count($r) > 1) {
 				foreach ($r as $id => $result) {
@@ -1049,7 +1087,10 @@ class Contact
 			}
 
 			$profile = array_shift($r);
+		}
 
+		if (!empty($profile)) {
+			$authoritativeResult = true;
 			// "bd" always contains the upcoming birthday of a contact.
 			// "birthday" might contain the birthday including the year of birth.
 			if ($profile["birthday"] > DBA::NULL_DATE) {
@@ -1168,7 +1209,7 @@ class Contact
 		if (!DBA::isResult($r)) {
 			$data = Probe::uri($addr);
 
-			$profile = self::getDetailsByURL($data['url'], $uid);
+			$profile = self::getDetailsByURL($data['url'], $uid, $data);
 		} else {
 			$profile = $r[0];
 		}
@@ -1235,7 +1276,7 @@ class Contact
 		}
 
 		if (($contact['network'] == Protocol::DFRN) && !$contact['self'] && empty($contact['pending'])) {
-			$poke_link = DI::baseUrl() . '/poke/?c=' . $contact['id'];
+			$poke_link = 'contact/' . $contact['id'] . '/poke';
 		}
 
 		$contact_url = DI::baseUrl() . '/contact/' . $contact['id'];
@@ -1450,7 +1491,7 @@ class Contact
 	 */
 	public static function getIdForURL($url, $uid = 0, $no_update = false, $default = [], $in_loop = false)
 	{
-		Logger::log("Get contact data for url " . $url . " and user " . $uid . " - " . System::callstack(), Logger::DEBUG);
+		Logger::info('Get contact data', ['url' => $url, 'user' => $uid]);
 
 		$contact_id = 0;
 
@@ -1458,26 +1499,9 @@ class Contact
 			return 0;
 		}
 
-		/// @todo Verify if we can't use Contact::getDetailsByUrl instead of the following
+		$contact = self::getByURL($url, $uid, ['id', 'avatar', 'updated', 'network'], false);
-		// We first try the nurl (http://server.tld/nick), most common case
-		$fields = ['id', 'avatar', 'updated', 'network'];
-		$options = ['order' => ['id']];
-		$contact = DBA::selectFirst('contact', $fields, ['nurl' => Strings::normaliseLink($url), 'uid' => $uid, 'deleted' => false], $options);
 
-		// Then the addr (nick@server.tld)
+		if (!empty($contact)) {
-		if (!DBA::isResult($contact)) {
-			$contact = DBA::selectFirst('contact', $fields, ['addr' => str_replace('acct:', '', $url), 'uid' => $uid, 'deleted' => false], $options);
-		}
-
-		// Then the alias (which could be anything)
-		if (!DBA::isResult($contact)) {
-			// The link could be provided as http although we stored it as https
-			$ssl_url = str_replace('http://', 'https://', $url);
-			$condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $url, Strings::normaliseLink($url), $ssl_url, $uid];
-			$contact = DBA::selectFirst('contact', $fields, $condition, $options);
-		}
-
-		if (DBA::isResult($contact)) {
 			$contact_id = $contact["id"];
 			$update_contact = false;
 
@@ -1530,10 +1554,6 @@ class Contact
 
 		if (empty($data)) {
 			$data = Probe::uri($url, "", $uid);
-			// Ensure that there is a gserver entry
-			if (!empty($data['baseurl']) && ($data['network'] != Protocol::PHANTOM)) {
-				GServer::check($data['baseurl']);
-			}
 		}
 
 		// Take the default values when probing failed
@@ -1546,7 +1566,15 @@ class Contact
 			return 0;
 		}
 
-		if (!$contact_id && !empty($data['alias']) && ($data['alias'] != $url) && !$in_loop) {
+		if (!empty($data['baseurl'])) {
+			$data['baseurl'] = GServer::cleanURL($data['baseurl']);
+		}
+
+		if (!empty($data['baseurl']) && empty($data['gsid'])) {
+			$data['gsid'] = GServer::getID($data['baseurl']);
+		}
+
+		if (!$contact_id && !empty($data['alias']) && ($data['alias'] != $data['url']) && !$in_loop) {
 			$contact_id = self::getIdForURL($data["alias"], $uid, true, $default, true);
 		}
 
@@ -1575,6 +1603,7 @@ class Contact
 				'confirm'   => $data['confirm'] ?? '',
 				'poco'      => $data['poco'] ?? '',
 				'baseurl'   => $data['baseurl'] ?? '',
+				'gsid'      => $data['gsid'] ?? null,
 				'name-date' => DateTimeFormat::utcNow(),
 				'uri-date'  => DateTimeFormat::utcNow(),
 				'avatar-date' => DateTimeFormat::utcNow(),
@@ -1627,7 +1656,7 @@ class Contact
 				}
 			}
 		} else {
-			$fields = ['url', 'nurl', 'addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'avatar-date', 'baseurl'];
+			$fields = ['url', 'nurl', 'addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'avatar-date', 'baseurl', 'gsid'];
 			$contact = DBA::selectFirst('contact', $fields, ['id' => $contact_id]);
 
 			// This condition should always be true
@@ -1641,7 +1670,7 @@ class Contact
 				'updated' => DateTimeFormat::utcNow()
 			];
 
-			$fields = ['addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'baseurl'];
+			$fields = ['addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'baseurl', 'gsid'];
 
 			foreach ($fields as $field) {
 				$updated[$field] = ($data[$field] ?? '') ?: $contact[$field];
@@ -1756,7 +1785,6 @@ class Contact
 	 * Returns posts from a given contact url
 	 *
 	 * @param string $contact_url Contact URL
-	 *
 	 * @param bool   $thread_mode
 	 * @param int    $update
 	 * @return string posts in HTML
@@ -1764,9 +1792,21 @@ class Contact
 	 */
 	public static function getPostsFromUrl($contact_url, $thread_mode = false, $update = 0)
 	{
-		$a = DI::app();
+		return self::getPostsFromId(self::getIdForURL($contact_url), $thread_mode, $update);
+	}
 
-		$cid = self::getIdForURL($contact_url);
+	/**
+	 * Returns posts from a given contact id
+	 *
+	 * @param integer $cid
+	 * @param bool    $thread_mode
+	 * @param integer $update
+	 * @return string posts in HTML
+	 * @throws \Exception
+	 */
+	public static function getPostsFromId($cid, $thread_mode = false, $update = 0)
+	{
+		$a = DI::app();
 
 		$contact = DBA::selectFirst('contact', ['contact-type', 'network'], ['id' => $cid]);
 		if (!DBA::isResult($contact)) {
@@ -2057,6 +2097,7 @@ class Contact
 
 			Worker::add(PRIORITY_HIGH, 'MergeContact', $first, $duplicate['id'], $uid);
 		}
+		DBA::close($duplicates);
 		Logger::info('Duplicates handled', ['uid' => $uid, 'nurl' => $nurl]);
 		return true;
 	}
@@ -2079,9 +2120,9 @@ class Contact
 		// These fields aren't updated by this routine:
 		// 'xmpp', 'sensitive'
 
-		$fields = ['uid', 'avatar', 'name', 'nick', 'location', 'keywords', 'about',
+		$fields = ['uid', 'avatar', 'name', 'nick', 'location', 'keywords', 'about', 'subscribe',
 			'unsearchable', 'url', 'addr', 'batch', 'notify', 'poll', 'request', 'confirm', 'poco',
-			'network', 'alias', 'baseurl', 'forum', 'prv', 'contact-type', 'pubkey'];
+			'network', 'alias', 'baseurl', 'gsid', 'forum', 'prv', 'contact-type', 'pubkey'];
 		$contact = DBA::selectFirst('contact', $fields, ['id' => $id]);
 		if (!DBA::isResult($contact)) {
 			return false;
@@ -2255,20 +2296,20 @@ class Contact
 	 * $return['message'] error text if success is false.
 	 *
 	 * Takes a $uid and a url/handle and adds a new contact
-	 * @param int    $uid
+	 *
-	 * @param string $url
+	 * @param array  $user        The user the contact should be created for
+	 * @param string $url         The profile URL of the contact
 	 * @param bool   $interactive
 	 * @param string $network
 	 * @return array
 	 * @throws HTTPException\InternalServerErrorException
+	 * @throws HTTPException\NotFoundException
 	 * @throws \ImagickException
 	 */
-	public static function createFromProbe($uid, $url, $interactive = false, $network = '')
+	public static function createFromProbe(array $user, $url, $interactive = false, $network = '')
 	{
 		$result = ['cid' => -1, 'success' => false, 'message' => ''];
 
-		$a = DI::app();
-
 		// remove ajax junk, e.g. Twitter
 		$url = str_replace('/#!/', '/', $url);
 
@@ -2299,7 +2340,7 @@ class Contact
 		if (!empty($arr['contact']['name'])) {
 			$ret = $arr['contact'];
 		} else {
-			$ret = Probe::uri($url, $network, $uid, false);
+			$ret = Probe::uri($url, $network, $user['uid'], false);
 		}
 
 		if (($network != '') && ($ret['network'] != $network)) {
@@ -2311,21 +2352,21 @@ class Contact
 		// the poll url is more reliable than the profile url, as we may have
 		// indirect links or webfinger links
 
-		$condition = ['uid' => $uid, 'poll' => [$ret['poll'], Strings::normaliseLink($ret['poll'])], 'network' => $ret['network'], 'pending' => false];
+		$condition = ['uid' => $user['uid'], 'poll' => [$ret['poll'], Strings::normaliseLink($ret['poll'])], 'network' => $ret['network'], 'pending' => false];
 		$contact = DBA::selectFirst('contact', ['id', 'rel'], $condition);
 		if (!DBA::isResult($contact)) {
-			$condition = ['uid' => $uid, 'nurl' => Strings::normaliseLink($url), 'network' => $ret['network'], 'pending' => false];
+			$condition = ['uid' => $user['uid'], 'nurl' => Strings::normaliseLink($ret['url']), 'network' => $ret['network'], 'pending' => false];
 			$contact = DBA::selectFirst('contact', ['id', 'rel'], $condition);
 		}
 
-		$protocol = self::getProtocol($url, $ret['network']);
+		$protocol = self::getProtocol($ret['url'], $ret['network']);
 
 		if (($protocol === Protocol::DFRN) && !DBA::isResult($contact)) {
 			if ($interactive) {
 				if (strlen(DI::baseUrl()->getUrlPath())) {
-					$myaddr = bin2hex(DI::baseUrl() . '/profile/' . $a->user['nickname']);
+					$myaddr = bin2hex(DI::baseUrl() . '/profile/' . $user['nickname']);
 				} else {
-					$myaddr = bin2hex($a->user['nickname'] . '@' . DI::baseUrl()->getHostname());
+					$myaddr = bin2hex($user['nickname'] . '@' . DI::baseUrl()->getHostname());
 				}
 
 				DI::baseUrl()->redirect($ret['request'] . "&addr=$myaddr");
@@ -2355,7 +2396,7 @@ class Contact
 			if (empty($ret['url'])) {
 				$result['message'] .= DI::l10n()->t('No browser URL could be matched to this address.') . EOL;
 			}
-			if (strpos($url, '@') !== false) {
+			if (strpos($ret['url'], '@') !== false) {
 				$result['message'] .= DI::l10n()->t('Unable to match @-style Identity Address with a known protocol or email contact.') . EOL;
 				$result['message'] .= DI::l10n()->t('Use mailto: in front of address to force email check.') . EOL;
 			}
@@ -2379,7 +2420,7 @@ class Contact
 
 		$pending = false;
 		if ($protocol == Protocol::ACTIVITYPUB) {
-			$apcontact = APContact::getByURL($url, false);
+			$apcontact = APContact::getByURL($ret['url'], false);
 			if (isset($apcontact['manually-approve'])) {
 				$pending = (bool)$apcontact['manually-approve'];
 			}
@@ -2400,7 +2441,7 @@ class Contact
 
 			// create contact record
 			self::insert([
-				'uid'     => $uid,
+				'uid'     => $user['uid'],
 				'created' => DateTimeFormat::utcNow(),
 				'url'     => $ret['url'],
 				'nurl'    => Strings::normaliseLink($ret['url']),
@@ -2414,6 +2455,7 @@ class Contact
 				'nick'    => $ret['nick'],
 				'network' => $ret['network'],
 				'baseurl' => $ret['baseurl'],
+				'gsid'    => $ret['gsid'] ?? null,
 				'protocol' => $protocol,
 				'pubkey'  => $ret['pubkey'],
 				'rel'     => $new_relation,
@@ -2427,7 +2469,7 @@ class Contact
 			]);
 		}
 
-		$contact = DBA::selectFirst('contact', [], ['url' => $ret['url'], 'network' => $ret['network'], 'uid' => $uid]);
+		$contact = DBA::selectFirst('contact', [], ['url' => $ret['url'], 'network' => $ret['network'], 'uid' => $user['uid']]);
 		if (!DBA::isResult($contact)) {
 			$result['message'] .= DI::l10n()->t('Unable to retrieve contact information.') . EOL;
 			return $result;
@@ -2436,27 +2478,28 @@ class Contact
 		$contact_id = $contact['id'];
 		$result['cid'] = $contact_id;
 
-		Group::addMember(User::getDefaultGroup($uid, $contact["network"]), $contact_id);
+		Group::addMember(User::getDefaultGroup($user['uid'], $contact["network"]), $contact_id);
 
 		// Update the avatar
-		self::updateAvatar($ret['photo'], $uid, $contact_id);
+		self::updateAvatar($ret['photo'], $user['uid'], $contact_id);
 
 		// pull feed and consume it, which should subscribe to the hub.
 
 		Worker::add(PRIORITY_HIGH, "OnePoll", $contact_id, "force");
 
-		$owner = User::getOwnerDataById($uid);
+		$owner = User::getOwnerDataById($user['uid']);
 
 		if (DBA::isResult($owner)) {
 			if (in_array($protocol, [Protocol::OSTATUS, Protocol::DFRN])) {
 				// create a follow slap
 				$item = [];
 				$item['verb'] = Activity::FOLLOW;
+				$item['gravity'] = GRAVITY_ACTIVITY;
 				$item['follow'] = $contact["url"];
 				$item['body'] = '';
 				$item['title'] = '';
 				$item['guid'] = '';
-				$item['tag'] = '';
+				$item['uri-id'] = 0;
 				$item['attach'] = '';
 
 				$slap = OStatus::salmon($item, $owner);
@@ -2465,7 +2508,7 @@ class Contact
 					Salmon::slapper($owner, $contact['notify'], $slap);
 				}
 			} elseif ($protocol == Protocol::DIASPORA) {
-				$ret = Diaspora::sendShare($a->user, $contact);
+				$ret = Diaspora::sendShare($owner, $contact);
 				Logger::log('share returns: ' . $ret);
 			} elseif ($protocol == Protocol::ACTIVITYPUB) {
 				$activity_id = ActivityPub\Transmitter::activityIDFromContact($contact_id);
@@ -2474,7 +2517,7 @@ class Contact
 					return false;
 				}
 
-				$ret = ActivityPub\Transmitter::sendActivity('Follow', $contact['url'], $uid, $activity_id);
+				$ret = ActivityPub\Transmitter::sendActivity('Follow', $contact['url'], $user['uid'], $activity_id);
 				Logger::log('Follow returns: ' . $ret);
 			}
 		}
@@ -2659,7 +2702,7 @@ class Contact
 				}
 			} elseif (DBA::isResult($user) && in_array($user['page-flags'], [User::PAGE_FLAGS_SOAPBOX, User::PAGE_FLAGS_FREELOVE, User::PAGE_FLAGS_COMMUNITY])) {
 				if (($user['page-flags'] == User::PAGE_FLAGS_FREELOVE) && ($network != Protocol::DIASPORA)) {
-					self::createFromProbe($importer['uid'], $url, false, $network);
+					self::createFromProbe($importer, $url, false, $network);
 				}
 
 				$condition = ['uid' => $importer['uid'], 'url' => $url, 'pending' => true];
@@ -2732,6 +2775,7 @@ class Contact
 				);
 			}
 		}
+		DBA::close($contacts);
 	}
 
 	/**

src/Model/Event.php

@@ -24,6 +24,7 @@ namespace Friendica\Model;
 use Friendica\Content\Text\BBCode;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
+use Friendica\Core\Protocol;
 use Friendica\Core\Renderer;
 use Friendica\Core\System;
 use Friendica\Database\DBA;
@@ -370,6 +371,7 @@ class Event
 				$item_arr['origin']        = $event['cid'] === 0 ? 1 : 0;
 				$item_arr['body']          = self::getBBCode($event);
 				$item_arr['event-id']      = $event['id'];
+				$item_arr['network']       = Protocol::DFRN;
 
 				$item_arr['object']  = '<object><type>' . XML::escape(Activity\ObjectType::EVENT) . '</type><title></title><id>' . XML::escape($event['uri']) . '</id>';
 				$item_arr['object'] .= '<content>' . XML::escape(self::getBBCode($event)) . '</content>';
@@ -611,14 +613,12 @@ class Event
 
 			$title = BBCode::convert(Strings::escapeHtml($event['summary']));
 			if (!$title) {
-				list($title, $_trash) = explode("<br", BBCode::convert(Strings::escapeHtml($event['desc'])), 2);
+				list($title, $_trash) = explode("<br", BBCode::convert(Strings::escapeHtml($event['desc'])), BBCode::API);
 			}
 
 			$author_link = $event['author-link'];
-			$plink       = $event['plink'];
 
 			$event['author-link'] = Contact::magicLink($author_link);
-			$event['plink']       = Contact::magicLink($author_link, $plink);
 
 			$html = self::getHTML($event);
 			$event['summary']  = BBCode::convert(Strings::escapeHtml($event['summary']));
@@ -638,7 +638,7 @@ class Event
 				'is_first' => $is_first,
 				'item'     => $event,
 				'html'     => $html,
-				'plink'    => [$event['plink'], DI::l10n()->t('link to source'), '', ''],
+				'plink'    => Item::getPlink($event),
 			];
 		}
 

src/Model/FileTag.php

@@ -23,6 +23,7 @@ namespace Friendica\Model;
 
 use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\Post\Category;
 
 /**
  * This class handles FileTag related functions
@@ -195,11 +196,11 @@ class FileTag
 			if ($type == 'file') {
 				$lbracket = '[';
 				$rbracket = ']';
-				$termtype = TERM_FILE;
+				$termtype = Category::FILE;
 			} else {
 				$lbracket = '<';
 				$rbracket = '>';
-				$termtype = TERM_CATEGORY;
+				$termtype = Category::CATEGORY;
 			}
 
 			$filetags_updated = $saved;
@@ -223,13 +224,7 @@ class FileTag
 			}
 
 			foreach ($deleted_tags as $key => $tag) {
-				$r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d",
+				if (DBA::exists('category-view', ['name' => $tag, 'type' => $termtype, 'uid' => $uid])) {
-					DBA::escape($tag),
-					intval(Term::OBJECT_TYPE_POST),
-					intval($termtype),
-					intval($uid));
-
-				if (DBA::isResult($r)) {
 					unset($deleted_tags[$key]);
 				} else {
 					$filetags_updated = str_replace($lbracket . self::encode($tag) . $rbracket, '', $filetags_updated);
@@ -302,10 +297,10 @@ class FileTag
 
 		if ($cat == true) {
 			$pattern = '<' . self::encode($file) . '>';
-			$termtype = Term::CATEGORY;
+			$termtype = Category::CATEGORY;
 		} else {
 			$pattern = '[' . self::encode($file) . ']';
-			$termtype = Term::FILE;
+			$termtype = Category::FILE;
 		}
 
 		$item = Item::selectFirst(['file'], ['id' => $item_id, 'uid' => $uid]);
@@ -318,14 +313,7 @@ class FileTag
 
 		Item::update($fields, ['id' => $item_id]);
 
-		$r = q("SELECT `oid` FROM `term` WHERE `term` = '%s' AND `otype` = %d AND `type` = %d AND `uid` = %d",
+		if (!DBA::exists('category-view', ['name' => $file, 'type' => $termtype, 'uid' => $uid])) {
-			DBA::escape($file),
-			intval(Term::OBJECT_TYPE_POST),
-			intval($termtype),
-			intval($uid)
-		);
-
-		if (!DBA::isResult($r)) {
 			$saved = DI::pConfig()->get($uid, 'system', 'filetags');
 			DI::pConfig()->set($uid, 'system', 'filetags', str_replace($pattern, '', $saved));
 		}

src/Model/GContact.php

@@ -113,6 +113,7 @@ class GContact
 
 			$gcontacts[] = Contact::getDetailsByURL($result['nurl'], local_user());
 		}
+		DBA::close($results);
 		return $gcontacts;
 	}
 
@@ -229,8 +230,6 @@ class GContact
 				throw new Exception('Probing for URL ' . $gcontact['url'] . ' failed');
 			}
 
-			$orig_profile = $gcontact['url'];
-
 			$gcontact['server_url'] = $data['baseurl'];
 
 			$gcontact = array_merge($gcontact, $data);
@@ -563,6 +562,7 @@ class GContact
 				PortableContact::loadWorker(0, 0, 0, $base);
 			}
 		}
+		DBA::close($contacts);
 	}
 
 	/**
@@ -609,8 +609,6 @@ class GContact
 	 */
 	public static function getId($contact)
 	{
-		$gcontact_id = 0;
-
 		if (empty($contact['network'])) {
 			Logger::notice('Empty network', ['url' => $contact['url'], 'callstack' => System::callstack()]);
 			return false;
@@ -625,25 +623,21 @@ class GContact
 			$contact['network'] = Protocol::OSTATUS;
 		}
 
-		// All new contacts are hidden by default
-		if (!isset($contact['hide'])) {
-			$contact['hide'] = true;
-		}
-
 		// Remove unwanted parts from the contact url (e.g. '?zrl=...')
 		if (in_array($contact['network'], Protocol::FEDERATED)) {
 			$contact['url'] = self::cleanContactUrl($contact['url']);
 		}
 
-		DBA::lock('gcontact');
+		$condition = ['nurl' => Strings::normaliseLink($contact['url'])];
-		$fields = ['id', 'last_contact', 'last_failure', 'network'];
+		$gcontact = DBA::selectFirst('gcontact', ['id'], $condition, ['order' => ['id']]);
-		$gcnt = DBA::selectFirst('gcontact', $fields, ['nurl' => Strings::normaliseLink($contact['url'])]);
+		if (DBA::isResult($gcontact)) {
-		if (DBA::isResult($gcnt)) {
+			return $gcontact['id'];
-			$gcontact_id = $gcnt['id'];
+		}
-		} else {
+
 		$contact['location'] = $contact['location'] ?? '';
 		$contact['about'] = $contact['about'] ?? '';
 		$contact['generation'] = $contact['generation'] ?? 0;
+		$contact['hide'] = $contact['hide'] ?? true;
 
 		$fields = ['name' => $contact['name'], 'nick' => $contact['nick'] ?? '', 'addr' => $contact['addr'] ?? '', 'network' => $contact['network'],
 			'url' => $contact['url'], 'nurl' => Strings::normaliseLink($contact['url']), 'photo' => $contact['photo'],
@@ -652,15 +646,14 @@ class GContact
 
 		DBA::insert('gcontact', $fields);
 
-			$condition = ['nurl' => Strings::normaliseLink($contact['url'])];
+		// We intentionally aren't using lastInsertId here. There is a chance for duplicates.
-			$cnt = DBA::selectFirst('gcontact', ['id', 'network'], $condition, ['order' => ['id']]);
+		$gcontact = DBA::selectFirst('gcontact', ['id'], $condition, ['order' => ['id']]);
-			if (DBA::isResult($cnt)) {
+		if (!DBA::isResult($gcontact)) {
-				$gcontact_id = $cnt['id'];
+			Logger::info('GContact creation failed', $fields);
+			// Shouldn't happen
+			return 0;
 		}
-		}
+		return $gcontact['id'];
-		DBA::unlock();
-
-		return $gcontact_id;
 	}
 
 	/**
@@ -688,7 +681,7 @@ class GContact
 		}
 
 		$public_contact = DBA::selectFirst('gcontact', [
-			'name', 'nick', 'photo', 'location', 'about', 'addr', 'generation', 'birthday', 'keywords',
+			'name', 'nick', 'photo', 'location', 'about', 'addr', 'generation', 'birthday', 'keywords', 'gsid',
 			'contact-type', 'hide', 'nsfw', 'network', 'alias', 'notify', 'server_url', 'connect', 'updated', 'url'
 		], ['id' => $gcontact_id]);
 
@@ -750,6 +743,10 @@ class GContact
 			$contact['server_url'] = Strings::normaliseLink($contact['server_url']);
 		}
 
+		if (!empty($contact['server_url']) && empty($contact['gsid'])) {
+			$contact['gsid'] = GServer::getID($contact['server_url']);
+		}
+
 		if (empty($contact['addr']) && !empty($contact['server_url']) && !empty($contact['nick'])) {
 			$hostname = str_replace('http://', '', $contact['server_url']);
 			$contact['addr'] = $contact['nick'] . '@' . $hostname;
@@ -789,7 +786,8 @@ class GContact
 				'notify' => $contact['notify'], 'url' => $contact['url'],
 				'location' => $contact['location'], 'about' => $contact['about'],
 				'generation' => $contact['generation'], 'updated' => $contact['updated'],
-				'server_url' => $contact['server_url'], 'connect' => $contact['connect']
+				'server_url' => $contact['server_url'], 'connect' => $contact['connect'],
+				'gsid' => $contact['gsid']
 			];
 
 			DBA::update('gcontact', $updated, $condition, $fields);
@@ -1014,7 +1012,7 @@ class GContact
 		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords',
 			'bd', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archive', 'term-date',
 			'created', 'updated', 'avatar', 'success_update', 'failure_update', 'forum', 'prv',
-			'baseurl', 'sensitive', 'unsearchable'];
+			'baseurl', 'gsid', 'sensitive', 'unsearchable'];
 
 		$contact = DBA::selectFirst('contact', $fields, array_merge($condition, ['uid' => 0, 'network' => Protocol::FEDERATED]));
 		if (!DBA::isResult($contact)) {
@@ -1024,7 +1022,7 @@ class GContact
 		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'generation',
 			'birthday', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archived', 'archive_date',
 			'created', 'updated', 'photo', 'last_contact', 'last_failure', 'community', 'connect',
-			'server_url', 'nsfw', 'hide', 'id'];
+			'server_url', 'gsid', 'nsfw', 'hide', 'id'];
 
 		$old_gcontact = DBA::selectFirst('gcontact', $fields, ['nurl' => $contact['nurl']]);
 		$do_insert = !DBA::isResult($old_gcontact);
@@ -1035,7 +1033,7 @@ class GContact
 		$gcontact = [];
 
 		// These fields are identical in both contact and gcontact
-		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords',
+		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gsid',
 			'contact-type', 'network', 'addr', 'notify', 'alias', 'created', 'updated'];
 
 		foreach ($fields as $field) {

src/Model/GServer.php

@@ -34,6 +34,7 @@ use Friendica\Util\DateTimeFormat;
 use Friendica\Util\Strings;
 use Friendica\Util\XML;
 use Friendica\Core\Logger;
+use Friendica\Core\System;
 use Friendica\Protocol\PortableContact;
 use Friendica\Protocol\Diaspora;
 use Friendica\Network\Probe;
@@ -47,6 +48,57 @@ class GServer
 	const DT_NONE = 0;
 	const DT_POCO = 1;
 	const DT_MASTODON = 2;
+
+	// Methods to detect server types
+
+	// Non endpoint specific methods
+	const DETECT_MANUAL = 0;
+	const DETECT_HEADER = 1;
+	const DETECT_BODY = 2;
+
+	// Implementation specific endpoints
+	const DETECT_FRIENDIKA = 10;
+	const DETECT_FRIENDICA = 11;
+	const DETECT_STATUSNET = 12;
+	const DETECT_GNUSOCIAL = 13;
+	const DETECT_CONFIG_JSON = 14; // Statusnet, GNU Social, Older Hubzilla/Redmatrix
+	const DETECT_SITEINFO_JSON = 15; // Newer Hubzilla
+	const DETECT_MASTODON_API = 16;
+	const DETECT_STATUS_PHP = 17; // Nextcloud
+
+	// Standardized endpoints
+	const DETECT_STATISTICS_JSON = 100;
+	const DETECT_NODEINFO_1 = 101;
+	const DETECT_NODEINFO_2 = 102;
+
+	/**
+	 * Get the ID for the given server URL
+	 *
+	 * @param string $url
+	 * @param boolean $no_check Don't check if the server hadn't been found
+	 * @return int gserver id
+	 */
+	public static function getID(string $url, bool $no_check = false)
+	{
+		if (empty($url)) {
+			return null;
+		}
+
+		$url = self::cleanURL($url);
+
+		$gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]);
+		if (DBA::isResult($gserver)) {
+			Logger::info('Got ID for URL', ['id' => $gserver['id'], 'url' => $url, 'callstack' => System::callstack(20)]);
+			return $gserver['id'];
+		}
+
+		if ($no_check || !self::check($url)) {
+			return null;
+		}
+	
+		return self::getID($url, true);
+	}
+
 	/**
 	 * Checks if the given server is reachable
 	 *
@@ -131,14 +183,13 @@ class GServer
 	 * @param string  $server_url    URL of the given server
 	 * @param string  $network       Network value that is used, when detection failed
 	 * @param boolean $force         Force an update.
+	 * @param boolean $only_nodeinfo Only use nodeinfo for server detection
 	 *
 	 * @return boolean 'true' if server seems vital
 	 */
-	public static function check(string $server_url, string $network = '', bool $force = false)
+	public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false)
 	{
-		// Unify the server address
+		$server_url = self::cleanURL($server_url);
-		$server_url = trim($server_url, '/');
-		$server_url = str_replace('/index.php', '', $server_url);
 
 		if ($server_url == '') {
 			return false;
@@ -174,7 +225,58 @@ class GServer
 			Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]);
 		}
 
-		return self::detect($server_url, $network);
+		return self::detect($server_url, $network, $only_nodeinfo);
+	}
+
+	/**
+	 * Set failed server status
+	 *
+	 * @param string $url
+	 */
+	private static function setFailure(string $url)
+	{
+		if (DBA::exists('gserver', ['nurl' => Strings::normaliseLink($url)])) {
+			DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow(), 'detection-method' => null],
+			['nurl' => Strings::normaliseLink($url)]);
+			Logger::info('Set failed status for existing server', ['url' => $url]);
+			return;
+		}
+		DBA::insert('gserver', ['url' => $url, 'nurl' => Strings::normaliseLink($url),
+			'network' => Protocol::PHANTOM, 'created' => DateTimeFormat::utcNow(),
+			'last_failure' => DateTimeFormat::utcNow()]);
+		Logger::info('Set failed status for new server', ['url' => $url]);
+	}
+
+	/**
+	 * Remove unwanted content from the given URL
+	 *
+	 * @param string $url
+	 * @return string cleaned URL
+	 */
+	public static function cleanURL(string $url)
+	{
+		$url = trim($url, '/');
+		$url = str_replace('/index.php', '', $url);
+
+		$urlparts = parse_url($url);
+		unset($urlparts['user']);
+		unset($urlparts['pass']);
+		unset($urlparts['query']);
+		unset($urlparts['fragment']);
+		return Network::unparseURL($urlparts);
+	}
+
+	/**
+	 * Return the base URL
+	 *
+	 * @param string $url
+	 * @return string base URL
+	 */
+	private static function getBaseURL(string $url)
+	{
+		$urlparts = parse_url(self::cleanURL($url));
+		unset($urlparts['path']);
+		return Network::unparseURL($urlparts);
 	}
 
 	/**
@@ -183,23 +285,22 @@ class GServer
 	 *
 	 * @param string  $url           URL of the given server
 	 * @param string  $network       Network value that is used, when detection failed
+	 * @param boolean $only_nodeinfo Only use nodeinfo for server detection
 	 *
 	 * @return boolean 'true' if server could be detected
 	 */
-	public static function detect(string $url, string $network = '')
+	public static function detect(string $url, string $network = '', bool $only_nodeinfo = false)
 	{
 		Logger::info('Detect server type', ['server' => $url]);
-		$serverdata = [];
+		$serverdata = ['detection-method' => self::DETECT_MANUAL];
 
 		$original_url = $url;
 
 		// Remove URL content that is not supposed to exist for a server url
-		$urlparts = parse_url($url);
+		$url = self::cleanURL($url);
-		unset($urlparts['user']);
+
-		unset($urlparts['pass']);
+		// Get base URL
-		unset($urlparts['query']);
+		$baseurl = self::getBaseURL($url);
-		unset($urlparts['fragment']);
-		$url = Network::unparseURL($urlparts);
 
 		// If the URL missmatches, then we mark the old entry as failure
 		if ($url != $original_url) {
@@ -210,11 +311,16 @@ class GServer
 		$xrd_timeout = DI::config()->get('system', 'xrd_timeout');
 		$curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]);
 		if ($curlResult->isTimeout()) {
-			DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]);
+			self::setFailure($url);
 			return false;
 		}
 
 		$nodeinfo = self::fetchNodeinfo($url, $curlResult);
+		if ($only_nodeinfo && empty($nodeinfo)) {
+			Logger::info('Invalid nodeinfo in nodeinfo-mode, server is marked as failure', ['url' => $url]);
+			self::setFailure($url);
+			return false;
+		}
 
 		// When nodeinfo isn't present, we use the older 'statistics.json' endpoint
 		if (empty($nodeinfo)) {
@@ -224,18 +330,53 @@ class GServer
 		// If that didn't work out well, we use some protocol specific endpoints
 		// For Friendica and Zot based networks we have to dive deeper to reveal more details
 		if (empty($nodeinfo['network']) || in_array($nodeinfo['network'], [Protocol::DFRN, Protocol::ZOT])) {
+			if (!empty($nodeinfo['detection-method'])) {
+				$serverdata['detection-method'] = $nodeinfo['detection-method'];
+			}
+
 			// Fetch the landing page, possibly it reveals some data
 			if (empty($nodeinfo['network'])) {
-				$curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout]);
+				if ($baseurl == $url) {
+					$basedata = $serverdata;
+				} else {
+					$basedata = ['detection-method' => self::DETECT_MANUAL];
+				}
+
+				$curlResult = Network::curl($baseurl, false, ['timeout' => $xrd_timeout]);
 				if ($curlResult->isSuccess()) {
-					$serverdata = self::analyseRootHeader($curlResult, $serverdata);
+					$basedata = self::analyseRootHeader($curlResult, $basedata);
-					$serverdata = self::analyseRootBody($curlResult, $serverdata, $url);
+					$basedata = self::analyseRootBody($curlResult, $basedata, $baseurl);
 				}
 
 				if (!$curlResult->isSuccess() || empty($curlResult->getBody()) || self::invalidBody($curlResult->getBody())) {
-					DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]);
+					self::setFailure($url);
 					return false;
 				}
+
+				if ($baseurl == $url) {
+					$serverdata = $basedata;
+				} else {
+					// When the base path doesn't seem to contain a social network we try the complete path.
+					// Most detectable system have to be installed in the root directory.
+					// We checked the base to avoid false positives.
+					$curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout]);
+					if ($curlResult->isSuccess()) {
+						$urldata = self::analyseRootHeader($curlResult, $serverdata);
+						$urldata = self::analyseRootBody($curlResult, $urldata, $url);
+
+						$comparebase = $basedata;
+						unset($comparebase['info']);
+						unset($comparebase['site_name']);
+						$compareurl = $urldata;
+						unset($compareurl['info']);
+						unset($compareurl['site_name']);
+
+						// We assume that no one will install the identical system in the root and a subfolder
+						if (!empty(array_diff($comparebase, $compareurl))) {
+							$serverdata = $urldata;
+						}
+					}
+				}
 			}
 
 			if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::ACTIVITYPUB)) {
@@ -246,7 +387,7 @@ class GServer
 			// With this check we don't have to waste time and ressources for dead systems.
 			// Also this hopefully prevents us from receiving abuse messages.
 			if (empty($serverdata['network']) && !self::validHostMeta($url)) {
-				DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]);
+				self::setFailure($url);
 				return false;
 			}
 
@@ -271,6 +412,8 @@ class GServer
 			if (empty($serverdata['network'])) {
 				$serverdata = self::detectGNUSocial($url, $serverdata);
 			}
+
+			$serverdata = array_merge($nodeinfo, $serverdata);
 		} else {
 			$serverdata = $nodeinfo;
 		}
@@ -301,12 +444,7 @@ class GServer
 			$registeredUsers = 1;
 		}
 
-		if ($serverdata['network'] != Protocol::PHANTOM) {
+		if ($serverdata['network'] == Protocol::PHANTOM) {
-			$gcontacts = DBA::count('gcontact', ['server_url' => [$url, $serverdata['nurl']]]);
-			$apcontacts = DBA::count('apcontact', ['baseurl' => [$url, $serverdata['nurl']]]);
-			$contacts = DBA::count('contact', ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]);
-			$serverdata['registered-users'] = max($gcontacts, $apcontacts, $contacts, $registeredUsers);
-		} else {
 			$serverdata['registered-users'] = $registeredUsers;
 			$serverdata = self::detectNetworkViaContacts($url, $serverdata);
 		}
@@ -317,6 +455,7 @@ class GServer
 		if (!DBA::isResult($gserver)) {
 			$serverdata['created'] = DateTimeFormat::utcNow();
 			$ret = DBA::insert('gserver', $serverdata);
+			$id = DBA::lastInsertId();
 		} else {
 			// Don't override the network with 'unknown' when there had been a valid entry before
 			if (($serverdata['network'] == Protocol::PHANTOM) && !empty($gserver['network'])) {
@@ -324,6 +463,21 @@ class GServer
 			}
 
 			$ret = DBA::update('gserver', $serverdata, ['nurl' => $serverdata['nurl']]);
+			$gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => $serverdata['nurl']]);
+			if (DBA::isResult($gserver)) {
+				$id = $gserver['id'];
+			}
+		}
+
+		if (!empty($serverdata['network']) && !empty($id) && ($serverdata['network'] != Protocol::PHANTOM)) {
+			$gcontacts = DBA::count('gcontact', ['gsid' => $id]);
+			$apcontacts = DBA::count('apcontact', ['gsid' => $id]);
+			$contacts = DBA::count('contact', ['uid' => 0, 'gsid' => $id]);
+			$max_users = max($gcontacts, $apcontacts, $contacts, $registeredUsers);
+			if ($max_users > $registeredUsers) {
+				Logger::info('Update registered users', ['id' => $id, 'url' => $serverdata['nurl'], 'registered-users' => $max_users]);
+				DBA::update('gserver', ['registered-users' => $max_users], ['id' => $id]);
+			}
 		}
 
 		if (!empty($serverdata['network']) && in_array($serverdata['network'], [Protocol::DFRN, Protocol::DIASPORA])) {
@@ -353,6 +507,15 @@ class GServer
 			return;
 		}
 
+		// Sanitize incoming data, see https://github.com/friendica/friendica/issues/8565
+		$data['subscribe'] = (bool)$data['subscribe'] ?? false;
+
+		if (!$data['subscribe'] || empty($data['scope']) || !in_array(strtolower($data['scope']), ['all', 'tags'])) {
+			$data['scope'] = '';
+			$data['subscribe'] = false;
+			$data['tags'] = [];
+		}
+
 		$gserver = DBA::selectFirst('gserver', ['id', 'relay-subscribe', 'relay-scope'], ['nurl' => Strings::normaliseLink($server_url)]);
 		if (!DBA::isResult($gserver)) {
 			return;
@@ -425,7 +588,7 @@ class GServer
 			return [];
 		}
 
-		$serverdata = [];
+		$serverdata = ['detection-method' => self::DETECT_STATISTICS_JSON];
 
 		if (!empty($data['version'])) {
 			$serverdata['version'] = $data['version'];
@@ -472,6 +635,10 @@ class GServer
 	 */
 	private static function fetchNodeinfo(string $url, CurlResult $curlResult)
 	{
+		if (!$curlResult->isSuccess()) {
+			return [];
+		}
+
 		$nodeinfo = json_decode($curlResult->getBody(), true);
 
 		if (!is_array($nodeinfo) || empty($nodeinfo['links'])) {
@@ -522,7 +689,6 @@ class GServer
 	private static function parseNodeinfo1(string $nodeinfo_url)
 	{
 		$curlResult = Network::curl($nodeinfo_url);
-
 		if (!$curlResult->isSuccess()) {
 			return [];
 		}
@@ -533,9 +699,8 @@ class GServer
 			return [];
 		}
 
-		$server = [];
+		$server = ['detection-method' => self::DETECT_NODEINFO_1,
-
+			'register_policy' => Register::CLOSED];
-		$server['register_policy'] = Register::CLOSED;
 
 		if (!empty($nodeinfo['openRegistrations'])) {
 			$server['register_policy'] = Register::OPEN;
@@ -610,9 +775,8 @@ class GServer
 			return [];
 		}
 
-		$server = [];
+		$server = ['detection-method' => self::DETECT_NODEINFO_2,
-
+			'register_policy' => Register::CLOSED];
-		$server['register_policy'] = Register::CLOSED;
 
 		if (!empty($nodeinfo['openRegistrations'])) {
 			$server['register_policy'] = Register::OPEN;
@@ -687,6 +851,10 @@ class GServer
 			return $serverdata;
 		}
 
+		if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
+			$serverdata['detection-method'] = self::DETECT_SITEINFO_JSON;
+		}
+
 		if (!empty($data['url'])) {
 			$serverdata['platform'] = strtolower($data['platform']);
 			$serverdata['version'] = $data['version'];
@@ -747,7 +915,7 @@ class GServer
 			return false;
 		}
 
-		$xrd = XML::parseString($curlResult->getBody(), false);
+		$xrd = XML::parseString($curlResult->getBody());
 		if (!is_object($xrd)) {
 			return false;
 		}
@@ -796,13 +964,13 @@ class GServer
 		DBA::close($gcontacts);
 
 		$apcontacts = DBA::select('apcontact', ['url'], ['baseurl' => [$url, $serverdata['nurl']]]);
-		while ($gcontact = DBA::fetch($gcontacts)) {
+		while ($apcontact = DBA::fetch($apcontacts)) {
 			$contacts[Strings::normaliseLink($apcontact['url'])] = $apcontact['url'];
 		}
 		DBA::close($apcontacts);
 
 		$pcontacts = DBA::select('contact', ['url', 'nurl'], ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]);
-		while ($gcontact = DBA::fetch($gcontacts)) {
+		while ($pcontact = DBA::fetch($pcontacts)) {
 			$contacts[$pcontact['nurl']] = $pcontact['url'];
 		}
 		DBA::close($pcontacts);
@@ -896,7 +1064,6 @@ class GServer
 	private static function detectNextcloud(string $url, array $serverdata)
 	{
 		$curlResult = Network::curl($url . '/status.php');
-
 		if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
 			return $serverdata;
 		}
@@ -910,6 +1077,10 @@ class GServer
 			$serverdata['platform'] = 'nextcloud';
 			$serverdata['version'] = $data['version'];
 			$serverdata['network'] = Protocol::ACTIVITYPUB;
+
+			if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
+				$serverdata['detection-method'] = self::DETECT_STATUS_PHP;
+			}
 		}
 
 		return $serverdata;
@@ -926,7 +1097,6 @@ class GServer
 	private static function detectMastodonAlikes(string $url, array $serverdata)
 	{
 		$curlResult = Network::curl($url . '/api/v1/instance');
-
 		if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) {
 			return $serverdata;
 		}
@@ -936,6 +1106,10 @@ class GServer
 			return $serverdata;
 		}
 
+		if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
+			$serverdata['detection-method'] = self::DETECT_MASTODON_API;
+		}
+
 		if (!empty($data['version'])) {
 			$serverdata['platform'] = 'mastodon';
 			$serverdata['version'] = $data['version'] ?? '';
@@ -993,7 +1167,7 @@ class GServer
 		}
 
 		$data = json_decode($curlResult->getBody(), true);
-		if (empty($data)) {
+		if (empty($data) || empty($data['site'])) {
 			return $serverdata;
 		}
 
@@ -1041,11 +1215,16 @@ class GServer
 		}
 
 		if (!$closed && !$private and $inviteonly) {
-			$register_policy = Register::APPROVE;
+			$serverdata['register_policy'] = Register::APPROVE;
 		} elseif (!$closed && !$private) {
-			$register_policy = Register::OPEN;
+			$serverdata['register_policy'] = Register::OPEN;
 		} else {
-			$register_policy = Register::CLOSED;
+			$serverdata['register_policy'] = Register::CLOSED;
+		}
+
+		if (!empty($serverdata['network']) && in_array($serverdata['detection-method'],
+			[self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
+			$serverdata['detection-method'] = self::DETECT_CONFIG_JSON;
 		}
 
 		return $serverdata;
@@ -1089,6 +1268,11 @@ class GServer
 			$serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']);
 			$serverdata['version'] = trim($serverdata['version'], '"');
 			$serverdata['network'] = Protocol::OSTATUS;
+
+			if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
+				$serverdata['detection-method'] = self::DETECT_GNUSOCIAL;
+			}
+	
 			return $serverdata;
 		}
 
@@ -1110,6 +1294,10 @@ class GServer
 				$serverdata['platform'] = 'statusnet';
 				$serverdata['network'] = Protocol::OSTATUS;
 			}
+
+			if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {
+				$serverdata['detection-method'] = self::DETECT_STATUSNET;
+			}
 		}
 
 		return $serverdata;
@@ -1128,6 +1316,11 @@ class GServer
 		$curlResult = Network::curl($url . '/friendica/json');
 		if (!$curlResult->isSuccess()) {
 			$curlResult = Network::curl($url . '/friendika/json');
+			$friendika = true;
+			$platform = 'Friendika';
+		} else {
+			$friendika = false;
+			$platform = 'Friendica';
 		}
 
 		if (!$curlResult->isSuccess()) {
@@ -1139,6 +1332,10 @@ class GServer
 			return $serverdata;
 		}
 
+		if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) {			
+			$serverdata['detection-method'] = $friendika ? self::DETECT_FRIENDIKA : self::DETECT_FRIENDICA;
+		}
+
 		$serverdata['network'] = Protocol::DFRN;
 		$serverdata['version'] = $data['version'];
 
@@ -1174,7 +1371,7 @@ class GServer
 				break;
 		}
 
-		$serverdata['platform'] = strtolower($data['platform'] ?? '');
+		$serverdata['platform'] = strtolower($data['platform'] ?? $platform);
 
 		return $serverdata;
 	}
@@ -1222,7 +1419,8 @@ class GServer
 				$serverdata['info'] = $attr['content'];
 			}
 
-			if ($attr['name'] == 'application-name') {
+			if (in_array($attr['name'], ['application-name', 'al:android:app_name', 'al:ios:app_name',
+				'twitter:app:name:googleplay', 'twitter:app:name:iphone', 'twitter:app:name:ipad'])) {
 				$serverdata['platform'] = strtolower($attr['content']);
  				if (in_array($attr['content'], ['Misskey', 'Write.as'])) {
 					$serverdata['network'] = Protocol::ACTIVITYPUB;
@@ -1243,6 +1441,10 @@ class GServer
 						} else {
 							$serverdata['network'] = Protocol::FEED;
 						}
+
+						if ($serverdata['detection-method'] == self::DETECT_MANUAL) {
+							$serverdata['detection-method'] = self::DETECT_BODY;
+						}
 					}
 					if (in_array($version_part[0], ['Friendika', 'Friendica'])) {
 						$serverdata['platform'] = strtolower($version_part[0]);
@@ -1298,6 +1500,10 @@ class GServer
 			}
 		}
 
+		if (!empty($serverdata['network']) && ($serverdata['detection-method'] == self::DETECT_MANUAL)) {
+			$serverdata['detection-method'] = self::DETECT_BODY;
+		}
+
 		return $serverdata;
 	}
 
@@ -1313,16 +1519,23 @@ class GServer
 	{
 		if ($curlResult->getHeader('server') == 'Mastodon') {
 			$serverdata['platform'] = 'mastodon';
-			$serverdata['network'] = $network = Protocol::ACTIVITYPUB;
+			$serverdata['network'] = Protocol::ACTIVITYPUB;
 		} elseif ($curlResult->inHeader('x-diaspora-version')) {
 			$serverdata['platform'] = 'diaspora';
-			$serverdata['network'] = $network = Protocol::DIASPORA;
+			$serverdata['network'] = Protocol::DIASPORA;
 			$serverdata['version'] = $curlResult->getHeader('x-diaspora-version');
 		} elseif ($curlResult->inHeader('x-friendica-version')) {
 			$serverdata['platform'] = 'friendica';
-			$serverdata['network'] = $network = Protocol::DFRN;
+			$serverdata['network'] = Protocol::DFRN;
 			$serverdata['version'] = $curlResult->getHeader('x-friendica-version');
+		} else {
+			return $serverdata;
 		}
+
+		if ($serverdata['detection-method'] == self::DETECT_MANUAL) {
+			$serverdata['detection-method'] = self::DETECT_HEADER;
+		}
+
 		return $serverdata;
 	}
 
@@ -1414,14 +1627,18 @@ class GServer
 		}
 
 		// Discover federated servers
-		$curlResult = Network::fetchUrl("http://the-federation.info/pods.json");
+		$protocols = ['activitypub', 'diaspora', 'dfrn', 'ostatus'];
-
+		foreach ($protocols as $protocol) {
+			$query = '{nodes(protocol:"' . $protocol . '"){host}}';
+			$curlResult = Network::fetchUrl('https://the-federation.info/graphql?query=' . urlencode($query));
 			if (!empty($curlResult)) {
-			$servers = json_decode($curlResult, true);
+				$data = json_decode($curlResult, true);
-
+				if (!empty($data['data']['nodes'])) {
-			if (!empty($servers['pods'])) {
+					foreach ($data['data']['nodes'] as $server) {
-				foreach ($servers['pods'] as $server) {
+						// Using "only_nodeinfo" since servers that are listed on that page should always have it.
-					Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host']);
+						echo $server['host']."\n";
+						Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host'], true);
+					}
 				}
 			}
 		}
@@ -1433,7 +1650,6 @@ class GServer
 			$api = 'https://instances.social/api/1.0/instances/list?count=0';
 			$header = ['Authorization: Bearer '.$accesstoken];
 			$curlResult = Network::curl($api, false, ['headers' => $header]);
-
 			if ($curlResult->isSuccess()) {
 				$servers = json_decode($curlResult->getBody(), true);