Bluesky: don't remove hashtags upon posting

Reviewed-on: friendica/friendica-addons#1401

bluesky/bluesky.php

@@ -6,8 +6,12 @@
  * Author: Michael Vogel <https://pirati.ca/profile/heluecht>
  *
  * @todo
+ * Currently technical issues in the core:
  * - Outgoing mentions
  *
+ * At some point in time:
+ * - Sending Quote shares https://atproto.com/lexicons/app-bsky-embed#appbskyembedrecord and https://atproto.com/lexicons/app-bsky-embed#appbskyembedrecordwithmedia
+ *
  * Possibly not possible:
  * - only fetch new posts
  *
@@ -17,10 +21,6 @@
  *
  * Possibly interesting:
  * - https://atproto.com/lexicons/com-atproto-label#comatprotolabelsubscribelabels
- * - https://atproto.com/lexicons/com-atproto-moderation#comatprotomoderationcreatereport
- * - https://atproto.com/lexicons/com-atproto-repo#comatprotorepoputrecord
- * - https://atproto.com/lexicons/app-bsky-embed#appbskyembedrecordwithmedia
- * - https://atproto.com/lexicons/app-bsky-feed#appbskyfeedgetpostthread
  */
 
 use Friendica\Content\Text\BBCode;
@@ -118,7 +118,7 @@ function bluesky_probe_detect(array &$hookData)
 		return;
 	}
 
-	$data = bluesky_get($pconfig['uid'], '/xrpc/app.bsky.actor.getProfile?actor=' . $did, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $token]]]);
+	$data = bluesky_xrpc_get($pconfig['uid'], 'app.bsky.actor.getProfile', ['actor' => $did]);
 	if (empty($data)) {
 		return;
 	}
@@ -163,7 +163,7 @@ function bluesky_item_by_link(array &$hookData)
 
 	$uri = 'at://' . $did . '/app.bsky.feed.post/' . $matches[2];
 
-	$uri = bluesky_fetch_missing_post($uri, $hookData['uid'], 0);
+	$uri = bluesky_fetch_missing_post($uri, $hookData['uid'], 0, 0);
 	Logger::debug('Got post', ['profile' => $matches[1], 'cid' => $matches[2], 'result' => $uri]);
 	if (!empty($uri)) {
 		$item = Post::selectFirst(['id'], ['uri' => $uri, 'uid' => $hookData['uid']]);
@@ -212,7 +212,7 @@ function bluesky_follow(array &$hook_data)
 		'record'     => $record
 	];
 
-	$activity = bluesky_post($hook_data['uid'], '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]);
+	$activity = bluesky_xrpc_post($hook_data['uid'], 'com.atproto.repo.createRecord', $post);
 	if (!empty($activity->uri)) {
 		$hook_data['contact'] = $contact;
 		Logger::debug('Successfully start following', ['url' => $contact['url'], 'uri' => $activity->uri]);
@@ -230,7 +230,7 @@ function bluesky_unfollow(array &$hook_data)
 		return;
 	}
 
-	$data = bluesky_get($hook_data['uid'], '/xrpc/app.bsky.actor.getProfile?actor=' . $hook_data['contact']['url'], HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $token]]]);
+	$data = bluesky_xrpc_get($hook_data['uid'], 'app.bsky.actor.getProfile', ['actor' => $hook_data['contact']['url']]);
 	if (empty($data->viewer) || empty($data->viewer->following)) {
 		return;
 	}
@@ -265,7 +265,7 @@ function bluesky_block(array &$hook_data)
 		'record'     => $record
 	];
 
-	$activity = bluesky_post($hook_data['uid'], '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]);
+	$activity = bluesky_xrpc_post($hook_data['uid'], 'com.atproto.repo.createRecord', $post);
 	if (!empty($activity->uri)) {
 		$cdata = Contact::getPublicAndUserContactID($hook_data['contact']['id'], $hook_data['uid']);
 		if (!empty($cdata['user'])) {
@@ -286,7 +286,7 @@ function bluesky_unblock(array &$hook_data)
 		return;
 	}
 
-	$data = bluesky_get($hook_data['uid'], '/xrpc/app.bsky.actor.getProfile?actor=' . $hook_data['contact']['url'], HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . $token]]]);
+	$data = bluesky_xrpc_get($hook_data['uid'], 'app.bsky.actor.getProfile', ['actor' => $hook_data['contact']['url']]);
 	if (empty($data->viewer) || empty($data->viewer->blocking)) {
 		return;
 	}
@@ -420,22 +420,17 @@ function bluesky_cron()
 			}
 		}
 
-		Logger::notice('importing timeline - start', ['user' => $pconfig['uid']]);
-		bluesky_fetch_timeline($pconfig['uid']);
-		Logger::notice('importing timeline - done', ['user' => $pconfig['uid']]);
+		// Refresh the token now, so that it doesn't need to be refreshed in parallel by the following workers
+		bluesky_get_token($pconfig['uid']);
 
-		Logger::notice('importing notifications - start', ['user' => $pconfig['uid']]);
-		bluesky_fetch_notifications($pconfig['uid']);
-		Logger::notice('importing notifications - done', ['user' => $pconfig['uid']]);
+		Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_timeline.php', $pconfig['uid']);
+		Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_notifications.php', $pconfig['uid']);
 
 		if (DI::pConfig()->get($pconfig['uid'], 'bluesky', 'import_feeds')) {
-			Logger::notice('importing feeds - start', ['user' => $pconfig['uid']]);
 			$feeds = bluesky_get_feeds($pconfig['uid']);
 			foreach ($feeds as $feed) {
-				Logger::debug('Importing feed', ['user' => $pconfig['uid'], 'feed' => $feed]);
-				bluesky_fetch_feed($pconfig['uid'], $feed);
+				Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/bluesky/bluesky_feed.php', $pconfig['uid'], $feed);
 			}
-			Logger::notice('importing feeds - done', ['user' => $pconfig['uid']]);
 		}
 	}
 
@@ -595,7 +590,7 @@ function bluesky_create_activity(array $item, stdClass $parent = null)
 		];
 	}
 
-	$activity = bluesky_post($uid, '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]);
+	$activity = bluesky_xrpc_post($uid, 'com.atproto.repo.createRecord', $post);
 	if (empty($activity)) {
 		return;
 	}
@@ -614,12 +609,13 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren
 	}
 
 	$did  = DI::pConfig()->get($uid, 'bluesky', 'did');
-	$urls = bluesky_get_urls($item['body']);
+	$urls = bluesky_get_urls(Post\Media::removeFromBody($item['body']));
+	$item['body'] = $urls['body'];
 
 	$msg = Plaintext::getPost($item, 300, false, BBCode::BLUESKY);
 	foreach ($msg['parts'] as $key => $part) {
 
-		$facets = bluesky_get_facets($part, $urls);
+		$facets = bluesky_get_facets($part, $urls['urls']);
 
 		$record = [
 			'text'      => $facets['body'],
@@ -645,7 +641,7 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren
 			'record'     => $record
 		];
 
-		$parent = bluesky_post($uid, '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]);
+		$parent = bluesky_xrpc_post($uid, 'com.atproto.repo.createRecord', $post);
 		if (empty($parent)) {
 			return;
 		}
@@ -663,25 +659,51 @@ function bluesky_create_post(array $item, stdClass $root = null, stdClass $paren
 
 function bluesky_get_urls(string $body): array
 {
-	// Remove all hashtags and mentions
-	$body = preg_replace("/([#@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '', $body);
+	// Remove all hashtag and mention links
+	$body = preg_replace("/([#@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '$1$3', $body);
 
 	$urls = [];
 
 	// Search for pure links
-	if (preg_match_all("/\[url\](https?:.*?)\[\/url\]/ism", $body, $matches)) {
-		foreach ($matches[1] as $url) {
-			$urls[] = $url;
+	if (preg_match_all("/\[url\](https?:.*?)\[\/url\]/ism", $body, $matches, PREG_SET_ORDER)) {
+		foreach ($matches as $match) {
+			$text = Strings::getStyledURL($match[1]);
+			$hash = bluesky_get_hash_for_url($match[0], mb_strlen($text));
+			$urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash];
+			$body = str_replace($match[0], $hash, $body);
 		}
 	}
 
 	// Search for links with descriptions
-	if (preg_match_all("/\[url\=(https?:.*?)\].*?\[\/url\]/ism", $body, $matches)) {
-		foreach ($matches[1] as $url) {
-			$urls[] = $url;
+	if (preg_match_all("/\[url\=(https?:.*?)\](.*?)\[\/url\]/ism", $body, $matches, PREG_SET_ORDER)) {
+		foreach ($matches as $match) {
+			if ($match[1] == $match[2]) {
+				$text = Strings::getStyledURL($match[1]);
+			} else {
+				$text = $match[2];
+			}
+			if (mb_strlen($text) < 100) {
+				$hash = bluesky_get_hash_for_url($match[0], mb_strlen($text));
+				$urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash];
+				$body = str_replace($match[0], $hash, $body);
+			} else {
+				$text = Strings::getStyledURL($match[1]);
+				$hash = bluesky_get_hash_for_url($match[0], mb_strlen($text));
+				$urls[] = ['url' => $match[1], 'text' => $text, 'hash' => $hash];
+				$body = str_replace($match[0], $text . ' ' . $hash, $body);
+			}
 		}
 	}
-	return $urls;
+
+	return ['body' => $body, 'urls' => $urls];
+}
+
+function bluesky_get_hash_for_url(string $text, int $linklength): string
+{
+	if ($linklength <= 10) {
+		return '|' . hash('crc32', $text) . '|';
+	}
+	return substr('|' . hash('crc32', $text) . base64_encode($text), 0, $linklength - 2) . '|';
 }
 
 function bluesky_get_facets(string $body, array $urls): array
@@ -689,7 +711,7 @@ function bluesky_get_facets(string $body, array $urls): array
 	$facets = [];
 
 	foreach ($urls as $url) {
-		$pos = strpos($body, $url);
+		$pos = strpos($body, $url['hash']);
 		if ($pos === false) {
 			continue;
 		}
@@ -698,16 +720,16 @@ function bluesky_get_facets(string $body, array $urls): array
 		} else {
 			$prefix = '';
 		}
-		$linktext = Strings::getStyledURL($url);
-		$body = $prefix . $linktext . substr($body, $pos + strlen($url));
+
+		$body = $prefix . $url['text'] . substr($body, $pos + strlen($url['hash']));
 
 		$facet = new stdClass;
 		$facet->index = new stdClass;
-		$facet->index->byteEnd   = $pos + strlen($linktext);
+		$facet->index->byteEnd   = $pos + strlen($url['text']);
 		$facet->index->byteStart = $pos;
 
 		$feature = new stdClass;
-		$feature->uri = $url;
+		$feature->uri = $url['url'];
 		$type = '$type';
 		$feature->$type = 'app.bsky.richtext.facet#link';
 
@@ -767,19 +789,18 @@ function bluesky_upload_blob(int $uid, array $photo): ?stdClass
 
 function bluesky_delete_post(string $uri, int $uid)
 {
-	$token = bluesky_get_token($uid);
 	$parts = bluesky_get_uri_parts($uri);
 	if (empty($parts)) {
 		Logger::debug('No uri delected', ['uri' => $uri]);
 		return;
 	}
-	bluesky_post($uid, '/xrpc/com.atproto.repo.deleteRecord', json_encode($parts), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]);
+	bluesky_xrpc_post($uid, 'com.atproto.repo.deleteRecord', $parts);
 	Logger::debug('Deleted', ['parts' => $parts]);
 }
 
 function bluesky_fetch_timeline(int $uid)
 {
-	$data = bluesky_get($uid, '/xrpc/app.bsky.feed.getTimeline', HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]);
+	$data = bluesky_xrpc_get($uid, 'app.bsky.feed.getTimeline');
 	if (empty($data)) {
 		return;
 	}
@@ -789,7 +810,7 @@ function bluesky_fetch_timeline(int $uid)
 	}
 
 	foreach (array_reverse($data->feed) as $entry) {
-		bluesky_process_post($entry->post, $uid, Item::PR_NONE);
+		bluesky_process_post($entry->post, $uid, Item::PR_NONE, 0);
 		if (!empty($entry->reason)) {
 			bluesky_process_reason($entry->reason, bluesky_get_uri($entry->post), $uid);
 		}
@@ -830,6 +851,7 @@ function bluesky_process_reason(stdClass $reason, string $uri, int $uid)
 		return;
 	}
 
+	$item['guid']         = Item::guidFromUri($item['uri'], $contact['alias']);
 	$item['owner-name']   = $item['author-name'];
 	$item['owner-link']   = $item['author-link'];
 	$item['owner-avatar'] = $item['author-avatar'];
@@ -841,11 +863,11 @@ function bluesky_process_reason(stdClass $reason, string $uri, int $uid)
 
 function bluesky_fetch_notifications(int $uid)
 {
-	$result = bluesky_get($uid, '/xrpc/app.bsky.notification.listNotifications', HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]);
-	if (empty($result->notifications)) {
+	$data = bluesky_xrpc_get($uid, 'app.bsky.notification.listNotifications');
+	if (empty($data->notifications)) {
 		return;
 	}
-	foreach ($result->notifications as $notification) {
+	foreach ($data->notifications as $notification) {
 		$uri = bluesky_get_uri($notification);
 		if (Post::exists(['uri' => $uri, 'uid' => $uid]) || Post::exists(['extid' => $uri, 'uid' => $uid])) {
 			Logger::debug('Notification already processed', ['uid' => $uid, 'reason' => $notification->reason, 'uri' => $uri, 'indexedAt' => $notification->indexedAt]);
@@ -858,41 +880,51 @@ function bluesky_fetch_notifications(int $uid)
 				$item['gravity'] = Item::GRAVITY_ACTIVITY;
 				$item['body'] = $item['verb'] = Activity::LIKE;
 				$item['thr-parent'] = bluesky_get_uri($notification->record->subject);
-				$result = Item::insert($item);
-				Logger::debug('Got like', ['uid' => $uid, 'result' => $result]);
-				break;
+				$item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id'], 0);
+				if (!empty($item['thr-parent'])) {
+					$data = Item::insert($item);
+					Logger::debug('Got like', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
+				} else {
+					Logger::info('Thread parent not found', ['uid' => $uid, 'parent' => $$item['thr-parent'], 'uri' => $uri]);
+				}
+			break;
 
 			case 'repost':
 				$item = bluesky_get_header($notification, $uri, $uid, $uid);
 				$item['gravity'] = Item::GRAVITY_ACTIVITY;
 				$item['body'] = $item['verb'] = Activity::ANNOUNCE;
 				$item['thr-parent'] = bluesky_get_uri($notification->record->subject);
-				$result = Item::insert($item);
-				Logger::debug('Got repost', ['uid' => $uid, 'result' => $result]);
-				break;
+				$item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id'], 0);
+				if (!empty($item['thr-parent'])) {
+					$data = Item::insert($item);
+					Logger::debug('Got repost', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
+				} else {
+					Logger::info('Thread parent not found', ['uid' => $uid, 'parent' => $$item['thr-parent'], 'uri' => $uri]);
+				}
+			break;
 
 			case 'follow':
 				$contact = bluesky_get_contact($notification->author, $uid, $uid);
-				Logger::debug('New follower', ['uid' => $uid, 'nick' => $contact['nick']]);
+				Logger::debug('New follower', ['uid' => $uid, 'nick' => $contact['nick'], 'uri' => $uri]);
 				break;
 
 			case 'mention':
-				$result = bluesky_process_post($notification, $uid, Item::PR_PUSHED);
-				Logger::debug('Got mention', ['uid' => $uid, 'result' => $result]);
+				$data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0);
+				Logger::debug('Got mention', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
 				break;
 
 			case 'reply':
-				$result = bluesky_process_post($notification, $uid, Item::PR_PUSHED);
-				Logger::debug('Got reply', ['uid' => $uid, 'result' => $result]);
+				$data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0);
+				Logger::debug('Got reply', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
 				break;
 
 			case 'quote':
-				$result = bluesky_process_post($notification, $uid, Item::PR_PUSHED);
-				Logger::debug('Got quote', ['uid' => $uid, 'result' => $result]);
+				$data = bluesky_process_post($notification, $uid, Item::PR_PUSHED, 0);
+				Logger::debug('Got quote', ['uid' => $uid, 'result' => $data, 'uri' => $uri]);
 				break;
 
 			default:
-				Logger::notice('Unhandled reason', ['reason' => $notification->reason]);
+				Logger::notice('Unhandled reason', ['reason' => $notification->reason, 'uri' => $uri]);
 				break;
 		}
 	}
@@ -900,7 +932,7 @@ function bluesky_fetch_notifications(int $uid)
 
 function bluesky_fetch_feed(int $uid, string $feed)
 {
-	$data = bluesky_get($uid, '/xrpc/app.bsky.feed.getFeed?feed=' . $feed, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]);
+	$data = bluesky_xrpc_get($uid, 'app.bsky.feed.getFeed', ['feed' => $feed]);
 	if (empty($data)) {
 		return;
 	}
@@ -914,14 +946,14 @@ function bluesky_fetch_feed(int $uid, string $feed)
 			Logger::debug('Unwanted language detected', ['text' => $entry->post->record->text]);
 			continue;
 		}
-		bluesky_process_post($entry->post, $uid, Item::PR_TAG);
+		bluesky_process_post($entry->post, $uid, Item::PR_TAG, 0);
 		if (!empty($entry->reason)) {
 			bluesky_process_reason($entry->reason, bluesky_get_uri($entry->post), $uid);
 		}
 	}
 }
 
-function bluesky_process_post(stdClass $post, int $uid, int $post_reason): int
+function bluesky_process_post(stdClass $post, int $uid, int $post_reason, $level): int
 {
 	$uri = bluesky_get_uri($post);
 
@@ -929,14 +961,16 @@ function bluesky_process_post(stdClass $post, int $uid, int $post_reason): int
 		return 0;
 	}
 
-	Logger::debug('Importing post', ['uid' => $uid, 'indexedAt' => $post->indexedAt, 'uri' => $post->uri, 'cid' => $post->cid]);
+	Logger::debug('Importing post', ['uid' => $uid, 'indexedAt' => $post->indexedAt, 'uri' => $post->uri, 'cid' => $post->cid, 'root' => $post->record->reply->root ?? '']);
 
 	$item = bluesky_get_header($post, $uri, $uid, $uid);
-
-	$item = bluesky_get_content($item, $post->record, $uid);
+	$item = bluesky_get_content($item, $post->record, $uri, $uid, $level);
+	if (empty($item)) {
+		return 0;
+	}
 
 	if (!empty($post->embed)) {
-		$item = bluesky_add_media($post->embed, $item, $uid);
+		$item = bluesky_add_media($post->embed, $item, $uid, $level);
 	}
 
 	if (empty($item['post-reason'])) {
@@ -980,21 +1014,36 @@ function bluesky_get_header(stdClass $post, string $uri, int $uid, int $fetch_ui
 	return $item;
 }
 
-function bluesky_get_content(array $item, stdClass $record, int $uid): array
+function bluesky_get_content(array $item, stdClass $record, string $uri, int $uid, int $level): array
 {
-	if (!empty($record->reply)) {
-		$item['parent-uri'] = bluesky_get_uri($record->reply->root);
-		$item['parent-uri'] = bluesky_fetch_missing_post($item['parent-uri'], $uid, $item['contact-id']);
-		$item['thr-parent'] = bluesky_get_uri($record->reply->parent);
-		$item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id']);
+	if (empty($item)) {
+		return [];
 	}
 
-	$item['body']    = bluesky_get_text($record, $uid);
+	if (!empty($record->reply)) {
+		$item['parent-uri'] = bluesky_get_uri($record->reply->root);
+		if ($item['parent-uri'] != $uri) {
+			$item['parent-uri'] = bluesky_fetch_missing_post($item['parent-uri'], $uid, $item['contact-id'], $level);
+			if (empty($item['parent-uri'])) {
+				return [];
+			}
+		}
+
+		$item['thr-parent'] = bluesky_get_uri($record->reply->parent);
+		if (!in_array($item['thr-parent'], [$uri, $item['parent-uri']])) {
+			$item['thr-parent'] = bluesky_fetch_missing_post($item['thr-parent'], $uid, $item['contact-id'], $level, $item['parent-uri']);
+			if (empty($item['thr-parent'])) {
+				return [];
+			}
+		}
+	}
+
+	$item['body']    = bluesky_get_text($record);
 	$item['created'] = DateTimeFormat::utc($record->createdAt, DateTimeFormat::MYSQL);
 	return $item;
 }
 
-function bluesky_get_text(stdClass $record, int $uid): string
+function bluesky_get_text(stdClass $record): string
 {
 	$text = $record->text;
 
@@ -1023,7 +1072,7 @@ function bluesky_get_text(stdClass $record, int $uid): string
 					break;
 
 				case 'app.bsky.richtext.facet#mention':
-					$contact = Contact::selectFirst(['id'], ['nurl' => $feature->did, 'uid' => [0, $uid]]);
+					$contact = Contact::getByURL($feature->did, null, ['id']);
 					if (!empty($contact['id'])) {
 						$url = DI::baseUrl() . '/contact/' . $contact['id'];
 						if (substr($linktext, 0, 1) == '@') {
@@ -1045,7 +1094,7 @@ function bluesky_get_text(stdClass $record, int $uid): string
 	return $text;
 }
 
-function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid): array
+function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid, int $level): array
 {
 	$type = '$type';
 	switch ($embed->$type) {
@@ -1078,12 +1127,11 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid): array
 			$shared = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => $item['uid']]);
 			if (empty($shared)) {
 				$shared = bluesky_get_header($embed->record, $uri, 0, $fetch_uid);
+				$shared = bluesky_get_content($shared, $embed->record->value, $uri, $item['uid'], $level);
 				if (!empty($shared)) {
-					$shared = bluesky_get_content($shared, $embed->record->value, $item['uid']);
-
 					if (!empty($embed->record->embeds)) {
 						foreach ($embed->record->embeds as $single) {
-							$shared = bluesky_add_media($single, $shared, $fetch_uid);
+							$shared = bluesky_add_media($single, $shared, $fetch_uid, $level);
 						}
 					}
 					$id = Item::insert($shared);
@@ -1100,17 +1148,16 @@ function bluesky_add_media(stdClass $embed, array $item, int $fetch_uid): array
 			$shared = Post::selectFirst(['uri-id'], ['uri' => $uri, 'uid' => $item['uid']]);
 			if (empty($shared)) {
 				$shared = bluesky_get_header($embed->record->record, $uri, 0, $fetch_uid);
+				$shared = bluesky_get_content($shared, $embed->record->record->value, $uri, $item['uid'], $level);
 				if (!empty($shared)) {
-					$shared = bluesky_get_content($shared, $embed->record->record->value, $item['uid']);
-
 					if (!empty($embed->record->embeds)) {
 						foreach ($embed->record->record->embeds as $single) {
-							$shared = bluesky_add_media($single, $shared, $fetch_uid);
+							$shared = bluesky_add_media($single, $shared, $fetch_uid, $level);
 						}
 					}
 
 					if (!empty($embed->media)) {
-						bluesky_add_media($embed->media, $item, $fetch_uid);
+						bluesky_add_media($embed->media, $item, $fetch_uid, $level);
 					}
 
 					$id = Item::insert($shared);
@@ -1177,7 +1224,42 @@ function bluesky_get_uri_parts(string $uri): ?stdClass
 	return $class;
 }
 
-function bluesky_fetch_missing_post(string $uri, int $uid, int $causer): string
+function bluesky_fetch_missing_post(string $uri, int $uid, int $causer, int $level, string $fallback = ''): string
+{
+	$fetched_uri = bluesky_fetch_post($uri, $uid);
+	if (!empty($fetched_uri)) {
+		return $fetched_uri;
+	}
+
+	if (++$level > 100) {
+		Logger::info('Recursion level too deep', ['level' => $level, 'uid' => $uid, 'uri' => $uri, 'fallback' => $fallback]);
+		// When the level is too deep we will fallback to the parent uri.
+		// Allthough the threading won't be correct, we at least had stored all posts and won't try again 
+		return $fallback;
+	}
+
+	$class = bluesky_get_uri_class($uri);
+	$fetch_uri = $class->uri;
+
+	Logger::debug('Fetch missing post', ['level' => $level, 'uid' => $uid, 'uri' => $uri]);
+	$data = bluesky_xrpc_get($uid, 'app.bsky.feed.getPostThread', ['uri' => $fetch_uri]);
+	if (empty($data)) {
+		Logger::info('Thread was not fetched', ['level' => $level, 'uid' => $uid, 'uri' => $uri, 'fallback' => $fallback]);
+		return $fallback;
+	}
+	
+	Logger::debug('Reply count', ['replies' => $data->thread->post->replyCount, 'level' => $level, 'uid' => $uid, 'uri' => $uri]); 
+
+	if ($causer != 0) {
+		$cdata = Contact::getPublicAndUserContactID($causer, $uid);
+	} else {
+		$cdata = [];
+	}
+
+	return bluesky_process_thread($data->thread, $uid, $cdata, $level);
+}
+
+function bluesky_fetch_post(string $uri, int $uid): string
 {
 	if (Post::exists(['uri' => $uri, 'uid' => [$uid, 0]])) {
 		Logger::debug('Post exists', ['uri' => $uri]);
@@ -1186,38 +1268,48 @@ function bluesky_fetch_missing_post(string $uri, int $uid, int $causer): string
 
 	$reply = Post::selectFirst(['uri'], ['extid' => $uri, 'uid' => [$uid, 0]]);
 	if (!empty($reply['uri'])) {
+		Logger::debug('Post with extid exists', ['uri' => $uri]);
 		return $reply['uri'];
 	}
+	return '';
+}
 
-	Logger::debug('Fetch missing post', ['uri' => $uri]);
-	$class = bluesky_get_uri_class($uri);
-	$fetch_uri = $class->uri;
+function bluesky_process_thread(stdClass $thread, int $uid, array $cdata, int $level): string
+{
+	$uri = bluesky_get_uri($thread->post);
+	$fetched_uri = bluesky_fetch_post($uri, $uid);
+	if (empty($fetched_uri)) {
+		Logger::debug('Process missing post', ['uri' => $uri]);
+		$item = bluesky_get_header($thread->post, $uri, $uid, $uid);
+		$item = bluesky_get_content($item, $thread->post->record, $uri, $uid, $level);
+		if (!empty($item)) {
+			$item['post-reason'] = Item::PR_FETCHED;
 
-	$data = bluesky_get($uid, '/xrpc/app.bsky.feed.getPosts?uris=' . urlencode($fetch_uri), HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]);
-	if (empty($data)) {
-		return '';
+			if (!empty($cdata['public'])) {
+				$item['causer-id']   = $cdata['public'];
+			}
+
+			if (!empty($thread->post->embed)) {
+				$item = bluesky_add_media($thread->post->embed, $item, $uid, $level);
+			}
+			$id = Item::insert($item);
+			if (!$id) {
+				Logger::info('Item has not not been stored', ['uri' => $uri]);
+				return '';
+			}
+			Logger::debug('Stored item', ['id' => $id, 'uri' => $uri]);
+		} else {
+			Logger::info('Post has not not been fetched', ['uri' => $uri]);
+			return '';
+		}
+	} else {
+		Logger::debug('Post exists', ['uri' => $uri]);
+		$uri = $fetched_uri;
 	}
 
-	if ($causer != 0) {
-		$cdata = Contact::getPublicAndUserContactID($causer, $uid);
-	}
-
-	foreach ($data->posts as $post) {
-		$uri = bluesky_get_uri($post);
-		$item = bluesky_get_header($post, $uri, $uid, $uid);
-		$item = bluesky_get_content($item, $post->record, $uid);
-
-		$item['post-reason'] = Item::PR_FETCHED;
-
-		if (!empty($cdata['public'])) {
-			$item['causer-id']   = $cdata['public'];
-		}
-
-		if (!empty($post->embed)) {
-			$item = bluesky_add_media($post->embed, $item, $uid);
-		}
-		$id = Item::insert($item);
-		Logger::debug('Stored item', ['id' => $id, 'uri' => $uri]);
+	foreach ($thread->replies ?? [] as $reply) {
+		$reply_uri = bluesky_process_thread($reply, $uid, $cdata, $level);
+		Logger::debug('Reply has been processed', ['uri' => $uri, 'reply' => $reply_uri]);
 	}
 
 	return $uri;
@@ -1295,7 +1387,7 @@ function bluesky_get_contact_fields(stdClass $author, int $uid, bool $update): a
 		return $fields;
 	}
 
-	$data = bluesky_get($uid, '/xrpc/app.bsky.actor.getProfile?actor=' . $author->did, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]);
+	$data = bluesky_xrpc_get($uid, 'app.bsky.actor.getProfile', ['actor' => $author->did]);
 	if (empty($data)) {
 		Logger::debug('Error fetching contact fields', ['uid' => $uid, 'url' => $fields['url']]);
 		return $fields;
@@ -1347,7 +1439,7 @@ function bluesky_get_preferences(int $uid): stdClass
 		return $data;
 	}
 
-	$data = bluesky_get($uid, '/xrpc/app.bsky.actor.getPreferences', HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]);
+	$data = bluesky_xrpc_get($uid, 'app.bsky.actor.getPreferences');
 
 	DI::cache()->set($cachekey, $data, Duration::HOUR);
 	return $data;
@@ -1355,7 +1447,7 @@ function bluesky_get_preferences(int $uid): stdClass
 
 function bluesky_get_did(int $uid, string $handle): string
 {
-	$data = bluesky_get($uid, '/xrpc/com.atproto.identity.resolveHandle?handle=' . $handle);
+	$data = bluesky_get($uid, '/xrpc/com.atproto.identity.resolveHandle?handle=' . urlencode($handle));
 	if (empty($data)) {
 		return '';
 	}
@@ -1409,6 +1501,11 @@ function bluesky_create_token(int $uid, string $password): string
 	return $data->accessJwt;
 }
 
+function bluesky_xrpc_post(int $uid, string $url, $parameters): ?stdClass
+{
+	return bluesky_post($uid, '/xrpc/' . $url, json_encode($parameters),  ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . bluesky_get_token($uid)]]);
+}
+
 function bluesky_post(int $uid, string $url, string $params, array $headers): ?stdClass
 {
 	try {
@@ -1426,6 +1523,15 @@ function bluesky_post(int $uid, string $url, string $params, array $headers): ?s
 	return json_decode($curlResult->getBody());
 }
 
+function bluesky_xrpc_get(int $uid, string $url, array $parameters = []): ?stdClass
+{
+	if (!empty($parameters)) {
+		$url .= '?' . http_build_query($parameters);
+	}
+
+	return bluesky_get($uid, '/xrpc/' . $url, HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]);
+}
+
 function bluesky_get(int $uid, string $url, string $accept_content = HttpClientAccept::DEFAULT, array $opts = []): ?stdClass
 {
 	try {

bluesky/bluesky_feed.php

@@ -0,0 +1,16 @@
+<?php
+
+use Friendica\Core\Logger;
+
+function bluesky_feed_run($argv, $argc)
+{
+	require_once 'addon/bluesky/bluesky.php';
+
+	if ($argc != 3) {
+		return;
+	}
+
+	Logger::debug('Importing feed - start', ['user' => $argv[1], 'feed' => $argv[2]]);
+	bluesky_fetch_feed($argv[1], $argv[2]);
+	Logger::debug('Importing feed - done', ['user' => $argv[1], 'feed' => $argv[2]]);
+}

bluesky/bluesky_notifications.php

@@ -0,0 +1,16 @@
+<?php
+
+use Friendica\Core\Logger;
+
+function bluesky_notifications_run($argv, $argc)
+{
+	require_once 'addon/bluesky/bluesky.php';
+
+	if ($argc != 2) {
+		return;
+	}
+
+	Logger::notice('importing notifications - start', ['user' => $argv[1]]);
+	bluesky_fetch_notifications($argv[1]);
+	Logger::notice('importing notifications - done', ['user' => $argv[1]]);
+}

bluesky/bluesky_timeline.php

@@ -0,0 +1,16 @@
+<?php
+
+use Friendica\Core\Logger;
+
+function bluesky_timeline_run($argv, $argc)
+{
+	require_once 'addon/bluesky/bluesky.php';
+
+	if ($argc != 2) {
+		return;
+	}
+
+	Logger::notice('importing timeline - start', ['user' => $argv[1]]);
+	bluesky_fetch_timeline($argv[1]);
+	Logger::notice('importing timeline - done', ['user' => $argv[1]]);
+}

bluesky/lang/de/messages.po

@@ -0,0 +1,89 @@
+# ADDON bluesky
+# Copyright (C)
+# This file is distributed under the same license as the Friendica bluesky addon package.
+# 
+# 
+# Translators:
+# Tobias Diekershoff <tobias.diekershoff@gmx.net>, 2023
+# haheute <haheute@posteo.de>, 2023
+# Raroun, 2023
+# 
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2023-05-24 05:52+0000\n"
+"PO-Revision-Date: 2023-05-24 06:02+0000\n"
+"Last-Translator: Raroun, 2023\n"
+"Language-Team: German (https://app.transifex.com/Friendica/teams/12172/de/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: de\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: bluesky.php:100
+msgid ""
+"You are authenticated to Bluesky. For security reasons the password isn't "
+"stored."
+msgstr ""
+"Du bist auf Bluesky authentifiziert. Aus Sicherheitsgründen wird das "
+"Passwort nicht gespeichert."
+
+#: bluesky.php:100
+msgid "You are not authenticated. Please enter the app password."
+msgstr ""
+"Du bist derzeit nicht authentifiziert. Bitte gib dein App Passwort ein."
+
+#: bluesky.php:104
+msgid "Enable Bluesky Post Addon"
+msgstr "Bluesky Post Addon aktivieren"
+
+#: bluesky.php:105
+msgid "Post to Bluesky by default"
+msgstr "Standardmäßig auf Bluesky veröffentlichen"
+
+#: bluesky.php:106
+msgid "Import the remote timeline"
+msgstr "Importiere die entfernte Timeline"
+
+#: bluesky.php:107
+msgid "Bluesky host"
+msgstr "Bluesky Host"
+
+#: bluesky.php:108
+msgid "Bluesky handle"
+msgstr "Bluesky Handle"
+
+#: bluesky.php:109
+msgid "Bluesky DID"
+msgstr "Bluesky DID"
+
+#: bluesky.php:109
+msgid ""
+"This is the unique identifier. It will be fetched automatically, when the "
+"handle is entered."
+msgstr ""
+"Sobald das Handle eingegeben ist, wird diese einzigartige Kennung "
+"automatisch abgerufen."
+
+#: bluesky.php:110
+msgid "Bluesky app password"
+msgstr "Bluesky App Passwort"
+
+#: bluesky.php:110
+msgid ""
+"Please don't add your real password here, but instead create a specific app "
+"password in the Bluesky settings."
+msgstr ""
+"Bitte verwende hier nicht dein echtes Passwort, sondern stattdessen ein "
+"speziell für diese App in den Bluesky Einstellungen festgelegtes Passwort."
+
+#: bluesky.php:116
+msgid "Bluesky Import/Export"
+msgstr "Bluesky Import/Export"
+
+#: bluesky.php:167
+msgid "Post to Bluesky"
+msgstr "Auf Bluesky veröffentlichen"

bluesky/lang/de/strings.php

@@ -0,0 +1,20 @@
+<?php
+
+if(! function_exists("string_plural_select_de")) {
+function string_plural_select_de($n){
+	$n = intval($n);
+	return intval($n != 1);
+}}
+$a->strings['You are authenticated to Bluesky. For security reasons the password isn\'t stored.'] = 'Du bist auf Bluesky authentifiziert. Aus Sicherheitsgründen wird das Passwort nicht gespeichert.';
+$a->strings['You are not authenticated. Please enter the app password.'] = 'Du bist derzeit nicht authentifiziert. Bitte gib dein App Passwort ein.';
+$a->strings['Enable Bluesky Post Addon'] = 'Bluesky Post Addon aktivieren';
+$a->strings['Post to Bluesky by default'] = 'Standardmäßig auf Bluesky veröffentlichen';
+$a->strings['Import the remote timeline'] = 'Importiere die entfernte Timeline';
+$a->strings['Bluesky host'] = 'Bluesky Host';
+$a->strings['Bluesky handle'] = 'Bluesky Handle';
+$a->strings['Bluesky DID'] = 'Bluesky DID';
+$a->strings['This is the unique identifier. It will be fetched automatically, when the handle is entered.'] = 'Sobald das Handle eingegeben ist, wird diese einzigartige Kennung automatisch abgerufen.';
+$a->strings['Bluesky app password'] = 'Bluesky App Passwort';
+$a->strings['Please don\'t add your real password here, but instead create a specific app password in the Bluesky settings.'] = 'Bitte verwende hier nicht dein echtes Passwort, sondern stattdessen ein speziell für diese App in den Bluesky Einstellungen festgelegtes Passwort.';
+$a->strings['Bluesky Import/Export'] = 'Bluesky Import/Export';
+$a->strings['Post to Bluesky'] = 'Auf Bluesky veröffentlichen';

bluesky/lang/zh-cn/messages.po

@@ -0,0 +1,80 @@
+# ADDON bluesky
+# Copyright (C)
+# This file is distributed under the same license as the Friendica bluesky addon package.
+# 
+# 
+# Translators:
+# tslmuun, 2023
+# 
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2023-05-24 05:52+0000\n"
+"PO-Revision-Date: 2023-05-24 06:02+0000\n"
+"Last-Translator: tslmuun, 2023\n"
+"Language-Team: Chinese (China) (https://app.transifex.com/Friendica/teams/12172/zh_CN/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: zh_CN\n"
+"Plural-Forms: nplurals=1; plural=0;\n"
+
+#: bluesky.php:100
+msgid ""
+"You are authenticated to Bluesky. For security reasons the password isn't "
+"stored."
+msgstr "您已通过 Bluesky 的身份验证。 出于安全原因，不存储密码。"
+
+#: bluesky.php:100
+msgid "You are not authenticated. Please enter the app password."
+msgstr "您未通过身份验证。 请输入应用密码。"
+
+#: bluesky.php:104
+msgid "Enable Bluesky Post Addon"
+msgstr "启用Bluesky插件"
+
+#: bluesky.php:105
+msgid "Post to Bluesky by default"
+msgstr "默认发布到Bluesky"
+
+#: bluesky.php:106
+msgid "Import the remote timeline"
+msgstr "导入远程时间线"
+
+#: bluesky.php:107
+msgid "Bluesky host"
+msgstr "Bluesky地址"
+
+#: bluesky.php:108
+msgid "Bluesky handle"
+msgstr "Bluesky handle"
+
+#: bluesky.php:109
+msgid "Bluesky DID"
+msgstr "Bluesky DID"
+
+#: bluesky.php:109
+msgid ""
+"This is the unique identifier. It will be fetched automatically, when the "
+"handle is entered."
+msgstr "这是唯一标识符。 输入句柄时，它将自动获取。"
+
+#: bluesky.php:110
+msgid "Bluesky app password"
+msgstr "Bluesky 应用密码"
+
+#: bluesky.php:110
+msgid ""
+"Please don't add your real password here, but instead create a specific app "
+"password in the Bluesky settings."
+msgstr "请不要在此处填写您的真实密码，而是在 Bluesky 设置中创建一个特定的应用程序密码。"
+
+#: bluesky.php:116
+msgid "Bluesky Import/Export"
+msgstr "Bluesky 导入/导出"
+
+#: bluesky.php:167
+msgid "Post to Bluesky"
+msgstr "发布到Bluesky"

bluesky/lang/zh-cn/strings.php

@@ -0,0 +1,20 @@
+<?php
+
+if(! function_exists("string_plural_select_zh_cn")) {
+function string_plural_select_zh_cn($n){
+	$n = intval($n);
+	return intval(0);
+}}
+$a->strings['You are authenticated to Bluesky. For security reasons the password isn\'t stored.'] = '您已通过 Bluesky 的身份验证。 出于安全原因，不存储密码。';
+$a->strings['You are not authenticated. Please enter the app password.'] = '您未通过身份验证。 请输入应用密码。';
+$a->strings['Enable Bluesky Post Addon'] = '启用Bluesky插件';
+$a->strings['Post to Bluesky by default'] = '默认发布到Bluesky';
+$a->strings['Import the remote timeline'] = '导入远程时间线';
+$a->strings['Bluesky host'] = 'Bluesky地址';
+$a->strings['Bluesky handle'] = 'Bluesky handle';
+$a->strings['Bluesky DID'] = 'Bluesky DID';
+$a->strings['This is the unique identifier. It will be fetched automatically, when the handle is entered.'] = '这是唯一标识符。 输入句柄时，它将自动获取。';
+$a->strings['Bluesky app password'] = 'Bluesky 应用密码';
+$a->strings['Please don\'t add your real password here, but instead create a specific app password in the Bluesky settings.'] = '请不要在此处填写您的真实密码，而是在 Bluesky 设置中创建一个特定的应用程序密码。';
+$a->strings['Bluesky Import/Export'] = 'Bluesky 导入/导出';
+$a->strings['Post to Bluesky'] = '发布到Bluesky';

showmore_dyn/showmore_dyn.css

@@ -7,7 +7,7 @@
 	position: absolute;
 	top: auto;
 	bottom: 0;
-	z-index: 11;
+	z-index: 9;
 	left: 0;
 	display: none;
 }