Merge pull request #11740 from annando/json-ld

Improved JsonLD parsing

funkwhale.audio.jsonld

@@ -0,0 +1,63 @@
+{
+  "@context": {
+    "id": "@id",
+    "type": "@type",
+    "as": "https://www.w3.org/ns/activitystreams#",
+    "schema": "http://schema.org#",
+    "fw": "https://funkwhale.audio/ns#",
+    "xsd": "http://www.w3.org/2001/XMLSchema#",
+    "Album": "fw:Album",
+    "Track": "fw:Track",
+    "Artist": "fw:Artist",
+    "Library": "fw:Library",
+    "bitrate": {
+      "@id": "fw:bitrate",
+      "@type": "xsd:nonNegativeInteger"
+    },
+    "size": {
+      "@id": "fw:size",
+      "@type": "xsd:nonNegativeInteger"
+    },
+    "position": {
+      "@id": "fw:position",
+      "@type": "xsd:nonNegativeInteger"
+    },
+    "disc": {
+      "@id": "fw:disc",
+      "@type": "xsd:nonNegativeInteger"
+    },
+    "library": {
+      "@id": "fw:library",
+      "@type": "@id"
+    },
+    "track": {
+      "@id": "fw:track",
+      "@type": "@id"
+    },
+    "cover": {
+      "@id": "fw:cover",
+      "@type": "as:Link"
+    },
+    "album": {
+      "@id": "fw:album",
+      "@type": "@id"
+    },
+    "artists": {
+      "@id": "fw:artists",
+      "@type": "@id",
+      "@container": "@list"
+    },
+    "released": {
+      "@id": "fw:released",
+      "@type": "xsd:date"
+    },
+    "musicbrainzId": "fw:musicbrainzId",
+    "license": {
+      "@id": "fw:license",
+      "@type": "@id"
+    },
+    "copyright": "fw:copyright",
+    "category": "sc:category",
+    "language": "sc:inLanguage"
+  }
+}

src/Util/JsonLD.php

@@ -52,9 +52,23 @@ class JsonLD
 			case 'https://www.w3.org/ns/activitystreams':
 				$url = DI::baseUrl() . '/static/activitystreams.jsonld';
 				break;
-			default:
+			case 'https://funkwhale.audio/ns':
-				Logger::info('Got url', ['url' =>$url]);
+				$url = DI::baseUrl() . '/static/funkwhale.audio.jsonld';
 				break;
+			default:
+				switch (parse_url($url, PHP_URL_PATH)) {
+					case '/schemas/litepub-0.1.jsonld';
+						$url = DI::baseUrl() . '/static/litepub-0.1.jsonld';
+						break;
+					case '/apschema/v1.2':
+					case '/apschema/v1.9':
+					case '/apschema/v1.10':
+							$url = DI::baseUrl() . '/static/apschema.jsonld';
+						break;
+					default:
+						Logger::info('Got url', ['url' =>$url]);
+						break;
+				}
 		}
 
 		$recursion = 0;
@@ -121,11 +135,12 @@ class JsonLD
 	 * Compacts a given JSON array
 	 *
 	 * @param array $json
+	 * @param bool  $logfailed
 	 *
 	 * @return array Compacted JSON array
 	 * @throws Exception
 	 */
-	public static function compact($json)
+	public static function compact($json, bool $logfailed = true)
 	{
 		jsonld_set_document_loader('Friendica\Util\JsonLD::documentLoader');
 
@@ -144,21 +159,22 @@ class JsonLD
 			'mobilizon' => (object)['@id' => 'https://joinmobilizon.org/ns#', '@type' => '@id'],
 		];
 
+		$orig_json = $json;
+
 		// Preparation for adding possibly missing content to the context
 		if (!empty($json['@context']) && is_string($json['@context'])) {
 			$json['@context'] = [$json['@context']];
 		}
 
-		// Workaround for servers with missing context
-		// See issue https://github.com/nextcloud/social/issues/330
 		if (!empty($json['@context']) && is_array($json['@context'])) {
-			$json['@context'][] = 'https://w3id.org/security/v1';
+			// Remove empty entries from the context (a problem with WriteFreely)
-		}
+			$json['@context'] = array_filter($json['@context']);
 
-		// Trying to avoid memory problems with large content fields
+			// Workaround for servers with missing context
-		if (!empty($json['object']['source']['content'])) {
+			// See issue https://github.com/nextcloud/social/issues/330
-			$content = $json['object']['source']['content'];
+			if (!in_array('https://w3id.org/security/v1', $json['@context'])) {
-			$json['object']['source']['content'] = '';
+				$json['@context'][] = 'https://w3id.org/security/v1';
+			}
 		}
 
 		$jsonobj = json_decode(json_encode($json, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
@@ -168,15 +184,16 @@ class JsonLD
 		}
 		catch (Exception $e) {
 			$compacted = false;
-			Logger::notice('compacting error', ['line' => $e->getLine(), 'exception' => $e]);
+			Logger::notice('compacting error', ['msg' => $e->getMessage(), 'previous' => $e->getPrevious(), 'line' => $e->getLine()]);
+			if ($logfailed && DI::config()->get('debug', 'ap_log_failure')) {
+				$tempfile = tempnam(System::getTempPath(), 'failed-jsonld');
+				file_put_contents($tempfile, json_encode(['json' => $orig_json, 'callstack' => System::callstack(20), 'msg' => $e->getMessage(), 'previous' => $e->getPrevious()], JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT));
+				Logger::notice('Failed message stored', ['file' => $tempfile]);
+			}
 		}
 
 		$json = json_decode(json_encode($compacted, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), true);
 
-		if (isset($json['as:object']['as:source']['as:content']) && !empty($content)) {
-			$json['as:object']['as:source']['as:content'] = $content;
-		}
-
 		return $json;
 	}
 

static/apschema.jsonld

@@ -0,0 +1,39 @@
+{
+    "@context":{
+        "zot":"https://hubzilla.org/apschema#",
+        "id":"@id",
+        "type":"@type",
+        "commentPolicy":"zot:commentPolicy",
+        "meData":"zot:meData",
+        "meDataType":"zot:meDataType",
+        "meEncoding":"zot:meEncoding",
+        "meAlgorithm":"zot:meAlgorithm",
+        "meCreator":"zot:meCreator",
+        "meSignatureValue":"zot:meSignatureValue",
+        "locationAddress":"zot:locationAddress",
+        "locationPrimary":"zot:locationPrimary",
+        "locationDeleted":"zot:locationDeleted",
+        "nomadicLocation":"zot:nomadicLocation",
+        "nomadicHubs":"zot:nomadicHubs",
+        "emojiReaction":"zot:emojiReaction",
+        "expires":"zot:expires",
+        "directMessage":"zot:directMessage",
+        "schema":"http://schema.org#",
+        "PropertyValue":"schema:PropertyValue",
+        "value":"schema:value",
+        "manuallyApprovesFollowers":"as:manuallyApprovesFollowers",
+        "magicEnv":{
+            "@id":"zot:magicEnv",
+            "@type":"@id"
+        },
+        "nomadicLocations":{
+            "@id":"zot:nomadicLocations",
+            "@type":"@id"
+        },
+        "ostatus":"http://ostatus.org#",
+        "conversation":"ostatus:conversation",
+        "diaspora":"https://diasporafoundation.org/ns/",
+        "guid":"diaspora:guid",
+        "Hashtag":"as:Hashtag"
+    }
+}

static/defaults.config.php

@@ -669,5 +669,13 @@ return [
 		// total_ap_delivery (Boolean)
 		// Deliver via AP to every possible receiver and we suppress the delivery to these contacts with other protocols
 		'total_ap_delivery' => false,
+
+		// ap_log_unknown (Boolean)
+		// Logs every unknown ActivityPub activity
+		'ap_log_unknown' => false,
+
+		// ap_log_failure (Boolean)
+		// Logs every ActivityPub activity that couldn't be compacted
+		'ap_log_failure' => false,
 	]
 ];

static/litepub-0.1.jsonld

@@ -0,0 +1,41 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "Emoji": "toot:Emoji",
+            "Hashtag": "as:Hashtag",
+            "PropertyValue": "schema:PropertyValue",
+            "atomUri": "ostatus:atomUri",
+            "conversation": {
+                "@id": "ostatus:conversation",
+                "@type": "@id"
+            },
+            "discoverable": "toot:discoverable",
+            "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+            "capabilities": "litepub:capabilities",
+            "ostatus": "http://ostatus.org#",
+            "schema": "http://schema.org#",
+            "toot": "http://joinmastodon.org/ns#",
+            "value": "schema:value",
+            "sensitive": "as:sensitive",
+            "litepub": "http://litepub.social/ns#",
+            "invisible": "litepub:invisible",
+            "directMessage": "litepub:directMessage",
+            "listMessage": {
+                "@id": "litepub:listMessage",
+                "@type": "@id"
+            },
+            "oauthRegistrationEndpoint": {
+                "@id": "litepub:oauthRegistrationEndpoint",
+                "@type": "@id"
+            },
+            "EmojiReact": "litepub:EmojiReact",
+            "ChatMessage": "litepub:ChatMessage",
+            "alsoKnownAs": {
+                "@id": "as:alsoKnownAs",
+                "@type": "@id"
+            }
+        }
+    ]
+}