diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 39a0ba718a..b18159d40c 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -314,15 +314,10 @@ class Contact extends BaseObject
 			return;
 		}
 
-		$archive = PConfig::get($contact['uid'], 'system', 'archive_removed_contacts');
-		if ($archive) {
-			DBA::update('contact', ['archive' => true, 'network' => 'none', 'writable' => false], ['id' => $id]);
-			return;
-		}
+		// Archive the contact
+		DBA::update('contact', ['archive' => true, 'network' => Protocol::PHANTOM], ['id' => $id]);
 
-		DBA::delete('contact', ['id' => $id]);
-
-		// Delete the rest in the background
+		// Delete it in the background
 		Worker::add(PRIORITY_LOW, 'RemoveContact', $id);
 	}
 
diff --git a/src/Worker/RemoveContact.php b/src/Worker/RemoveContact.php
index 40ac937f10..01388516bc 100644
--- a/src/Worker/RemoveContact.php
+++ b/src/Worker/RemoveContact.php
@@ -12,13 +12,14 @@ require_once 'include/dba.php';
 class RemoveContact {
 	public static function execute($id) {
 
-		// Only delete if the contact doesn't exist (anymore)
-		$r = DBA::exists('contact', ['id' => $id]);
-		if ($r) {
+		// Only delete if the contact is archived
+		$condition = ['archive' => true, 'network' => Protocol::PHANTOM, 'id' => $id];
+		$r = DBA::exists('contact', $condition);
+		if (!DBA::isResult($r)) {
 			return;
 		}
 
-		// Now we delete all the depending table entries
+		// Now we delete the contact and all depending tables
 		DBA::delete('contact', ['id' => $id]);
 	}
 }