diff --git a/electrum/wallet.py b/electrum/wallet.py
index 097d4c1bf..20c983549 100644
--- a/electrum/wallet.py
+++ b/electrum/wallet.py
@@ -823,7 +823,7 @@ class Abstract_Wallet(AddressSynchronizer, ABC):
         for invoice in self.get_relevant_invoices_for_tx(tx):
             if invoice.message:
                 labels.append(invoice.message)
-        if labels:
+        if labels and not self.labels.get(tx_hash, ''):
             self.set_label(tx_hash, "; ".join(labels))
         return bool(labels)