Rework wallet history methods:

- wallet.get_full_history returns onchain and lightning
 - capital gains are returned by get_detailed_history
 - display lightning history in kivy
 - command line: separate lightning and onchain history

electrum/commands.py

@@ -497,14 +497,11 @@ class Commands:
         return tx.as_dict()
 
     @command('w')
-    def history(self, year=None, show_addresses=False, show_fiat=False, show_fees=False,
+    def onchain_history(self, year=None, show_addresses=False, show_fiat=False, show_fees=False):
-                from_height=None, to_height=None):
+        """Wallet onchain history. Returns the transaction history of your wallet."""
-        """Wallet history. Returns the transaction history of your wallet."""
         kwargs = {
             'show_addresses': show_addresses,
             'show_fees': show_fees,
-            'from_height': from_height,
-            'to_height': to_height,
         }
         if year:
             import time
@@ -516,7 +513,13 @@ class Commands:
             from .exchange_rate import FxThread
             fx = FxThread(self.config, None)
             kwargs['fx'] = fx
-        return json_encode(self.wallet.get_full_history(**kwargs))
+        return json_encode(self.wallet.get_detailed_history(**kwargs))
+
+    @command('w')
+    def lightning_history(self, show_fiat=False):
+        """ lightning history """
+        lightning_history = self.wallet.lnworker.get_history() if self.wallet.lnworker else []
+        return json_encode(lightning_history)
 
     @command('w')
     def setlabel(self, key, label):

electrum/gui/kivy/uix/screens.py

@@ -26,7 +26,7 @@ from electrum.util import profiler, parse_URI, format_time, InvalidPassword, Not
 from electrum import bitcoin, constants
 from electrum.transaction import TxOutput, Transaction, tx_from_str
 from electrum.util import send_exception_to_crash_reporter, parse_URI, InvalidBitcoinURI
-from electrum.util import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED
+from electrum.util import PR_UNPAID, PR_PAID, PR_UNKNOWN, PR_EXPIRED, TxMinedInfo
 from electrum.plugin import run_hook
 from electrum.wallet import InternalAddressCorruption
 from electrum import simple_config
@@ -145,34 +145,49 @@ class HistoryScreen(CScreen):
         d = LabelDialog(_('Enter Transaction Label'), text, callback)
         d.open()
 
-    def get_card(self, tx_hash, tx_mined_status, value, balance):
+    def get_card(self, tx_item): #tx_hash, tx_mined_status, value, balance):
-        status, status_str = self.app.wallet.get_tx_status(tx_hash, tx_mined_status)
+        is_lightning = tx_item.get('lightning', False)
-        icon = "atlas://electrum/gui/kivy/theming/light/" + TX_ICONS[status]
+        timestamp = tx_item['timestamp']
-        label = self.app.wallet.get_label(tx_hash) if tx_hash else _('Pruned transaction outputs')
+        if is_lightning:
+            status = 0
+            txpos = tx_item['txpos']
+            if timestamp is None:
+                status_str = 'unconfirmed'
+            else:
+                status_str = format_time(int(timestamp))
+            icon = "atlas://electrum/gui/kivy/theming/light/lightning"
+        else:
+            tx_hash = tx_item['txid']
+            conf = tx_item['confirmations']
+            txpos = tx_item['txpos_in_block'] or 0
+            height = tx_item['height']
+            tx_mined_info = TxMinedInfo(height=tx_item['height'],
+                                        conf=tx_item['confirmations'],
+                                        timestamp=tx_item['timestamp'])
+            status, status_str = self.app.wallet.get_tx_status(tx_hash, tx_mined_info)
+            icon = "atlas://electrum/gui/kivy/theming/light/" + TX_ICONS[status]
         ri = {}
         ri['screen'] = self
-        ri['tx_hash'] = tx_hash
         ri['icon'] = icon
         ri['date'] = status_str
-        ri['message'] = label
+        ri['message'] = tx_item['label']
-        ri['confirmations'] = tx_mined_status.conf
+        value = tx_item['value'].value
         if value is not None:
             ri['is_mine'] = value < 0
             if value < 0: value = - value
             ri['amount'] = self.app.format_amount_and_units(value)
-            if self.app.fiat_unit:
+            if 'fiat_value' in tx_item:
-                fx = self.app.fx
+                ri['quote_text'] = tx_item['fiat_value'].to_ui_string()
-                fiat_value = value / Decimal(bitcoin.COIN) * self.app.wallet.price_at_timestamp(tx_hash, fx.timestamp_rate)
-                fiat_value = Fiat(fiat_value, fx.ccy)
-                ri['quote_text'] = fiat_value.to_ui_string()
         return ri
 
     def update(self, see_all=False):
-        if self.app.wallet is None:
+        import operator
+        wallet = self.app.wallet
+        if wallet is None:
             return
-        history = reversed(self.app.wallet.get_history())
+        history = sorted(wallet.get_full_history(self.app.fx).values(), key=lambda x: x.get('timestamp') or float('inf'), reverse=True)
         history_card = self.screen.ids.history_container
-        history_card.data = [self.get_card(*item) for item in history]
+        history_card.data = [self.get_card(item) for item in history]
 
 
 class SendScreen(CScreen):

electrum/gui/qt/history_list.py

@@ -116,7 +116,6 @@ class HistoryModel(QAbstractItemModel, Logger):
         self.view = None  # type: HistoryList
         self.transactions = OrderedDictWithIndex()
         self.tx_status_cache = {}  # type: Dict[str, Tuple[int, str]]
-        self.summary = None
 
     def set_view(self, history_list: 'HistoryList'):
         # FIXME HistoryModel and HistoryList mutually depend on each other.
@@ -173,7 +172,7 @@ class HistoryModel(QAbstractItemModel, Logger):
                 HistoryColumns.DESCRIPTION:
                     tx_item['label'] if 'label' in tx_item else None,
                 HistoryColumns.AMOUNT:
-                    tx_item['value'].value if 'value' in tx_item else None,
+                    tx_item['bc_value'].value if 'bc_value' in tx_item else None,
                 HistoryColumns.LN_AMOUNT:
                     tx_item['ln_value'].value if 'ln_value' in tx_item else None,
                 HistoryColumns.BALANCE:
@@ -217,8 +216,8 @@ class HistoryModel(QAbstractItemModel, Logger):
             return QVariant(status_str)
         elif col == HistoryColumns.DESCRIPTION and 'label' in tx_item:
             return QVariant(tx_item['label'])
-        elif col == HistoryColumns.AMOUNT and 'value' in tx_item:
+        elif col == HistoryColumns.AMOUNT and 'bc_value' in tx_item:
-            value = tx_item['value'].value
+            value = tx_item['bc_value'].value
             v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True)
             return QVariant(v_str)
         elif col == HistoryColumns.LN_AMOUNT and 'ln_value' in tx_item:
@@ -276,44 +275,22 @@ class HistoryModel(QAbstractItemModel, Logger):
         fx = self.parent.fx
         if fx: fx.history_used_spot = False
         wallet = self.parent.wallet
-        r = wallet.get_full_history(domain=self.get_domain(), from_timestamp=None, to_timestamp=None, fx=fx)
-        lightning_history = wallet.lnworker.get_history() if wallet.lnworker else []
         self.set_visibility_of_columns()
-        #if r['transactions'] == list(self.transactions.values()):
+        transactions = wallet.get_full_history(self.parent.fx)
-        #    return
+        if transactions == list(self.transactions.values()):
+            return
         old_length = len(self.transactions)
         if old_length != 0:
             self.beginRemoveRows(QModelIndex(), 0, old_length)
             self.transactions.clear()
             self.endRemoveRows()
-
-        transactions = OrderedDictWithIndex()
-        for tx_item in r['transactions']:
-            txid = tx_item['txid']
-            transactions[txid] = tx_item
-        for i, tx_item in enumerate(lightning_history):
-            txid = tx_item.get('txid')
-            ln_value = tx_item['amount_msat']/1000.
-            if txid and txid in transactions:
-                item = transactions[txid]
-                item['label'] = tx_item['label']
-                item['ln_value'] = Satoshis(ln_value)
-                item['balance_msat'] = tx_item['balance_msat']
-            else:
-                tx_item['lightning'] = True
-                tx_item['ln_value'] = Satoshis(ln_value)
-                tx_item['txpos'] = i # for sorting
-                key = tx_item['payment_hash'] if 'payment_hash' in tx_item else tx_item['type'] + tx_item['channel_id']
-                transactions[key] = tx_item
-
         self.beginInsertRows(QModelIndex(), 0, len(transactions)-1)
         self.transactions = transactions
         self.endInsertRows()
         if selected_row:
             self.view.selectionModel().select(self.createIndex(selected_row, 0), QItemSelectionModel.Rows | QItemSelectionModel.SelectCurrent)
         self.view.filter()
-        # update summary
+        # update time filter
-        self.summary = r['summary']
         if not self.view.years and self.transactions:
             start_date = date.today()
             end_date = date.today()
@@ -535,7 +512,7 @@ class HistoryList(MyTreeView, AcceptFileDragDrop):
             return datetime.datetime(date.year, date.month, date.day)
 
     def show_summary(self):
-        h = self.model().sourceModel().summary
+        h = self.parent.wallet.get_detailed_history()['summary']
         if not h:
             self.parent.show_message(_("Nothing to summarize."))
             return

electrum/wallet.py

@@ -45,7 +45,7 @@ from .util import (NotEnoughFunds, UserCancelled, profiler,
                    format_satoshis, format_fee_satoshis, NoDynamicFeeEstimates,
                    WalletFileException, BitcoinException,
                    InvalidPassword, format_time, timestamp_to_datetime, Satoshis,
-                   Fiat, bfh, bh2u, TxMinedInfo, quantize_feerate, create_bip21_uri)
+                   Fiat, bfh, bh2u, TxMinedInfo, quantize_feerate, create_bip21_uri, OrderedDictWithIndex)
 from .simple_config import get_config
 from .bitcoin import (COIN, TYPE_ADDRESS, is_address, address_to_script,
                       is_minikey, relayfee, dust_threshold)
@@ -482,45 +482,79 @@ class Abstract_Wallet(AddressSynchronizer):
         # return last balance
         return balance
 
+    def get_onchain_history(self):
+        for tx_hash, tx_mined_status, value, balance in self.get_history():
+            yield {
+                'txid': tx_hash,
+                'height': tx_mined_status.height,
+                'confirmations': tx_mined_status.conf,
+                'timestamp': tx_mined_status.timestamp,
+                'incoming': True if value>0 else False,
+                'bc_value': Satoshis(value),
+                'balance': Satoshis(balance),
+                'date': timestamp_to_datetime(tx_mined_status.timestamp),
+                'label': self.get_label(tx_hash),
+                'txpos_in_block': tx_mined_status.txpos,
+            }
+
     @profiler
-    def get_full_history(self, domain=None, from_timestamp=None, to_timestamp=None,
+    def get_full_history(self, fx=None):
-                         fx=None, show_addresses=False, show_fees=False,
+        transactions = OrderedDictWithIndex()
-                         from_height=None, to_height=None):
+        onchain_history = self.get_onchain_history()
-        if (from_timestamp is not None or to_timestamp is not None) \
+        for tx_item in onchain_history:
-                and (from_height is not None or to_height is not None):
+            txid = tx_item['txid']
-            raise Exception('timestamp and block height based filtering cannot be used together')
+            transactions[txid] = tx_item
+        lightning_history = self.lnworker.get_history() if self.lnworker else []
+        for i, tx_item in enumerate(lightning_history):
+            txid = tx_item.get('txid')
+            ln_value = Decimal(tx_item['amount_msat']) / 1000
+            if txid and txid in transactions:
+                item = transactions[txid]
+                item['label'] = tx_item['label']
+                item['ln_value'] = Satoshis(ln_value)
+                item['balance_msat'] = tx_item['balance_msat']
+            else:
+                tx_item['lightning'] = True
+                tx_item['ln_value'] = Satoshis(ln_value)
+                tx_item['txpos'] = i # for sorting
+                key = tx_item['payment_hash'] if 'payment_hash' in tx_item else tx_item['type'] + tx_item['channel_id']
+                transactions[key] = tx_item
+        now = time.time()
+        for item in transactions.values():
+            # add on-chain and lightning values
+            value = Decimal(0)
+            if item.get('bc_value'):
+                value += item['bc_value'].value
+            if item.get('ln_value'):
+                value += item.get('ln_value').value
+            item['value'] = Satoshis(value)
+            if fx:
+                timestamp = item['timestamp'] or now
+                fiat_value = value / Decimal(bitcoin.COIN) * fx.timestamp_rate(timestamp)
+                item['fiat_value'] = Fiat(fiat_value, fx.ccy)
+                item['fiat_default'] = True
+        return transactions
+
+    @profiler
+    def get_detailed_history(self, from_timestamp=None, to_timestamp=None,
+                             fx=None, show_addresses=False, show_fees=False):
+        # History with capital gains, using utxo pricing
+        # FIXME: Lightning capital gains would requires FIFO
         out = []
         income = 0
         expenditures = 0
         capital_gains = Decimal(0)
         fiat_income = Decimal(0)
         fiat_expenditures = Decimal(0)
-        h = self.get_history(domain)
         now = time.time()
-        for tx_hash, tx_mined_status, value, balance in h:
+        for item in self.get_onchain_history():
-            timestamp = tx_mined_status.timestamp
+            timestamp = item['timestamp']
             if from_timestamp and (timestamp or now) < from_timestamp:
                 continue
             if to_timestamp and (timestamp or now) >= to_timestamp:
                 continue
-            height = tx_mined_status.height
+            tx_hash = item['txid']
-            if from_height is not None and height < from_height:
-                continue
-            if to_height is not None and height >= to_height:
-                continue
             tx = self.db.get_transaction(tx_hash)
-            item = {
-                'txid': tx_hash,
-                'height': height,
-                'confirmations': tx_mined_status.conf,
-                'timestamp': timestamp,
-                'incoming': True if value>0 else False,
-                'value': Satoshis(value),
-                'balance': Satoshis(balance),
-                'date': timestamp_to_datetime(timestamp),
-                'label': self.get_label(tx_hash),
-                'txpos_in_block': tx_mined_status.txpos,
-            }
             tx_fee = None
             if show_fees:
                 tx_fee = self.get_tx_fee(tx)
@@ -529,10 +563,8 @@ class Abstract_Wallet(AddressSynchronizer):
                 item['inputs'] = list(map(lambda x: dict((k, x[k]) for k in ('prevout_hash', 'prevout_n')), tx.inputs()))
                 item['outputs'] = list(map(lambda x:{'address':x.address, 'value':Satoshis(x.value)},
                                            tx.get_outputs_for_UI()))
-            # value may be None if wallet is not fully synchronized
-            if value is None:
-                continue
             # fixme: use in and out values
+            value = item['bc_value'].value
             if value < 0:
                 expenditures += -value
             else:
@@ -550,7 +582,7 @@ class Abstract_Wallet(AddressSynchronizer):
             out.append(item)
         # add summary
         if out:
-            b, v = out[0]['balance'].value, out[0]['value'].value
+            b, v = out[0]['balance'].value, out[0]['bc_value'].value
             start_balance = None if b is None or v is None else b - v
             end_balance = out[-1]['balance'].value
             if from_timestamp is not None and to_timestamp is not None:
@@ -562,15 +594,13 @@ class Abstract_Wallet(AddressSynchronizer):
             summary = {
                 'start_date': start_date,
                 'end_date': end_date,
-                'from_height': from_height,
-                'to_height': to_height,
                 'start_balance': Satoshis(start_balance),
                 'end_balance': Satoshis(end_balance),
                 'incoming': Satoshis(income),
                 'outgoing': Satoshis(expenditures)
             }
             if fx and fx.is_enabled() and fx.get_history_config():
-                unrealized = self.unrealized_gains(domain, fx.timestamp_rate, fx.ccy)
+                unrealized = self.unrealized_gains(None, fx.timestamp_rate, fx.ccy)
                 summary['fiat_currency'] = fx.ccy
                 summary['fiat_capital_gains'] = Fiat(capital_gains, fx.ccy)
                 summary['fiat_incoming'] = Fiat(fiat_income, fx.ccy)