Detect blockchain splits and validate multiple chains

RELEASE-NOTES

@@ -1,3 +1,10 @@
+# Release 2.9 - Independence
+  * Blockchain fork detection and management:
+    - The SPV module will download and verify block headers from
+      multiple branches
+    - Branching points are located using binary search
+    - The desired branch of a fork can be selected using the network dialog
+
 # Release 2.8.3
   * Fix crash on reading older wallet formats.
   * TrustedCoin: remove pay-per-tx option

gui/kivy/uix/dialogs/checkpoint_dialog.py

@@ -48,7 +48,6 @@ Builder.load_string('''
                 height: '36dp'
                 size_hint_y: None
                 text: '%d'%root.cp_height
-                on_focus: root.on_height_str()
             TopLabel:
                 text: _('Block hash') + ':'
             TxHashLabel:
@@ -85,23 +84,5 @@ class CheckpointDialog(Factory.Popup):
     def __init__(self, network, callback):
         Factory.Popup.__init__(self)
         self.network = network
-        self.cp_height, self.cp_value = self.network.blockchain.get_checkpoint()
         self.callback = callback
-
+        self.is_split = len(self.network.blockchains) > 1
-    def on_height_str(self):
-        try:
-            new_height = int(self.ids.height_input.text)
-        except:
-            new_height = self.cp_height
-        self.ids.height_input.text = '%d'%new_height
-        if new_height == self.cp_height:
-            return
-        try:
-            header = self.network.synchronous_get(('blockchain.block.get_header', [new_height]), 5)
-            new_value = self.network.blockchain.hash_header(header)
-        except BaseException as e:
-            self.network.print_error(str(e))
-            new_value = ''
-        if new_value:
-            self.cp_height = new_height
-            self.cp_value = new_value

gui/qt/network_dialog.py

@@ -190,39 +190,10 @@ class NetworkChoiceLayout(object):
         from amountedit import AmountEdit
         grid = QGridLayout(blockchain_tab)
         n = len(network.get_interfaces())
-        status = _("Connected to %d nodes.")%n if n else _("Not connected")
+        n_chains = len(network.blockchains)
-        height_str = "%d "%(network.get_local_height()) + _("blocks")
+        self.checkpoint_height = network.get_checkpoint()
-        self.checkpoint_height, self.checkpoint_value = network.blockchain.get_checkpoint()
-        self.cph_label = QLabel(_('Height'))
-        self.cph = QLineEdit("%d"%self.checkpoint_height)
-        self.cph.setFixedWidth(80)
-        self.cpv_label = QLabel(_('Hash'))
-        self.cpv = QLineEdit(self.checkpoint_value)
-        self.cpv.setCursorPosition(0)
-        self.cpv.setFocusPolicy(Qt.NoFocus)
-        self.cpv.setReadOnly(True)
-        def on_cph():
-            try:
-                height = int(self.cph.text())
-            except:
-                height = 0
-            self.cph.setText('%d'%height)
-            if height == self.checkpoint_height:
-                return
-            try:
-                self.network.print_error("fetching header")
-                header = self.network.synchronous_get(('blockchain.block.get_header', [height]), 5)
-                _hash = self.network.blockchain.hash_header(header)
-            except BaseException as e:
-                self.network.print_error(str(e))
-                _hash = ''
-            self.cpv.setText(_hash)
-            self.cpv.setCursorPosition(0)
-            if _hash:
-                self.checkpoint_height = height
-                self.checkpoint_value = _hash
-        self.cph.editingFinished.connect(on_cph)
 
+        status = _("Connected to %d nodes.")%n if n else _("Not connected")
         msg =  ' '.join([
             _("Electrum connects to several nodes in order to download block headers and find out the longest blockchain."),
             _("This blockchain is used to verify the transactions sent by your transaction server.")
@@ -230,23 +201,26 @@ class NetworkChoiceLayout(object):
         grid.addWidget(QLabel(_('Status') + ':'), 0, 0)
         grid.addWidget(QLabel(status), 0, 1, 1, 3)
         grid.addWidget(HelpButton(msg), 0, 4)
+        if n_chains == 1:
+            height_str = "%d "%(network.get_local_height()) + _("blocks")
             msg = _('This is the height of your local copy of the blockchain.')
             grid.addWidget(QLabel(_("Height") + ':'), 1, 0)
             grid.addWidget(QLabel(height_str), 1, 1)
             grid.addWidget(HelpButton(msg), 1, 4)
-        msg = ''.join([
+        else:
-            _('A checkpoint can be used to verify that you are on the correct blockchain.'), ' ',
+            checkpoint = network.get_checkpoint()
-            _('By default, your checkpoint is the genesis block.'), '\n\n',
+            self.cph_label = QLabel(_('Chain split detected'))
-            _('If you edit the height field, the corresponding block hash will be fetched from your current server.'), ' ',
-            _('If you press OK, the checkpoint will be saved, and Electrum will only accept headers from nodes that pass this checkpoint.'), '\n\n',
-            _('If there is a hard fork, you will have to check the block hash from an independent source, in order to be sure that you are on the desired side of the fork.'),
-        ])
-        grid.addWidget(QLabel(_('Checkpoint') +':'), 3, 0, 1, 2)
-        grid.addWidget(HelpButton(msg), 3, 4)
             grid.addWidget(self.cph_label, 4, 0)
-        grid.addWidget(self.cph, 4, 1)
+            chains_list_widget = QTreeWidget()
-        grid.addWidget(self.cpv_label, 5, 0)
+            chains_list_widget.setHeaderLabels( [ _('Nodes'), _('Blocks'), _('Checkpoint'), _('Hash') ] )
-        grid.addWidget(self.cpv, 5, 1, 1, 4)
+            chains_list_widget.setMaximumHeight(150)
+            grid.addWidget(chains_list_widget, 5, 0, 1, 5)
+            for b in network.blockchains.values():
+                _hash = b.get_hash(checkpoint)
+                height = b.height()
+                count = sum([i.blockchain == b for i in network.interfaces.values()])
+                chains_list_widget.addTopLevelItem(QTreeWidgetItem( [ '%d'%count, '%d'%height, '%d'%checkpoint, _hash ] ))
+
         grid.setRowStretch(7, 1)
         vbox = QVBoxLayout()
         vbox.addWidget(tabs)
@@ -328,7 +302,7 @@ class NetworkChoiceLayout(object):
             proxy = None
         auto_connect = self.autoconnect_cb.isChecked()
         self.network.set_parameters(host, port, protocol, proxy, auto_connect)
-        self.network.blockchain.set_checkpoint(self.checkpoint_height, self.checkpoint_value)
+        #self.network.blockchain.set_checkpoint(self.checkpoint_height, self.checkpoint_value)
 
     def suggest_proxy(self, found_proxy):
         self.tor_proxy = found_proxy

lib/blockchain.py

@@ -32,50 +32,56 @@ from bitcoin import *
 
 MAX_TARGET = 0x00000000FFFF0000000000000000000000000000000000000000000000000000
 
+def serialize_header(res):
+    s = int_to_hex(res.get('version'), 4) \
+        + rev_hex(res.get('prev_block_hash')) \
+        + rev_hex(res.get('merkle_root')) \
+        + int_to_hex(int(res.get('timestamp')), 4) \
+        + int_to_hex(int(res.get('bits')), 4) \
+        + int_to_hex(int(res.get('nonce')), 4)
+    return s
+
+def deserialize_header(s, height):
+    hex_to_int = lambda s: int('0x' + s[::-1].encode('hex'), 16)
+    h = {}
+    h['version'] = hex_to_int(s[0:4])
+    h['prev_block_hash'] = hash_encode(s[4:36])
+    h['merkle_root'] = hash_encode(s[36:68])
+    h['timestamp'] = hex_to_int(s[68:72])
+    h['bits'] = hex_to_int(s[72:76])
+    h['nonce'] = hex_to_int(s[76:80])
+    h['block_height'] = height
+    return h
+
+def hash_header(header):
+    if header is None:
+        return '0' * 64
+    if header.get('prev_block_hash') is None:
+        header['prev_block_hash'] = '00'*32
+    return hash_encode(Hash(serialize_header(header).decode('hex')))
+
+
 class Blockchain(util.PrintError):
     '''Manages blockchain headers and their verification'''
-    def __init__(self, config, network):
+    def __init__(self, config, checkpoint):
         self.config = config
-        self.network = network
+        self.checkpoint = checkpoint
-        self.checkpoint_height, self.checkpoint_hash = self.get_checkpoint()
+        self.filename = 'blockchain_headers' if checkpoint == 0 else 'blockchain_fork_%d'%checkpoint
-        self.check_truncate_headers()
         self.set_local_height()
+        self.catch_up = None # interface catching up
 
     def height(self):
         return self.local_height
 
-    def init(self):
-        import threading
-        if os.path.exists(self.path()):
-            self.downloading_headers = False
-            return
-        self.downloading_headers = True
-        t = threading.Thread(target = self.init_headers_file)
-        t.daemon = True
-        t.start()
-
-    def pass_checkpoint(self, header):
-        if type(header) is not dict:
-            return False
-        if header.get('block_height') != self.checkpoint_height:
-            return True
-        if header.get('prev_block_hash') is None:
-            header['prev_block_hash'] = '00'*32
-        try:
-            _hash = self.hash_header(header)
-        except:
-            return False
-        return _hash == self.checkpoint_hash
-
     def verify_header(self, header, prev_header, bits, target):
-        prev_hash = self.hash_header(prev_header)
+        prev_hash = hash_header(prev_header)
-        _hash = self.hash_header(header)
+        _hash = hash_header(header)
         if prev_hash != header.get('prev_block_hash'):
             raise BaseException("prev hash mismatch: %s vs %s" % (prev_hash, header.get('prev_block_hash')))
-        if not self.pass_checkpoint(header):
+        #if not self.pass_checkpoint(header):
-            raise BaseException('failed checkpoint')
+        #    raise BaseException('failed checkpoint')
-        if self.checkpoint_height == header.get('block_height'):
+        #if self.checkpoint_height == header.get('block_height'):
-            self.print_error("validated checkpoint", self.checkpoint_height)
+        #    self.print_error("validated checkpoint", self.checkpoint_height)
         if bitcoin.TESTNET:
             return
         if bits != header.get('bits'):
@@ -100,70 +106,31 @@ class Blockchain(util.PrintError):
         bits, target = self.get_target(index)
         for i in range(num):
             raw_header = data[i*80:(i+1) * 80]
-            header = self.deserialize_header(raw_header, index*2016 + i)
+            header = deserialize_header(raw_header, index*2016 + i)
             self.verify_header(header, prev_header, bits, target)
             prev_header = header
 
-    def serialize_header(self, res):
-        s = int_to_hex(res.get('version'), 4) \
-            + rev_hex(res.get('prev_block_hash')) \
-            + rev_hex(res.get('merkle_root')) \
-            + int_to_hex(int(res.get('timestamp')), 4) \
-            + int_to_hex(int(res.get('bits')), 4) \
-            + int_to_hex(int(res.get('nonce')), 4)
-        return s
-
-    def deserialize_header(self, s, height):
-        hex_to_int = lambda s: int('0x' + s[::-1].encode('hex'), 16)
-        h = {}
-        h['version'] = hex_to_int(s[0:4])
-        h['prev_block_hash'] = hash_encode(s[4:36])
-        h['merkle_root'] = hash_encode(s[36:68])
-        h['timestamp'] = hex_to_int(s[68:72])
-        h['bits'] = hex_to_int(s[72:76])
-        h['nonce'] = hex_to_int(s[76:80])
-        h['block_height'] = height
-        return h
-
-    def hash_header(self, header):
-        if header is None:
-            return '0' * 64
-        return hash_encode(Hash(self.serialize_header(header).decode('hex')))
-
     def path(self):
-        return util.get_headers_path(self.config)
+        d = util.get_headers_dir(self.config)
-
+        return os.path.join(d, self.filename)
-    def init_headers_file(self):
-        filename = self.path()
-        try:
-            import urllib, socket
-            socket.setdefaulttimeout(30)
-            self.print_error("downloading ", bitcoin.HEADERS_URL)
-            urllib.urlretrieve(bitcoin.HEADERS_URL, filename + '.tmp')
-            os.rename(filename + '.tmp', filename)
-            self.print_error("done.")
-        except Exception:
-            self.print_error("download failed. creating file", filename)
-            open(filename, 'wb+').close()
-        self.downloading_headers = False
-        self.set_local_height()
-        self.print_error("%d blocks" % self.local_height)
 
     def save_chunk(self, index, chunk):
         filename = self.path()
         f = open(filename, 'rb+')
         f.seek(index * 2016 * 80)
+        f.truncate()
         h = f.write(chunk)
         f.close()
         self.set_local_height()
 
     def save_header(self, header):
-        data = self.serialize_header(header).decode('hex')
+        data = serialize_header(header).decode('hex')
         assert len(data) == 80
         height = header.get('block_height')
         filename = self.path()
         f = open(filename, 'rb+')
         f.seek(height * 80)
+        f.truncate()
         h = f.write(data)
         f.close()
         self.set_local_height()
@@ -184,9 +151,12 @@ class Blockchain(util.PrintError):
             h = f.read(80)
             f.close()
             if len(h) == 80:
-                h = self.deserialize_header(h, block_height)
+                h = deserialize_header(h, block_height)
                 return h
 
+    def get_hash(self, height):
+        return bitcoin.GENESIS if height == 0 else hash_header(self.read_header(height))
+
     def BIP9(self, height, flag):
         v = self.read_header(height)['version']
         return ((v & 0xE0000000) == 0x20000000) and ((v & flag) == flag)
@@ -195,15 +165,6 @@ class Blockchain(util.PrintError):
         h = self.local_height
         return sum([self.BIP9(h-i, 2) for i in range(N)])*10000/N/100.
 
-    def check_truncate_headers(self):
-        checkpoint = self.read_header(self.checkpoint_height)
-        if checkpoint is None:
-            return
-        if self.hash_header(checkpoint) == self.checkpoint_hash:
-            return
-        self.print_error('checkpoint mismatch:', self.hash_header(checkpoint), self.checkpoint_hash)
-        self.truncate_headers(self.checkpoint_height)
-
     def truncate_headers(self, height):
         self.print_error('Truncating headers file at height %d'%height)
         name = self.path()
@@ -212,6 +173,17 @@ class Blockchain(util.PrintError):
         f.truncate()
         f.close()
 
+    def fork(self, height):
+        import shutil
+        filename = "blockchain_fork_%d"%height
+        new_path = os.path.join(util.get_headers_dir(self.config), filename)
+        shutil.copy(self.path(), new_path)
+        with open(new_path, 'rb+') as f:
+            f.seek((height) * 80)
+            f.truncate()
+            f.close()
+        return filename
+
     def get_target(self, index, chain=None):
         if bitcoin.TESTNET:
             return 0, 0
@@ -255,7 +227,7 @@ class Blockchain(util.PrintError):
         previous_header = self.read_header(previous_height)
         if not previous_header:
             return False
-        prev_hash = self.hash_header(previous_header)
+        prev_hash = hash_header(previous_header)
         if prev_hash != header.get('prev_block_hash'):
             return False
         height = header.get('block_height')
@@ -270,21 +242,9 @@ class Blockchain(util.PrintError):
         try:
             data = hexdata.decode('hex')
             self.verify_chunk(idx, data)
-            self.print_error("validated chunk %d" % idx)
+            #self.print_error("validated chunk %d" % idx)
             self.save_chunk(idx, data)
             return True
         except BaseException as e:
             self.print_error('verify_chunk failed', str(e))
             return False
-
-    def get_checkpoint(self):
-        height = self.config.get('checkpoint_height', 0)
-        value = self.config.get('checkpoint_value', bitcoin.GENESIS)
-        return (height, value)
-
-    def set_checkpoint(self, height, value):
-        self.checkpoint_height = height
-        self.checkpoint_hash = value
-        self.config.set_key('checkpoint_height', height)
-        self.config.set_key('checkpoint_value', value)
-        self.check_truncate_headers()

lib/network.py

@@ -30,7 +30,7 @@ import random
 import select
 import traceback
 from collections import defaultdict, deque
-from threading import Lock
+import threading
 
 import socks
 import socket
@@ -204,7 +204,17 @@ class Network(util.DaemonThread):
         util.DaemonThread.__init__(self)
         self.config = SimpleConfig(config) if type(config) == type({}) else config
         self.num_server = 8 if not self.config.get('oneserver') else 0
-        self.blockchain = Blockchain(self.config, self)
+        self.blockchains = { 0:Blockchain(self.config, 0) }
+        for x in os.listdir(self.config.path):
+            if x.startswith('blockchain_fork_'):
+                n = int(x[16:])
+                b = Blockchain(self.config, n)
+                self.blockchains[n] = b
+        self.print_error("blockchains", self.blockchains.keys())
+        self.blockchain_index = config.get('blockchain_index', 0)
+        if self.blockchain_index not in self.blockchains.keys():
+            self.blockchain_index = 0
+
         # Server for addresses and transactions
         self.default_server = self.config.get('server')
         # Sanitize default server
@@ -215,13 +225,12 @@ class Network(util.DaemonThread):
         if not self.default_server:
             self.default_server = pick_random_server()
 
-        self.lock = Lock()
+        self.lock = threading.Lock()
         self.pending_sends = []
         self.message_id = 0
         self.debug = False
         self.irc_servers = {} # returned by interface (list from irc)
         self.recent_servers = self.read_recent_servers()
-        self.catch_up = None # interface catching up
 
         self.banner = ''
         self.donation_address = ''
@@ -493,14 +502,11 @@ class Network(util.DaemonThread):
         if servers:
             self.switch_to_interface(random.choice(servers))
 
-    def switch_lagging_interface(self, suggestion = None):
+    def switch_lagging_interface(self):
         '''If auto_connect and lagging, switch interface'''
         if self.server_is_lagging() and self.auto_connect:
-            if suggestion and self.protocol == deserialize_server(suggestion)[2]:
-                self.switch_to_interface(suggestion)
-            else:
             # switch to one that has the correct header (not height)
-                header = self.get_header(self.get_local_height())
+            header = self.blockchain().read_header(self.get_local_height())
             filtered = map(lambda x:x[0], filter(lambda x: x[1]==header, self.headers.items()))
             if filtered:
                 choice = random.choice(filtered)
@@ -688,15 +694,31 @@ class Network(util.DaemonThread):
             self.close_interface(self.interfaces[server])
             self.headers.pop(server, None)
             self.notify('interfaces')
-        if server == self.catch_up:
+        for b in self.blockchains.values():
-            self.catch_up = None
+            if b.catch_up == server:
+                b.catch_up = None
+
+    def get_checkpoint(self):
+        return max(self.blockchains.keys())
+
+    def get_blockchain(self, header):
+        from blockchain import hash_header
+        if type(header) is not dict:
+            return False
+        header_hash = hash_header(header)
+        height = header.get('block_height')
+        for b in self.blockchains.values():
+            if header_hash == b.get_hash(height):
+                return b
+        return False
 
     def new_interface(self, server, socket):
         self.add_recent_server(server)
         interface = Interface(server, socket)
+        interface.blockchain = None
         interface.mode = 'checkpoint'
         self.interfaces[server] = interface
-        self.request_header(interface, self.blockchain.checkpoint_height)
+        self.request_header(interface, self.get_checkpoint())
         if server == self.default_server:
             self.switch_to_interface(server)
         self.notify('interfaces')
@@ -758,26 +780,27 @@ class Network(util.DaemonThread):
         index = response['params'][0]
         if interface.request != index:
             return
-        connect = self.blockchain.connect_chunk(index, response['result'])
+        connect = interface.blockchain.connect_chunk(index, response['result'])
         # If not finished, get the next chunk
         if not connect:
             return
-        if self.get_local_height() < interface.tip:
+        if interface.blockchain.height() < interface.tip:
             self.request_chunk(interface, index+1)
         else:
             interface.request = None
+            interface.mode = 'default'
+            interface.print_error('catch up done')
+            interface.blockchain.catch_up = None
         self.notify('updated')
 
     def request_header(self, interface, height):
-        interface.print_error("requesting header %d" % height)
+        #interface.print_error("requesting header %d" % height)
         self.queue_request('blockchain.block.get_header', [height], interface)
         interface.request = height
         interface.req_time = time.time()
 
     def on_get_header(self, interface, response):
         '''Handle receiving a single block header'''
-        if self.blockchain.downloading_headers:
-            return
         header = response.get('result')
         if not header:
             interface.print_error(response)
@@ -790,19 +813,26 @@ class Network(util.DaemonThread):
             return
         self.on_header(interface, header)
     
+    def can_connect(self, header):
+        for blockchain in self.blockchains.values():
+            if blockchain.can_connect(header):
+                return blockchain
+
     def on_header(self, interface, header):
         height = header.get('block_height')
         if interface.mode == 'checkpoint':
-            if self.blockchain.pass_checkpoint(header):
+            b = self.get_blockchain(header)
+            if b:
                 interface.mode = 'default'
+                interface.blockchain = b
+                #interface.print_error('passed checkpoint', b.filename)
                 self.queue_request('blockchain.headers.subscribe', [], interface)
             else:
-                if interface != self.interface or self.auto_connect:
                 interface.print_error("checkpoint failed")
                 self.connection_down(interface.server)
             interface.request = None
             return
-        can_connect = self.blockchain.can_connect(header)
+        can_connect = self.can_connect(header)
         if interface.mode == 'backward':
             if can_connect:
                 interface.good = height
@@ -821,36 +851,56 @@ class Network(util.DaemonThread):
                 interface.good = height
             else:
                 interface.bad = height
-            if interface.good == interface.bad - 1:
+            if interface.bad != interface.good + 1:
-                interface.print_error("catching up from %d"% interface.good)
+                next_height = (interface.bad + interface.good) // 2
-                interface.mode = 'default'
+            else:
+                interface.print_error("found connection at %d"% interface.good)
+                delta1 = interface.blockchain.height() - interface.good
+                delta2 = interface.tip - interface.good
+                if delta1 > 10 and delta2 > 10:
+                    interface.print_error("chain split detected: %d (%d %d)"% (interface.good, delta1, delta2))
+                    interface.blockchain.fork(interface.bad)
+                    interface.blockchain = Blockchain(self.config, interface.bad)
+                    self.blockchains[interface.bad] = interface.blockchain
+                if interface.blockchain.catch_up is None:
+                    interface.blockchain.catch_up = interface.server
+                    interface.print_error("catching up")
+                    interface.mode = 'catch_up'
                     next_height = interface.good
                 else:
-                next_height = (interface.bad + interface.good) // 2
+                    # todo: if current catch_up is too slow, queue others
-        elif interface.mode == 'default':
+                    next_height = None
+        elif interface.mode == 'catch_up':
             if can_connect:
-                self.blockchain.save_header(header)
+                interface.blockchain.save_header(header)
                 self.notify('updated')
                 next_height = height + 1 if height < interface.tip else None
             else:
+                next_height = None
+
+            if next_height is None:
+                # exit catch_up state
+                interface.request = None
+                interface.mode = 'default'
+                interface.print_error('catch up done', interface.blockchain.catch_up)
+                interface.blockchain.catch_up = None
+
+        elif interface.mode == 'default':
+            assert not can_connect
             interface.print_error("cannot connect %d"% height)
             interface.mode = 'backward'
             interface.bad = height
+            # save height where we failed
+            interface.blockchain_height = interface.blockchain.height()
             next_height = height - 1
         else:
             raise BaseException(interface.mode)
         # If not finished, get the next header
         if next_height:
-            if interface.mode != 'default':
+            if interface.mode == 'catch_up' and interface.tip > next_height + 50:
-                self.request_header(interface, next_height)
-            else:
-                if interface.tip > next_height + 50:
                 self.request_chunk(interface, next_height // 2016)
             else:
                 self.request_header(interface, next_height)
-        else:
-            interface.request = None
-            self.catch_up = None
 
     def maintain_requests(self):
         for interface in self.interfaces.values():
@@ -879,8 +929,33 @@ class Network(util.DaemonThread):
         for interface in rout:
             self.process_responses(interface)
 
+    def init_headers_file(self):
+        filename = self.blockchains[0].path()
+        if os.path.exists(filename):
+            self.downloading_headers = False
+            return
+        def download_thread():
+            try:
+                import urllib, socket
+                socket.setdefaulttimeout(30)
+                self.print_error("downloading ", bitcoin.HEADERS_URL)
+                urllib.urlretrieve(bitcoin.HEADERS_URL, filename + '.tmp')
+                os.rename(filename + '.tmp', filename)
+                self.print_error("done.")
+            except Exception:
+                self.print_error("download failed. creating file", filename)
+                open(filename, 'wb+').close()
+            self.downloading_headers = False
+            self.blockchains[0].set_local_height()
+        self.downloading_headers = True
+        t = threading.Thread(target = download_thread)
+        t.daemon = True
+        t.start()
+
     def run(self):
-        self.blockchain.init()
+        self.init_headers_file()
+        while self.is_running() and self.downloading_headers:
+            time.sleep(1)
         while self.is_running():
             self.maintain_sockets()
             self.wait_on_sockets()
@@ -890,35 +965,51 @@ class Network(util.DaemonThread):
         self.stop_network()
         self.on_stop()
 
-    def on_notify_header(self, i, header):
+    def on_notify_header(self, interface, header):
         height = header.get('block_height')
         if not height:
             return
-        self.headers[i.server] = header
+        self.headers[interface.server] = header
-        i.tip = height
+        interface.tip = height
-        local_height = self.get_local_height()
+        local_height = interface.blockchain.height()
-
+        if interface.mode != 'default':
-        if i.tip > local_height:
+            return
-            i.print_error("better height", height)
+        if interface.tip > local_height + 1:
-            # if I can connect, do it right away
+            if interface.blockchain.catch_up is None:
-            if self.blockchain.can_connect(header):
+                interface.blockchain.catch_up = interface.server
-                self.blockchain.save_header(header)
+                interface.mode = 'catch_up' # must transition to search if it does not connect
+                self.request_header(interface, local_height + 1)
+            else:
+                # another interface is catching up
+                pass
+        elif interface.tip == local_height + 1:
+            if interface.blockchain.can_connect(header):
+                interface.blockchain.save_header(header)
                 self.notify('updated')
-            # otherwise trigger a search
+            else:
-            elif self.catch_up is None:
+                interface.mode = 'backward'
-                self.catch_up = i.server
+                interface.bad = height 
-                self.on_header(i, header)
+                self.request_header(interface, local_height)
-
+        else:
-        if i == self.interface:
+            if not interface.blockchain.can_connect(header):
+                interface.mode = 'backward'
+                interface.bad = height
+                self.request_header(interface, height - 1)
+            else:
+                pass
+        if interface == self.interface:
             self.switch_lagging_interface()
             self.notify('updated')
 
+    def blockchain(self):
+        if self.interface and self.interface.blockchain is not None:
+            self.blockchain_index = self.interface.blockchain.checkpoint
+            self.config.set_key('blockchain_index', self.blockchain_index)
 
-    def get_header(self, tx_height):
+        return self.blockchains[self.blockchain_index]
-        return self.blockchain.read_header(tx_height)
 
     def get_local_height(self):
-        return self.blockchain.height()
+        return self.blockchain().height()
 
     def synchronous_get(self, request, timeout=30):
         queue = Queue.Queue()

lib/util.py

@@ -213,12 +213,11 @@ def android_data_dir():
     PythonActivity = jnius.autoclass('org.kivy.android.PythonActivity')
     return PythonActivity.mActivity.getFilesDir().getPath() + '/data'
 
-def android_headers_path():
+def android_headers_dir():
-    path = android_ext_dir() + '/org.electrum.electrum/blockchain_headers'
+    d = android_ext_dir() + '/org.electrum.electrum'
-    d = os.path.dirname(path)
     if not os.path.exists(d):
         os.mkdir(d)
-    return path
+    return d
 
 def android_check_data_dir():
     """ if needed, move old directory to sandbox """
@@ -227,7 +226,7 @@ def android_check_data_dir():
     old_electrum_dir = ext_dir + '/electrum'
     if not os.path.exists(data_dir) and os.path.exists(old_electrum_dir):
         import shutil
-        new_headers_path = android_headers_path()
+        new_headers_path = android_headers_dir() + '/blockchain_headers'
         old_headers_path = old_electrum_dir + '/blockchain_headers'
         if not os.path.exists(new_headers_path) and os.path.exists(old_headers_path):
             print_error("Moving headers file to", new_headers_path)
@@ -236,11 +235,8 @@ def android_check_data_dir():
         shutil.move(old_electrum_dir, data_dir)
     return data_dir
 
-def get_headers_path(config):
+def get_headers_dir(config):
-    if 'ANDROID_DATA' in os.environ:
+    return android_headers_dir() if 'ANDROID_DATA' in os.environ else config.path
-        return android_headers_path()
-    else:
-        return os.path.join(config.path, 'blockchain_headers')
 
 def user_dir():
     if 'ANDROID_DATA' in os.environ:

lib/verifier.py

@@ -64,7 +64,7 @@ class SPV(ThreadJob):
         tx_height = merkle.get('block_height')
         pos = merkle.get('pos')
         merkle_root = self.hash_merkle_root(merkle['merkle'], tx_hash, pos)
-        header = self.network.get_header(tx_height)
+        header = self.network.blockchain().read_header(tx_height)
         if not header or header.get('merkle_root') != merkle_root:
             # FIXME: we should make a fresh connection to a server to
             # recover from this, as this TX will now never verify