Update and Bugfix

.travis.yml

@@ -1,4 +1,4 @@
-dist: xenial
+dist: bionic
 language: python
 python:
     - 3.6
@@ -6,15 +6,10 @@ python:
     - 3.8
 git:
   depth: false
-addons:
-  apt:
-    sources:
-      - sourceline: 'ppa:tah83/secp256k1'
-    packages:
-      - libsecp256k1-0
 before_install:
   - git tag
 install:
+  - sudo apt-get -y install libsecp256k1-0
   - pip install -r contrib/requirements/requirements-travis.txt
 cache:
   - pip: true
@@ -31,11 +26,12 @@ jobs:
       language: python
       python: 3.7
       before_install:
-        - sudo add-apt-repository -y ppa:bitcoin/bitcoin
+        - sudo add-apt-repository -y ppa:luke-jr/bitcoincore
         - sudo apt-get -qq update
         - sudo apt-get install -yq bitcoind
       install:
-        - pip install -r contrib/requirements/requirements.txt
+        - sudo apt-get -y install libsecp256k1-0
+        - pip install .[tests]
         - pip install electrumx
       before_script:
           - electrum/tests/regtest/start_bitcoind.sh
@@ -48,7 +44,7 @@ jobs:
       install: pip install flake8
       script: flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
     - stage: binary builds
-      if: branch = master
+      if: (branch = master) OR (tag IS present)
       name: "Windows build"
       language: c
       python: false
@@ -61,7 +57,7 @@ jobs:
       script:
         - sudo docker run --name electrum-wine-builder-cont -v $PWD:/opt/wine64/drive_c/electrum --rm --workdir /opt/wine64/drive_c/electrum/contrib/build-wine electrum-wine-builder-img ./build.sh
       after_success: true
-    - if: branch = master
+    - if: (branch = master) OR (tag IS present)
       name: "Android build"
       language: python
       python: 3.7
@@ -70,18 +66,18 @@ jobs:
       install:
         - pip install requests && ./contrib/pull_locale
         - ./contrib/make_packages
-        - sudo docker build --no-cache -t electrum-android-builder-img electrum/gui/kivy/tools
+        - sudo docker build --no-cache -t electrum-android-builder-img contrib/android
       script:
         - sudo chown -R 1000:1000 .
         # Output something every minute or Travis kills the job
         - while sleep 60; do echo "=====[ $SECONDS seconds still running ]====="; done &
-        - sudo docker run -it -u 1000:1000 --rm --name electrum-android-builder-cont --env CI=true -v $PWD:/home/user/wspace/electrum --workdir /home/user/wspace/electrum electrum-android-builder-img ./contrib/make_apk
+        - sudo docker run -it -u 1000:1000 --rm --name electrum-android-builder-cont --env CI=true -v $PWD:/home/user/wspace/electrum --workdir /home/user/wspace/electrum electrum-android-builder-img ./contrib/android/make_apk
         # kill background sleep loop
         - kill %1
         - ls -la bin
         - if [ $(ls bin | grep -c Electrum-*) -eq 0 ]; then exit 1; fi
       after_success: true
-    - if: branch = master
+    - if: (branch = master) OR (tag IS present)
       name: "MacOS build"
       os: osx
       language: c
@@ -93,7 +89,7 @@ jobs:
       script: ./contrib/osx/make_osx
       after_script: ls -lah dist && md5 dist/*
       after_success: true
-    - if: branch = master
+    - if: (branch = master) OR (tag IS present)
       name: "AppImage build"
       language: c
       python: false
@@ -104,6 +100,26 @@ jobs:
       script:
         - sudo docker run --name electrum-appimage-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/appimage electrum-appimage-builder-img ./build.sh
       after_success: true
+      - if: (branch = master) OR (tag IS present)
+      name: "tarball build"
+      language: c
+      python: false
+      services:
+        - docker
+      before_install:
+        # hack: travis already cloned the repo, but we re-clone now, as we need to have umask set BEFORE cloning
+        - umask 0022
+        - mkdir fresh_clone && cd fresh_clone
+        - git clone https://github.com/$TRAVIS_REPO_SLUG.git && cd electrum
+        - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then git fetch origin pull/$TRAVIS_PULL_REQUEST/merge; fi
+        - git checkout $TRAVIS_COMMIT
+        - echo "Second git clone ready at $PWD"
+      install:
+        - sudo docker build --no-cache -t electrum-sdist-builder-img ./contrib/build-linux/sdist/
+      script:
+        - echo "Building sdist at $PWD"
+        - sudo docker run --name electrum-sdist-builder-cont -v $PWD:/opt/electrum --rm --workdir /opt/electrum/contrib/build-linux/sdist electrum-sdist-builder-img ./build.sh
+      after_success: true
     - stage: release check
       install:
           - git fetch --all --tags

README.rst

@@ -8,25 +8,45 @@ https://kodxana.github.io/LBRY-Vault-website/
 Getting started
 ===============
 
-LBRY Vault itself is pure Python, and so are most of the required dependencies.
+LBRY Vault itself is pure Python, and so are most of the required dependencies,
+but not everything. The following sections describe how to run from source, but here
+is a TL;DR::
 
-Non-python dependencies
+    sudo apt-get install libsecp256k1-0
------------------------
+    python3 -m pip install --user .[gui,crypto]
+
+
+Not pure-python dependencies
+----------------------------
 
 If you want to use the Qt interface, install the Qt dependencies::
 
     sudo apt-get install python3-pyqt5
 
-For elliptic curve operations, libsecp256k1 is a required dependency::
+For elliptic curve operations, `libsecp256k1`_ is a required dependency::
 
     sudo apt-get install libsecp256k1-0
 
 Alternatively, when running from a cloned repository, a script is provided to build
 libsecp256k1 yourself::
-
+    sudo apt-get install automake libtool
     ./contrib/make_libsecp256k1.sh
 
 
+    Due to the need for fast symmetric ciphers, either one of `pycryptodomex`_
+    or `cryptography`_ is required. Install from your package manager
+    (or from pip)::
+
+        sudo apt-get install python3-cryptography
+
+
+    If you would like hardware wallet support, see `this`_.
+
+    .. _libsecp256k1: https://github.com/bitcoin-core/secp256k1
+    .. _pycryptodomex: https://github.com/Legrandin/pycryptodome
+    .. _cryptography: https://github.com/pyca/cryptography
+    .. _this: https://github.com/spesmilo/electrum-docs/blob/master/hardware-linux.rst
+
 Running from tar.gz
 -------------------
 
@@ -45,9 +65,8 @@ You can also install Electrum on your system, by running this command::
 This will download and install the Python dependencies used by
 LBRY Vault instead of using the 'packages' directory.
 
-If you cloned the git repository, you need to compile extra files
+It will also place an executable named :code:`electrum` in :code:`~/.local/bin`,
-before you can run LBRY Vault. Read the next section, "Development
+so make sure that is on your :code:`PATH` variable.
-version".
 
 
 Development version
@@ -61,7 +80,7 @@ Check out the code from GitHub::
 
 Run install (this should install dependencies)::
 
-    python3 -m pip install --user .
+     python3 -m pip install --user -e .
 
 
 Compile the protobuf description file::
@@ -74,6 +93,9 @@ Create translations (optional)::
     sudo apt-get install python-requests gettext
     ./contrib/pull_locale
 
+    Finally, to start Electrum::
+
+        ./run_electrum
 
 
 
@@ -83,7 +105,7 @@ Creating Binaries
 Linux (tarball)
 ---------------
 
-See :code:`contrib/build-linux/README.md`.
+See :code:`contrib/build-linux/sdist/README.md`.
 
 
 Linux (AppImage)
@@ -107,4 +129,4 @@ See :code:`contrib/build-wine/README.md`.
 Android
 -------
 
-See :code:`electrum/gui/kivy/Readme.md`.
+See :code:`contrib/android/Readme.md`.

contrib/build-linux/appimage/Dockerfile

@@ -1,29 +1,30 @@
-FROM ubuntu:16.04@sha256:97b54e5692c27072234ff958a7442dde4266af21e7b688e7fca5dc5acc8ed7d9
+ROM ubuntu:16.04@sha256:a4fc0c40360ff2224db3a483e5d80e9164fe3fdce2a8439d2686270643974632
 
 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
 
 RUN apt-get update -q && \
     apt-get install -qy \
-        git=1:2.7.4-0ubuntu1.7 \
+        git=1:2.7.4-0ubuntu1.9 \
         wget=1.17.1-1ubuntu1.5 \
         make=4.1-6 \
         autotools-dev=20150820.1 \
         autoconf=2.69-9 \
         libtool=2.4.6-0.1 \
         xz-utils=5.1.1alpha+20120614-2ubuntu2 \
-        libssl-dev=1.0.2g-1ubuntu4.15 \
+        libssl-dev=1.0.2g-1ubuntu4.16 \
-        libssl1.0.0=1.0.2g-1ubuntu4.15 \
+        libssl1.0.0=1.0.2g-1ubuntu4.16 \
-        openssl=1.0.2g-1ubuntu4.15 \
+        openssl=1.0.2g-1ubuntu4.16 \
         zlib1g-dev=1:1.2.8.dfsg-2ubuntu4.3 \
         libffi-dev=3.2.1-4 \
         libncurses5-dev=6.0+20160213-1ubuntu1 \
-        libsqlite3-dev=3.11.0-1ubuntu1.3 \
+        libsqlite3-dev=3.11.0-1ubuntu1.5 \
         libusb-1.0-0-dev=2:1.0.20-1 \
-        libudev-dev=229-4ubuntu21.27 \
+        libudev-dev=229-4ubuntu21.28 \
         gettext=0.19.7-2ubuntu3.1 \
         libzbar0=0.10+doc-10ubuntu1  \
-        libdbus-1-3=1.10.6-1ubuntu3.4 \
+        libdbus-1-3=1.10.6-1ubuntu3.6 \
         libxkbcommon-x11-0=0.5.0-1ubuntu2.1 \
+        libc6-dev=2.23-0ubuntu11.2 \
         && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get autoremove -y && \

contrib/build-linux/appimage/README.md

@@ -61,6 +61,13 @@ diff sha256sum1 sha256sum2 > d
 cat d
 ```
 
+For file metadata, e.g. timestamps:
+```
+rsync -n -a -i --delete squashfs-root1/ squashfs-root2/
+```
+
+
+
 Useful binary comparison tools:
 - vbindiff
 - diffoscope

contrib/build-linux/appimage/build.sh

@@ -13,7 +13,7 @@ CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage"
 export GCC_STRIP_BINARIES="1"
 
 # pinned versions
-PYTHON_VERSION=3.7.6
+PYTHON_VERSION=3.7.7
 PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15"
 SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386"
 
@@ -38,7 +38,7 @@ download_if_not_exist "$CACHEDIR/appimagetool" "https://github.com/AppImage/AppI
 verify_hash "$CACHEDIR/appimagetool" "d918b4df547b388ef253f3c9e7f6529ca81a885395c31f619d9aaf7030499a13"
 
 download_if_not_exist "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tar.xz"
-verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "55a2cce72049f0794e9a11a84862e9039af9183603b78bc60d89539f82cf533f"
+verify_hash "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" "06a0a9f1bf0d8cd1e4121194d666c4e28ddae4dd54346de6c343206599f02136"
 
 
 
@@ -71,7 +71,7 @@ info "Building squashfskit"
 git clone "https://github.com/squashfskit/squashfskit.git" "$BUILDDIR/squashfskit"
 (
     cd "$BUILDDIR/squashfskit"
-    git checkout "$SQUASHFSKIT_COMMIT"
+    git checkout "${SQUASHFSKIT_COMMIT}^{commit}"
     make -C squashfs-tools mksquashfs || fail "Could not build squashfskit"
 )
 MKSQUASHFS="$BUILDDIR/squashfskit/squashfs-tools/mksquashfs"
@@ -206,13 +206,11 @@ rm -rf "$PYDIR"/site-packages/PyQt5/Qt.so
 
 # these are deleted as they were not deterministic; and are not needed anyway
 find "$APPDIR" -path '*/__pycache__*' -delete
-# note that jsonschema-*.dist-info is needed by that package as it uses 'pkg_resources.get_distribution'
+# note that *.dist-info is needed by certain packages.
-# also, see https://gitlab.com/python-devs/importlib_metadata/issues/71
+# e.g. see https://gitlab.com/python-devs/importlib_metadata/issues/71
-for f in "$PYDIR"/site-packages/jsonschema-*.dist-info; do mv "$f" "$(echo "$f" | sed s/\.dist-info/\.dist-info2/)"; done
 for f in "$PYDIR"/site-packages/importlib_metadata-*.dist-info; do mv "$f" "$(echo "$f" | sed s/\.dist-info/\.dist-info2/)"; done
 rm -rf "$PYDIR"/site-packages/*.dist-info/
 rm -rf "$PYDIR"/site-packages/*.egg-info/
-for f in "$PYDIR"/site-packages/jsonschema-*.dist-info2; do mv "$f" "$(echo "$f" | sed s/\.dist-info2/\.dist-info/)"; done
 for f in "$PYDIR"/site-packages/importlib_metadata-*.dist-info2; do mv "$f" "$(echo "$f" | sed s/\.dist-info2/\.dist-info/)"; done
 
 

contrib/build-linux/appimage/sdist/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:20.04@sha256:5747316366b8cc9e3021cd7286f42b2d6d81e3d743e2ab571f55bcd5df788cc8
+
+ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -q && \
+    apt-get install -qy \
+        git \
+        gettext \
+        python3 \
+        python3-pip \
+        python3-setuptools \
+        faketime \
+        && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get autoremove -y && \
+    apt-get clean

contrib/build-linux/appimage/sdist/README.md

@@ -0,0 +1,52 @@
+Source tarballs
+===============
+
+✓ _This file should be reproducible, meaning you should be able to generate
+   distributables that match the official releases._
+
+This assumes an Ubuntu (x86_64) host, but it should not be too hard to adapt to another
+similar system. The docker commands should be executed in the project's root
+folder.
+
+1. Install Docker
+
+    ```
+    $ curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+    $ sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
+    $ sudo apt-get update
+    $ sudo apt-get install -y docker-ce
+    ```
+
+2. Build image
+
+    ```
+    $ sudo docker build -t electrum-sdist-builder-img contrib/build-linux/sdist
+    ```
+
+3. Build source tarballs
+
+    It's recommended to build from a fresh clone
+    (but you can skip this if reproducibility is not necessary).
+
+    ```
+    $ FRESH_CLONE=contrib/build-linux/sdist/fresh_clone && \
+        sudo rm -rf $FRESH_CLONE && \
+        umask 0022 && \
+        mkdir -p $FRESH_CLONE && \
+        cd $FRESH_CLONE  && \
+        git clone https://github.com/spesmilo/electrum.git && \
+        cd electrum
+    ```
+
+    And then build from this directory:
+    ```
+    $ git checkout $REV
+    $ sudo docker run -it \
+        --name electrum-sdist-builder-cont \
+        -v $PWD:/opt/electrum \
+        --rm \
+        --workdir /opt/electrum/contrib/build-linux/sdist \
+        electrum-sdist-builder-img \
+        ./build.sh
+    ```
+4. The generated distributables are in `./dist`.

contrib/build-linux/appimage/sdist/build.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -e
+
+PROJECT_ROOT="$(dirname "$(readlink -e "$0")")/../../.."
+CONTRIB="$PROJECT_ROOT/contrib"
+CONTRIB_SDIST="$CONTRIB/build-linux/sdist"
+DISTDIR="$PROJECT_ROOT/dist"
+
+. "$CONTRIB"/build_tools_util.sh
+
+# note that at least py3.7 is needed, to have https://bugs.python.org/issue30693
+python3 --version || fail "python interpreter not found"
+
+# upgrade to modern pip so that it knows the flags we need.
+# we will then install a pinned version of pip as part of requirements-sdist-build
+python3 -m pip install --upgrade pip
+
+info "Installing pinned requirements."
+python3 -m pip install --no-dependencies --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-sdist-build.txt
+
+
+"$CONTRIB"/make_packages || fail "make_packages failed"
+
+"$CONTRIB_SDIST"/make_tgz || fail "make_tgz failed"
+
+
+info "done."
+ls -la "$DISTDIR"
+sha256sum "$DISTDIR"/*

contrib/build-linux/appimage/sdist/make_tgz

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -e
+
+ONTRIB_SDIST="$(dirname "$(readlink -e "$0")")"
+CONTRIB="$CONTRIB_SDIST"/../..
+ROOT_FOLDER="$CONTRIB"/..
+PACKAGES="$ROOT_FOLDER"/packages/
+LOCALE="$ROOT_FOLDER"/electrum/locale/
+
+if [ ! -d "$PACKAGES" ]; then
+  echo "Run make_packages first!"
+  exit 1
+fi
+
+git submodule update --init
+
+(
+    rm -rf "$LOCALE"
+    cd "$CONTRIB/deterministic-build/electrum-locale/"
+    if ! which msgfmt > /dev/null 2>&1; then
+        echo "Please install gettext"
+        exit 1
+    fi
+    for i in ./locale/*; do
+        dir="$ROOT_FOLDER"/electrum/$i/LC_MESSAGES
+        mkdir -p $dir
+        msgfmt --output-file=$dir/electrum.mo $i/electrum.po || true
+        cp $i/electrum.po "$ROOT_FOLDER"/electrum/$i/electrum.po
+    done
+)
+
+(
+    cd "$ROOT_FOLDER"
+
+    echo "'git clean -fd' would delete the following files: >>>"
+    git clean -fd --dry-run
+    echo "<<<"
+
+    # we could build the kivy atlas potentially?
+    #(cd electrum/gui/kivy/; make theming) || echo "building kivy atlas failed! skipping."
+
+    find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
+
+    # note: .zip sdists would not be reproducible due to https://bugs.python.org/issue40963
+    TZ=UTC faketime -f '2000-11-11 11:11:11' python3 setup.py --quiet sdist --format=gztar
+)

contrib/build-wine/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04@sha256:5f4bdc3467537cbbe563e80db2c3ec95d548a9145d64453b06939c4592d67b6d
+FROM ubuntu:18.04@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
 
 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8
 
@@ -13,7 +13,7 @@ RUN dpkg --add-architecture i386 && \
 
 RUN apt-get update -q && \
         apt-get install -qy \
-        git=1:2.17.1-1ubuntu0.5 \
+        git=1:2.17.1-1ubuntu0.7 \
         p7zip-full=16.02+dfsg-6 \
         make=4.1-9.1ubuntu1 \
         mingw-w64=5.0.3-1 \

contrib/build-wine/build-electrum-git.sh

@@ -52,11 +52,6 @@ info "Pip installing Electrum. This might take a long time if the project folder
 $PYTHON -m pip install --no-dependencies --no-warn-script-location .
 popd
 
-
-# these are deleted as they were not deterministic; and are not needed anyway
-rm "$WINEPREFIX"/drive_c/python3/Lib/site-packages/jsonschema-*.dist-info/RECORD
-
-
 rm -rf dist/
 
 # build standalone and portable versions

contrib/build-wine/deterministic.spec

@@ -16,19 +16,16 @@ home = 'C:\\electrum\\'
 
 # see https://github.com/pyinstaller/pyinstaller/issues/2005
 hiddenimports = []
+hiddenimports += collect_submodules('pkg_resources')  # workaround for https://github.com/pypa/setuptools/issues/1963
 hiddenimports += collect_submodules('trezorlib')
 hiddenimports += collect_submodules('safetlib')
 hiddenimports += collect_submodules('btchip')
 hiddenimports += collect_submodules('keepkeylib')
 hiddenimports += collect_submodules('websocket')
 hiddenimports += collect_submodules('ckcc')
+hiddenimports += collect_submodules('bitbox02')
 hiddenimports += ['PyQt5.QtPrintSupport']  # needed by Revealer
 
-# safetlib imports PyQt5.Qt.  We use a local updated copy of pinmatrix.py until they
-# release a new version that includes https://github.com/archos-safe-t/python-safet/commit/b1eab3dba4c04fdfc1fcf17b66662c28c5f2380e
-hiddenimports.remove('safetlib.qt.pinmatrix')
-
-
 binaries = []
 
 # Workaround for "Retro Look":
@@ -39,6 +36,7 @@ binaries += [('C:/tmp/libusb-1.0.dll', '.')]
 
 datas = [
     (home+'electrum/*.json', 'electrum'),
+    (home+'electrum/lnwire/*.csv', 'electrum/lnwire'),
     (home+'electrum/wordlist/english.txt', 'electrum/wordlist'),
     (home+'electrum/locale', 'electrum/locale'),
     (home+'electrum/plugins', 'electrum/plugins'),
@@ -50,8 +48,7 @@ datas += collect_data_files('safetlib')
 datas += collect_data_files('btchip')
 datas += collect_data_files('keepkeylib')
 datas += collect_data_files('ckcc')
-datas += collect_data_files('jsonrpcserver')
+datas += collect_data_files('bitbox02')
-datas += collect_data_files('jsonrpcclient')
 
 # We don't put these files in to actually include them in the script but to make the Analysis method scan them for imports
 a = Analysis([home+'run_electrum',

contrib/build-wine/prepare-wine.sh

@@ -10,14 +10,14 @@ ZBAR_URL=https://sourceforge.net/projects/zbarw/files/$ZBAR_FILENAME/download
 ZBAR_SHA256=177e32b272fa76528a3af486b74e9cb356707be1c5ace4ed3fcee9723e2c2c02
 
 LIBUSB_REPO="https://github.com/libusb/libusb.git"
-LIBUSB_COMMIT=e782eeb2514266f6738e242cdcb18e3ae1ed06fa
+LIBUSB_COMMIT="e782eeb2514266f6738e242cdcb18e3ae1ed06fa"
 # ^ tag v1.0.23
 
 PYINSTALLER_REPO="https://github.com/SomberNight/pyinstaller.git"
-PYINSTALLER_COMMIT=e934539374e30d1500fcdbe8e4eb0860413935b2
+PYINSTALLER_COMMIT="e934539374e30d1500fcdbe8e4eb0860413935b2"
 # ^ tag 3.6, plus a custom commit that fixes cross-compilation with MinGW
 
-PYTHON_VERSION=3.6.8
+PYTHON_VERSION=3.7.7
 
 ## These settings probably don't need change
 export WINEPREFIX=/opt/wine64
@@ -88,7 +88,7 @@ info "Compiling libusb..."
     git init
     git remote add origin $LIBUSB_REPO
     git fetch --depth 1 origin $LIBUSB_COMMIT
-    git checkout -b pinned FETCH_HEAD
+    git checkout -b pinned "${LIBUSB_COMMIT}^{commit}"
     echo "libusb_1_0_la_LDFLAGS += -Wc,-static" >> libusb/Makefile.am
     ./bootstrap.sh || fail "Could not bootstrap libusb"
     host="i686-w64-mingw32"
@@ -119,7 +119,7 @@ info "Building PyInstaller."
     git init
     git remote add origin $PYINSTALLER_REPO
     git fetch --depth 1 origin $PYINSTALLER_COMMIT
-    git checkout -b pinned FETCH_HEAD
+    git checkout -b pinned "${PYINSTALLER_COMMIT}^{commit}"
     rm -fv PyInstaller/bootloader/Windows-*/run*.exe || true
     # add reproducible randomness. this ensures we build a different bootloader for each commit.
     # if we built the same one for all releases, that might also get anti-virus false positives

contrib/build-wine/sign.sh

@@ -23,6 +23,7 @@ echo "Found $(ls *.exe | wc -w) files to sign."
 for f in $(ls *.exe); do
     echo "Signing $f..."
     osslsigncode sign \
+      -h sha256 \
       -certs "$CERT_FILE" \
       -key "$KEY_FILE" \
       -n "Electrum" \

contrib/deterministic-build/requirements-binaries.txt

@@ -1,27 +1,64 @@
-pip==19.3.1 \
+pip==20.1.1 \
-    --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
-    --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
-PyQt5==5.11.3 \
+pycryptodomex==3.9.7 \
-    --hash=sha256:517e4339135c4874b799af0d484bc2e8c27b54850113a68eec40a0b56534f450 \
+    --hash=sha256:1537d2d15b604b303aef56e7f440895a1c81adbee786b91f1f06eddc34da5314 \
-    --hash=sha256:ac1eb5a114b6e7788e8be378be41c5e54b17d5158994504e85e43b5fca006a39 \
+    --hash=sha256:1d20ab8369b7558168fc014a0745c678613f9f486dae468cca2d68145196b8a4 \
-    --hash=sha256:d2309296a5a79d0a1c0e6c387c30f0398b65523a6dcc8a19cc172e46b949e00d \
+    --hash=sha256:1ecc9db7409db67765eb008e558879d298406642d33ade43a6488224d23e8081 \
-    --hash=sha256:e85936bae1581bcb908847d2038e5b34237a5e6acc03130099a78930770e7ead
+    --hash=sha256:37033976f72af829fe15f7fe5fe1dbed308cc43a98d9dd9d2a0a76de8ca5ee78 \
-PyQt5-sip==4.19.13 \
+    --hash=sha256:3c3dd9d4c9c1e279d3945ae422895c901f98987333acc132dc094faf52afec35 \
-    --hash=sha256:125f77c087572c9272219cda030a63c2f996b8507592b2a54d7ef9b75f9f054d \
+    --hash=sha256:3c9b3fba037ea52c626060c5a87ee6de7e86c99e8a7c6ee07302539985d2bd64 \
-    --hash=sha256:14c37b06e3fb7c2234cb208fa461ec4e62b4ba6d8b32ca3753c0b2cfd61b00e3 \
+    --hash=sha256:45ee555fc5e28c119a46d44ce373f5237e54a35c61b750fb3a94446b09855dbc \
-    --hash=sha256:1cb2cf52979f9085fc0eab7e0b2438eb4430d4aea8edec89762527e17317175b \
+    --hash=sha256:4c93038ac011b36512cb0bf2ee3e2aec774e8bc81021d015917c89fe02bb0ee5 \
-    --hash=sha256:4babef08bccbf223ec34464e1ed0a23caeaeea390ca9a3529227d9a57f0d6ee4 \
+    --hash=sha256:50163324834edd0c9ce3e4512ded3e221c969086e10fdd5d3fdcaadac5e24a78 \
-    --hash=sha256:53cb9c1208511cda0b9ed11cffee992a5a2f5d96eb88722569b2ce65ecf6b960 \
+    --hash=sha256:59b0ea9cda5490f924771456912a225d8d9e678891f9f986661af718534719b2 \
-    --hash=sha256:549449d9461d6c665cbe8af4a3808805c5e6e037cd2ce4fd93308d44a049bfac \
+    --hash=sha256:5cf306a17cccc327a33cdc3845629fa13f4573a4ec620ed607c79cf6785f2e27 \
-    --hash=sha256:5f5b3089b200ff33de3f636b398e7199b57a6b5c1bb724bdb884580a072a14b5 \
+    --hash=sha256:5fff8da399af16a1855f58771223acbbdac720b9969cd03fc5013d2e9a7bd9a4 \
-    --hash=sha256:a4d9bf6e1fa2dd6e73f1873f1a47cee11a6ba0cf9ba8cf7002b28c76823600d0 \
+    --hash=sha256:68650ce5b9f7152b8283302a4617269f821695a612692640dd247bd12ab21c0b \
-    --hash=sha256:a4ee6026216f1fbe25c8847f9e0fbce907df5b908f84816e21af16ec7666e6fe \
+    --hash=sha256:6b3a9a562688996f760b5077714c3ab8b62ca56061b6e9ab7906841e43e19f91 \
-    --hash=sha256:a91a308a5e0cc99de1e97afd8f09f46dd7ca20cfaa5890ef254113eebaa1adff \
+    --hash=sha256:7e938ed51a59e29431ea86fab60423ada2757728db0f78952329fa02a789bd31 \
-    --hash=sha256:b0342540da479d2713edc68fb21f307473f68da896ad5c04215dae97630e0069 \
+    --hash=sha256:87aa70daad6f039e814790a06422a3189311198b674b62f13933a2bdcb6b1bcc \
-    --hash=sha256:f997e21b4e26a3397cb7b255b8d1db5b9772c8e0c94b6d870a5a0ab5c27eacaa
+    --hash=sha256:99be3a1df2b2b9f731ebe1c264a2c07c465e71cee68e35e1640b645b5213a755 \
-setuptools==42.0.2 \
+    --hash=sha256:a3f2908666e6f74b8c4893f86dd02e16170f50e4a78ae7f3468b6208d54bc205 \
-    --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \
+    --hash=sha256:ae3d44a639fd11dbdeca47e35e94febb1ee8bc15daf26673331add37146e0b85 \
-    --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6
+    --hash=sha256:afb4c2fa3c6f492fd9a8b38d76e13f32d429b8e5e1e00238309391b5591cde0d \
-wheel==0.33.6 \
+    --hash=sha256:b1515ce3a8a2c3fa537d137c5ca5f8b7a902044d04e07d7c3aa26c3e026120fb \
-    --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \
+    --hash=sha256:bf391b377413a197000b43ef2b74359974d8927d329a897c9f5ba7b63dca7b9c \
-    --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28
+    --hash=sha256:c436919117c23355740c669f89720673578b9aa4569bbfe105f6c10101fc1966 \
+    --hash=sha256:d2c3c280975638e2a2c2fd9cb36ab111980219757fa163a2755594b9448e4138 \
+    --hash=sha256:e585d530764c459cbd5d460aed0288807bb881f376ca9a20e653645217895961 \
+    --hash=sha256:e76e6638ead4a7d93262a24218f0ff3ff74de6b6c823b7e19dccb31b6a481978 \
+    --hash=sha256:ebfc2f885cafda076c31ae30fa0dd81e7e919ec34059a88d3018ed66e83fcce3 \
+    --hash=sha256:f5797a39933a3d41526da60856735e6684b2b71a8ca99d5f79555ca121be2f4b \
+    --hash=sha256:f7e5fc5e124200b19a14be33fb0099e956e6ebb5e25d287b0829ef0a78ed76c7 \
+    --hash=sha256:fb350e31e55211fec8ddc89fc0256f3b9bc3b44b68a8bde1cf44b3b4e80c0e42
+PyQt5==5.14.2 \
+    --hash=sha256:3b91dd1d0cbfaea85ad057247ba621187e511434b0c9d6d40de69fd5e833b109 \
+    --hash=sha256:a9bdc46ab1f6397770e6b8dca84ac07a0250d26b1a31587f25619cf31a075532 \
+    --hash=sha256:bd230c6fd699eabf1ceb51e13a8b79b74c00a80272c622427b80141a22269eb0 \
+    --hash=sha256:ee168a486c9a758511568147815e2959652cd0aabea832fa5e87cf6b241d2180 \
+    --hash=sha256:f61ddc78547d6ca763323ccd4a9e374c71b29feda1f5ce2d3e91e4f8d2cf1942
+PyQt5-sip==12.8.0 \
+    --hash=sha256:0a34b6596bdd28d52da3a51fa8d9bb0b287bcb605c2512aa3251b9028cc71f4d \
+    --hash=sha256:1d65ce08a56282fb0273dd06585b8927b88d4fba71c01a54f8e2ac87ac1ed387 \
+    --hash=sha256:224e2fbb7088595940c348d168a317caa2110cbb7a5b957a8c3fc0d9296ee069 \
+    --hash=sha256:2a1153cda63f2632d3d5698f0cf29f6b1f1d5162305dc6f5b23336ad8f1039ed \
+    --hash=sha256:2a2239d16a49ce6eaf10166a84424543111f8ebe49d3c124d02af91b01a58425 \
+    --hash=sha256:58eae636e0b1926cddec98a703319a47f671cef07d73aaa525ba421cd4adfeb5 \
+    --hash=sha256:5c19c4ad67af087e8f4411da7422391b236b941f5f0697f615c5816455d1355d \
+    --hash=sha256:61aa60fb848d740581646603a12c2dcb8d7c4cbd2a9c476a1c891ec360ff0b87 \
+    --hash=sha256:8d9f4dc7dbae9783c5dafd66801875a2ebf9302c3addd5739f772285c1c1e91c \
+    --hash=sha256:94c80677b1e8c92fa080e24045d54ace5e4343c4ee6d0216675cd91d6f8e122a \
+    --hash=sha256:9b69db29571dde679908fb237784a8e7af4a2cbf1b7bb25bdb86e487210e04d2 \
+    --hash=sha256:9ef12754021bcc1246f97e00ea62b5594dd5c61192830639ab4a1640bd4b7940 \
+    --hash=sha256:b1bbe763d431d26f9565cba3e99866768761366ab6d609d2506d194882156fa7 \
+    --hash=sha256:d7b8a8f89385ad9e3da38e0123c22c0efc18005e0e2731b6b95e4c21db2049d2 \
+    --hash=sha256:e6254647fa35e1260282aeb9c32a3dd363287b9a1ffcc4f22bd27e54178e92e4 \
+    --hash=sha256:f4c294bfaf2be8004583266d4621bfd3a387e12946f548f966a7fbec91845f1b \
+    --hash=sha256:fa3d70f370604efc67085849d3d1d3d2109faa716c520faf601d15845df64de6
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-hw.txt

@@ -1,130 +1,187 @@
-btchip-python==0.1.28 \
+base58==2.0.1 \
-    --hash=sha256:da09d0d7a6180d428833795ea9a233c3b317ddfcccea8cc6f0eba59435e5dd83
+    --hash=sha256:365c9561d9babac1b5f18ee797508cd54937a724b6e419a130abad69cec5ca79 \
-certifi==2019.11.28 \
+    --hash=sha256:447adc750d6b642987ffc6d397ecd15a799852d5f6a1d308d384500243825058
-    --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \
+bitbox02==4.0.0 \
-    --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f
+    --hash=sha256:b893223a021407145bc45830c1ef11c479044e32c1ae284712919baaa03dda9a \
+    --hash=sha256:fce1b438a4837fd164ce208929f0f5c452a4fdcc858d629872ce7a3984d5e9fc
+btchip-python==0.1.30 \
+    --hash=sha256:6869c67a712969ae86af23617f6418049076626f8a8c34d1000b1c58a9702ad7
+certifi==2020.4.5.2 \
+    --hash=sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1 \
+    --hash=sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc
+cffi==1.14.0 \
+    --hash=sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff \
+    --hash=sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b \
+    --hash=sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac \
+    --hash=sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0 \
+    --hash=sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384 \
+    --hash=sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26 \
+    --hash=sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6 \
+    --hash=sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b \
+    --hash=sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e \
+    --hash=sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd \
+    --hash=sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2 \
+    --hash=sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66 \
+    --hash=sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc \
+    --hash=sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8 \
+    --hash=sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55 \
+    --hash=sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4 \
+    --hash=sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5 \
+    --hash=sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d \
+    --hash=sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78 \
+    --hash=sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa \
+    --hash=sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793 \
+    --hash=sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f \
+    --hash=sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a \
+    --hash=sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f \
+    --hash=sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30 \
+    --hash=sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f \
+    --hash=sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3 \
+    --hash=sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c
 chardet==3.0.4 \
     --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
     --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
-ckcc-protocol==0.8.0 \
+    ckcc-protocol==1.0.2 \
-    --hash=sha256:bad1d1448423472df95ba67621fdd0ad919e625fbe0a4d3ba93648f34ea286e0 \
+        --hash=sha256:2a34e1b2db2dc4f3e5503fac598e010370250dbb07224090eb475b3361f87ab3 \
-    --hash=sha256:f0851c98b91825d19567d0d3bac1b28044d40a3d5f194c8b04c5338f114d7ad5
+        --hash=sha256:31c01e4e460b949d6a570501996c54ee17f5ea25c1ec70b4e1535fe5631df67e
-click==7.0 \
+    click==7.1.2 \
-    --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
+        --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \
-    --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7
+        --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc
-construct==2.9.45 \
+    construct==2.10.56 \
-    --hash=sha256:2271a0efd0798679dea825ff47e22a4c550456a5db0ba8baa82f7eae0af0118c
+        --hash=sha256:97ba13edcd98546f10f7555af41c8ce7ae9d8221525ec4062c03f9adbf940661
-Cython==0.29.10 \
+    cryptography==2.9.2 \
-    --hash=sha256:0afa0b121b89de619e71587e25702e2b7068d7da2164c47e6eee80c17823a62f \
+        --hash=sha256:091d31c42f444c6f519485ed528d8b451d1a0c7bf30e8ca583a0cac44b8a0df6 \
-    --hash=sha256:1c608ba76f7a20cc9f0c021b7fe5cb04bc1a70327ae93a9298b1bc3e0edddebe \
+        --hash=sha256:18452582a3c85b96014b45686af264563e3e5d99d226589f057ace56196ec78b \
-    --hash=sha256:26229570d6787ff3caa932fe9d802960f51a89239b990d275ae845405ce43857 \
+        --hash=sha256:1dfa985f62b137909496e7fc182dac687206d8d089dd03eaeb28ae16eec8e7d5 \
-    --hash=sha256:2a9deafa437b6154cac2f25bb88e0bfd075a897c8dc847669d6f478d7e3ee6b1 \
+        --hash=sha256:1e4014639d3d73fbc5ceff206049c5a9a849cefd106a49fa7aaaa25cc0ce35cf \
-    --hash=sha256:2f28396fbce6d9d68a40edbf49a6729cf9d92a4d39ff0f501947a89188e9099f \
+        --hash=sha256:22e91636a51170df0ae4dcbd250d318fd28c9f491c4e50b625a49964b24fe46e \
-    --hash=sha256:3983dd7b67297db299b403b29b328d9e03e14c4c590ea90aa1ad1d7b35fb178b \
+        --hash=sha256:3b3eba865ea2754738616f87292b7f29448aec342a7c720956f8083d252bf28b \
-    --hash=sha256:4100a3f8e8bbe47d499cdac00e56d5fe750f739701ea52dc049b6c56f5421d97 \
+        --hash=sha256:651448cd2e3a6bc2bb76c3663785133c40d5e1a8c1a9c5429e4354201c6024ae \
-    --hash=sha256:51abfaa7b6c66f3f18028876713c8804e73d4c2b6ceddbcbcfa8ec62429377f0 \
+        --hash=sha256:726086c17f94747cedbee6efa77e99ae170caebeb1116353c6cf0ab67ea6829b \
-    --hash=sha256:61c24f4554efdb8fb1ac6c8e75dab301bcdf2b7b739ed0c2b267493bb43163c5 \
+        --hash=sha256:844a76bc04472e5135b909da6aed84360f522ff5dfa47f93e3dd2a0b84a89fa0 \
-    --hash=sha256:700ccf921b2fdc9b23910e95b5caae4b35767685e0812343fa7172409f1b5830 \
+        --hash=sha256:88c881dd5a147e08d1bdcf2315c04972381d026cdb803325c03fe2b4a8ed858b \
-    --hash=sha256:7b41eb2e792822a790cb2a171df49d1a9e0baaa8e81f58077b7380a273b93d5f \
+        --hash=sha256:96c080ae7118c10fcbe6229ab43eb8b090fccd31a09ef55f83f690d1ef619a1d \
-    --hash=sha256:803987d3b16d55faa997bfc12e8b97f1091f145930dee229b020487aed8a1f44 \
+        --hash=sha256:a0c30272fb4ddda5f5ffc1089d7405b7a71b0b0f51993cb4e5dbb4590b2fc229 \
-    --hash=sha256:99af5cfcd208c81998dcf44b3ca466dee7e17453cfb50e98b87947c3a86f8753 \
+        --hash=sha256:bb1f0281887d89617b4c68e8db9a2c42b9efebf2702a3c5bf70599421a8623e3 \
-    --hash=sha256:9faea1cca34501c7e139bc7ef8e504d532b77865c58592493e2c154a003b450f \
+        --hash=sha256:c447cf087cf2dbddc1add6987bbe2f767ed5317adb2d08af940db517dd704365 \
-    --hash=sha256:a7ba4c9a174db841cfee9a0b92563862a0301d7ca543334666c7266b541f141a \
+        --hash=sha256:c4fd17d92e9d55b84707f4fd09992081ba872d1a0c610c109c18e062e06a2e55 \
-    --hash=sha256:b26071c2313d1880599c69fd831a07b32a8c961ba69d7ccbe5db1cd8d319a4ca \
+        --hash=sha256:d0d5aeaedd29be304848f1c5059074a740fa9f6f26b84c5b63e8b29e73dfc270 \
-    --hash=sha256:b49dc8e1116abde13a3e6a9eb8da6ab292c5a3325155fb872e39011b110b37e6 \
+        --hash=sha256:daf54a4b07d67ad437ff239c8a4080cfd1cc7213df57d33c97de7b4738048d5e \
-    --hash=sha256:bd40def0fd013569887008baa6da9ca428e3d7247adeeaeada153006227bb2e7 \
+        --hash=sha256:e993468c859d084d5579e2ebee101de8f5a27ce8e2159959b6673b418fd8c785 \
-    --hash=sha256:bfd0db770e8bd4e044e20298dcae6dfc42561f85d17ee546dcd978c8b23066ae \
+        --hash=sha256:f118a95c7480f5be0df8afeb9a11bd199aa20afab7a96bcf20409b411a3a85f0
-    --hash=sha256:c2fad1efae5889925c8fd7867fdd61f59480e4e0b510f9db096c912e884704f1 \
+    Cython==0.29.20 \
-    --hash=sha256:c81aea93d526ccf6bc0b842c91216ee9867cd8792f6725a00f19c8b5837e1715 \
+        --hash=sha256:0754ec9d45518d0dbb5da72db2c8b063d40c4c51779618c68431054de179387f \
-    --hash=sha256:da786e039b4ad2bce3d53d4799438cf1f5e01a0108f1b8d78ac08e6627281b1a \
+        --hash=sha256:0bb201124f67b8d5e6a3e7c02257ca56a90204611971ecca76c02897096f097d \
-    --hash=sha256:deab85a069397540987082d251e9c89e0e5b2e3e044014344ff81f60e211fc4b \
+        --hash=sha256:0f3488bf2a9e049d1907d35ad8834f542f8c03d858d1bca6d0cbc06b719163e0 \
-    --hash=sha256:e3f1e6224c3407beb1849bdc5ae3150929e593e4cffff6ca41c6ec2b10942c80 \
+        --hash=sha256:1024714b0f7829b0f712db9cebec92c2782b1f42409b8575cacc340aa438d4ba \
-    --hash=sha256:e74eb224e53aae3943d66e2d29fe42322d5753fd4c0641329bccb7efb3a46552 \
+        --hash=sha256:10b6d2e2125169158128b7f11dad8bb0d8f5fba031d5d4f8492f3afbd06491d7 \
-    --hash=sha256:ee697c7ea65cb14915a64f36874da8ffc2123df43cf8bc952172e04a26656cd6 \
+        --hash=sha256:16ed0260d031d90dda43997e9b0f0eebc3cf18e6ece91cad7b0fb17cd4bfb29b \
-    --hash=sha256:f37792b16d11606c28e428460bd6a3d14b8917b109e77cdbe4ca78b0b9a52c87 \
+        --hash=sha256:22d91af5fc2253f717a1b80b8bb45acb655f643611983fd6f782b9423f8171c7 \
-    --hash=sha256:fd2906b54cbf879c09d875ad4e4687c58d87f5ed03496063fec1c9065569fd5d
+        --hash=sha256:2d84e8d2a0c698c1bce7c2a4677f9f03b076e9f0af7095947ecd2a900ffceea5 \
-ecdsa==0.14.1 \
+        --hash=sha256:34dd57f5ac5a0e3d53da964994fc1b7e7ee3f86172d7a1f0bde8a1f90739e04d \
-    --hash=sha256:64c613005f13efec6541bb0a33290d0d03c27abab5f15fbab20fb0ee162bdd8e \
+        --hash=sha256:384582b5024007dfdabc9753e3e0f85d61837b0103b0ee3f8acf04a4bcfad175 \
-    --hash=sha256:e108a5fe92c67639abae3260e43561af914e7fd0d27bae6d2ec1312ae7934dfe
+        --hash=sha256:4473f169d6dd02174eb76396cb38ce469f377c08b21965ddf4f88bbbebd5816e \
-hidapi==0.7.99.post21 \
+        --hash=sha256:57f32d1095ad7fad1e7f2ff6e8c6a7197fa532c8e6f4d044ff69212e0bf05461 \
-    --hash=sha256:1ac170f4d601c340f2cd52fd06e85c5e77bad7ceac811a7bb54b529f7dc28c24 \
+        --hash=sha256:5dfe519e400a1672a3ac5bdfb5e957a9c14c52caafb01f4a923998ec9ae77736 \
-    --hash=sha256:6424ad75da0021ce8c1bcd78056a04adada303eff3c561f8d132b85d0a914cb3 \
+        --hash=sha256:60def282839ed81a2ffae29d2df0a6777fd74478c6e82c6c3f4b54e698b9d11c \
-    --hash=sha256:8d3be666f464347022e2b47caf9132287885d9eacc7895314fc8fefcb4e42946 \
+        --hash=sha256:7089fb2be9a9869b9aa277bc6de401928954ce70e139c3cf9b244ae5f490b8f2 \
-    --hash=sha256:92878bad7324dee619b7832fbfc60b5360d378aa7c5addbfef0a410d8fd342c7 \
+        --hash=sha256:714b8926a84e3e39c5278e43fb8823598db82a4b015cff263b786dc609a5e7d6 \
-    --hash=sha256:b4b1f6aff0192e9be153fe07c1b7576cb7a1ff52e78e3f76d867be95301a8e87 \
+        --hash=sha256:7352b88f2213325c1e111561496a7d53b0326e7f07e6f81f9b8b21420e40851c \
-    --hash=sha256:bf03f06f586ce7d8aeb697a94b7dba12dc9271aae92d7a8d4486360ff711a660 \
+        --hash=sha256:809f0a3f647052c4bcbc34a15f53a5dab90de1a83ebd77add37ed5d3e6ee5d97 \
-    --hash=sha256:c76de162937326fcd57aa399f94939ce726242323e65c15c67e183da1f6c26f7 \
+        --hash=sha256:8598b09f7973ccb15c03b21d3185dc129ae7c60d0a6caf8176b7099a4b83483e \
-    --hash=sha256:d4ad1e46aef98783a9e6274d523b8b1e766acfc3d72828cd44a337564d984cfa \
+        --hash=sha256:8dc68f93b257718ea0e2bc9be8e3c61d70b6e49ab82391125ba0112a30a21025 \
-    --hash=sha256:d4b5787a04613503357606bb10e59c3e2c1114fa00ee328b838dd257f41cbd7b \
+        --hash=sha256:9bfd42c1d40aa26bf76186cba0d89be66ba47e36fa7ea56d71f377585a53f7c4 \
-    --hash=sha256:e0be1aa6566979266a8fc845ab0e18613f4918cf2c977fe67050f5dc7e2a9a97 \
+        --hash=sha256:a21cb3423acd6dbf383c9e41e8e60c93741987950434c85145864458d30099f3 \
-    --hash=sha256:edfb16b16a298717cf05b8c8a9ad1828b6ff3de5e93048ceccd74e6ae4ff0922
+        --hash=sha256:a49d0f5c55ad0f4aacad32f058a71d0701cb8936d6883803e50698fa04cac8d2 \
-idna==2.8 \
+        --hash=sha256:a985a7e3c7f1663af398938029659a4381cfe9d1bd982cf19c46b01453e81775 \
-    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
+        --hash=sha256:b3233341c3fe352b1090168bd087686880b582b635d707b2c8f5d4f1cc1fa533 \
-    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
+        --hash=sha256:b32965445b8dbdc36c69fba47e024060f9b39b1b4ceb816da5028eea01924505 \
+        --hash=sha256:b553473c31297e4ca77fbaea2eb2329889d898c03941d90941679247c17e38fb \
+        --hash=sha256:b56c02f14f1708411d95679962b742a1235d33a23535ce4a7f75425447701245 \
+        --hash=sha256:b7bb0d54ff453c7516d323c3c78b211719f39a506652b79b7e85ba447d5fa9e7 \
+        --hash=sha256:c5df2c42d4066cda175cd4d075225501e1842cfdbdaeeb388eb7685c367cc3ce \
+        --hash=sha256:c5e29333c9e20df384645902bed7a67a287b979da1886c8f10f88e57b69e0f4b \
+        --hash=sha256:d0b445def03b4cd33bd2d1ae6fbbe252b6d1ef7077b3b5ba3f2c698a190d26e5 \
+        --hash=sha256:d490a54814b69d814b157ac86ada98c15fd77fabafc23732818ed9b9f1f0af80
+    ecdsa==0.15 \
+        --hash=sha256:867ec9cf6df0b03addc8ef66b56359643cb5d0c1dc329df76ba7ecfe256c8061 \
+        --hash=sha256:8f12ac317f8a1318efa75757ef0a651abe12e51fc1af8838fb91079445227277
+    hidapi==0.9.0.post2 \
+        --hash=sha256:03b9118749f6102a96af175b2b77832c0d6f8957acb46ced5aa7afcf358052bc \
+        --hash=sha256:3b31b396b6e95b635db4db8e9649cdb0aa2c205dd4cd8aaf3ee9807dddb1ebb8 \
+        --hash=sha256:448c2ba9f713a5ee754830b222c9bc54a4e0dca4ecd0d84e3bf14314949ec594 \
+        --hash=sha256:4c712309e2534a249721feb2abe7baedb9bfe7b3cc0e06cf4b78329684480932 \
+        --hash=sha256:9c4369499a322d91d9f697c6b84b78f78c42695743641cb8bf3b5fa8c3c9b09c \
+        --hash=sha256:a71dd3c153cb6bb2b73d2612b5ab262830d78c6428f33f0c06818749e64c9320 \
+        --hash=sha256:d8dd636b7da9dfeb4aa08da64aceb91fb311465faae347b885cb8b695b141364 \
+        --hash=sha256:da40dcf99ea15d440f3f3667f4166addd5676c485acf331c6e7c6c7879e11633 \
+        --hash=sha256:dc633b34e318ce4638b73beb531136ab02ab005bfb383c260a41b5dfd5d85f16
+    idna==2.9 \
+        --hash=sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb \
+        --hash=sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa
 keepkey==6.3.1 \
     --hash=sha256:88e2b5291c85c8e8567732f675697b88241082884aa1aba32257f35ee722fc09 \
     --hash=sha256:cef1e862e195ece3e42640a0f57d15a63086fd1dedc8b5ddfcbc9c2657f0bb1e \
     --hash=sha256:f369d640c65fec7fd8e72546304cdc768c04224a6b9b00a19dc2cd06fa9d2a6b
-libusb1==1.7.1 \
+libusb1==1.8 \
-    --hash=sha256:adf64a4f3f5c94643a1286f8153bcf4bc787c348b38934aacd7fe17fbeebc571
+    --hash=sha256:240f65ac70ba3fab77749ec84a412e4e89624804cb80d6c9d394eef5af8878d6
 mnemonic==0.19 \
     --hash=sha256:4e37eb02b2cbd56a0079cabe58a6da93e60e3e4d6e757a586d9f23d96abea931 \
     --hash=sha256:a8d78c5100acfa7df9bab6b9db7390831b0e54490934b718ff9efd68f0d731a6
-pip==19.3.1 \
+noiseprotocol==0.3.1 \
-    --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \
+        --hash=sha256:2e1a603a38439636cf0ffd8b3e8b12cee27d368a28b41be7dbe568b2abb23111
-    --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7
+pip==20.1.1 \
-protobuf==3.11.1 \
+        --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
-    --hash=sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd \
+        --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
-    --hash=sha256:200b77e51f17fbc1d3049045f5835f60405dec3a00fe876b9b986592e46d908c \
+protobuf==3.12.2 \
-    --hash=sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed \
+    --hash=sha256:304e08440c4a41a0f3592d2a38934aad6919d692bb0edfb355548786728f9a5e \
-    --hash=sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057 \
+    --hash=sha256:49ef8ab4c27812a89a76fa894fe7a08f42f2147078392c0dee51d4a444ef6df5 \
-    --hash=sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce \
+    --hash=sha256:50b5fee674878b14baea73b4568dc478c46a31dd50157a5b5d2f71138243b1a9 \
-    --hash=sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03 \
+    --hash=sha256:5524c7020eb1fb7319472cb75c4c3206ef18b34d6034d2ee420a60f99cddeb07 \
-    --hash=sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46 \
+    --hash=sha256:612bc97e42b22af10ba25e4140963fbaa4c5181487d163f4eb55b0b15b3dfcd2 \
-    --hash=sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33 \
+    --hash=sha256:6f349adabf1c004aba53f7b4633459f8ca8a09654bf7e69b509c95a454755776 \
-    --hash=sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c \
+    --hash=sha256:85b94d2653b0fdf6d879e39d51018bf5ccd86c81c04e18a98e9888694b98226f \
-    --hash=sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9 \
+    --hash=sha256:87535dc2d2ef007b9d44e309d2b8ea27a03d2fa09556a72364d706fcb7090828 \
-    --hash=sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef \
+    --hash=sha256:a7ab28a8f1f043c58d157bceb64f80e4d2f7f1b934bc7ff5e7f7a55a337ea8b0 \
-    --hash=sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b \
+    --hash=sha256:a96f8fc625e9ff568838e556f6f6ae8eca8b4837cdfb3f90efcb7c00e342a2eb \
-    --hash=sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d \
+    --hash=sha256:b5a114ea9b7fc90c2cc4867a866512672a47f66b154c6d7ee7e48ddb68b68122 \
-    --hash=sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8 \
+    --hash=sha256:be04fe14ceed7f8641e30f36077c1a654ff6f17d0c7a5283b699d057d150d82a \
-    --hash=sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6 \
+    --hash=sha256:bff02030bab8b969f4de597543e55bd05e968567acb25c0a87495a31eb09e925 \
-    --hash=sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941 \
+    --hash=sha256:c9ca9f76805e5a637605f171f6c4772fc4a81eced4e2f708f79c75166a2c99ea \
-    --hash=sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13
+    --hash=sha256:e1464a4a2cf12f58f662c8e6421772c07947266293fb701cb39cd9c1e183f63c \
+    --hash=sha256:e72736dd822748b0721f41f9aaaf6a5b6d5cfc78f6c8690263aef8bba4457f0e \
+    --hash=sha256:eafe9fa19fcefef424ee089fb01ac7177ff3691af7cc2ae8791ae523eb6ca907 \
+    --hash=sha256:f4b73736108a416c76c17a8a09bc73af3d91edaa26c682aaa460ef91a47168d3
 pyaes==1.6.1 \
     --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f
-pyblake2==1.1.2 \
+    pycparser==2.20 \
-    --hash=sha256:3757f7ad709b0e1b2a6b3919fa79fe3261f166fc375cd521f2be480f8319dde9 \
+        --hash=sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0 \
-    --hash=sha256:407e02c7f8f36fcec1b7aa114ddca0c1060c598142ea6f6759d03710b946a7e3 \
+        --hash=sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705
-    --hash=sha256:4d47b4a2c1d292b1e460bde1dda4d13aa792ed2ed70fcc263b6bc24632c8e902 \
+    requests==2.23.0 \
-    --hash=sha256:5ccc7eb02edb82fafb8adbb90746af71460fbc29aa0f822526fc976dff83e93f \
+        --hash=sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee \
-    --hash=sha256:8043267fbc0b2f3748c6920591cd0b8b5609dcce60c504c32858aa36206386f2 \
+        --hash=sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6
-    --hash=sha256:982295a87907d50f4723db6bc724660da76b6547826d52160171d54f95b919ac \
+    safet==0.1.5 \
-    --hash=sha256:baa2190bfe549e36163aa44664d4ee3a9080b236fc5d42f50dc6fd36bbdc749e \
+        --hash=sha256:a7fd4b68bb1bc6185298af665c8e8e00e2bb2bcbddbb22844ead929b845c635e \
-    --hash=sha256:c53417ee0bbe77db852d5fd1036749f03696ebc2265de359fe17418d800196c4 \
+        --hash=sha256:f966a23243312f64d14c7dfe02e8f13f6eeba4c3f51341f2c11ae57831f07de3
-    --hash=sha256:fbc9fcde75713930bc2a91b149e97be2401f7c9c56d735b46a109210f58d7358
+    semver==2.10.1 \
-requests==2.22.0 \
+        --hash=sha256:21eb9deafc627dfd122e294f96acd0deadf1b5b7758ab3bbdf3698155dca4705 \
-    --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
+        --hash=sha256:b08a84f604ef579e474ce448672a05c8d50d1ee0b24cee9fb58a12b260e4d0dc
-    --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31
+    setuptools==46.4.0 \
-safet==0.1.4 \
+        --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
-    --hash=sha256:522c257910f9472e9c77c487425ed286f6721c314653e232bc41c6cedece1bb1 \
+        --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
-    --hash=sha256:b152874acdc89ff0c8b2d680bfbf020b3e53527c2ad3404489dd61a548aa56a1
+    six==1.15.0 \
-setuptools==42.0.2 \
+        --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
-    --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \
+        --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
-    --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6
+    trezor==0.12.0 \
-six==1.13.0 \
+        --hash=sha256:da5b750ada03830fd1f0b9010f7d5d30e77ec3e1458230e3d08fe4588a0741b2 \
-    --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd \
+        --hash=sha256:f6bc821bddec06e67a1abd0be1d9fbc61c59b08272c736522ae2f6b225bf9579
-    --hash=sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66
+    typing-extensions==3.7.4.2 \
-trezor==0.11.5 \
+        --hash=sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5 \
-    --hash=sha256:711137bb83e7e0aef4009745e0da1b7d258146f246b43e3f7f5b849405088ef1 \
+        --hash=sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae \
-    --hash=sha256:cd8aafd70a281daa644c4a3fb021ffac20b7a88e86226ecc8bb3e78e1734a184
+        --hash=sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392
-typing-extensions==3.7.4.1 \
+    urllib3==1.25.9 \
-    --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \
+        --hash=sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527 \
-    --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \
+        --hash=sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115
-    --hash=sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575
+    wheel==0.34.2 \
-urllib3==1.25.7 \
+        --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
-    --hash=sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293 \
+        --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e
-    --hash=sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745
-wheel==0.33.6 \
-    --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \
-    --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28

contrib/deterministic-build/requirements-mac-build.txt

@@ -0,0 +1,17 @@
+altgraph==0.17 \
+    --hash=sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa \
+    --hash=sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe
+macholib==1.14 \
+    --hash=sha256:0c436bc847e7b1d9bda0560351bf76d7caf930fb585a828d13608839ef42c432 \
+    --hash=sha256:c500f02867515e6c60a27875b408920d18332ddf96b4035ef03beddd782d4281
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+PyInstaller==3.6 \
+    --hash=sha256:3730fa80d088f8bb7084d32480eb87cbb4ddb64123363763cf8f2a1378c1c4b7
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-sdist-build.txt

@@ -0,0 +1,9 @@
+pip==20.1.1 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
+setuptools==46.4.0 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
+wheel==0.34.2 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements-wine-build.txt

@@ -1,19 +1,19 @@
-altgraph==0.16.1 \
+altgraph==0.17 \
-    --hash=sha256:d6814989f242b2b43025cba7161fc1b8fb487a62cd49c49245d6fd01c18ac997 \
+    --hash=sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa \
-    --hash=sha256:ddf5320017147ba7b810198e0b6619bd7b5563aa034da388cea8546b877f9b0c
+    --hash=sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe
 future==0.18.2 \
     --hash=sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d
 pefile==2019.4.18 \
     --hash=sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645
-pip==19.3.1 \
+pip==20.1.1 \
-    --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
-    --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
 pywin32-ctypes==0.2.0 \
     --hash=sha256:24ffc3b341d457d48e8922352130cf2644024a4ff09762a2261fd34c36ee5942 \
     --hash=sha256:9dc2d991b3479cc2df15930958b674a48a227d5361d413827a4cfd0b5876fc98
-setuptools==42.0.2 \
+setuptools==46.4.0 \
-    --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
-    --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
-wheel==0.33.6 \
+wheel==0.34.2 \
-    --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
-    --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e

contrib/deterministic-build/requirements.txt

@@ -11,173 +11,128 @@ aiohttp==3.6.2 \
     --hash=sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48 \
     --hash=sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59 \
     --hash=sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965
-aiohttp-socks==0.2.2 \
+aiohttp-socks==0.3.9 \
-    --hash=sha256:e473ee222b001fe33798957b9ce3352b32c187cf41684f8e2259427925914993 \
+    --hash=sha256:5e5638d0e472baa441eab7990cf19e034960cc803f259748cc359464ccb3c2d6 \
-    --hash=sha256:eebd8939a7c3c1e3e7e1b2552c60039b4c65ef6b8b2351efcbdd98290538e310
+    --hash=sha256:ccd483d7677d7ba80b7ccb738a9be27a3ad6dce4b2756509bc71c9d679d96105
 aiorpcX==0.18.4 \
     --hash=sha256:bec9c0feb328d62ba80b79931b07f7372c98f2891ad51300be0b7163d5ccfb4a \
     --hash=sha256:d424a55bcf52ebf1b3610a7809c0748fac91ce926854ad33ce952463bc6017e8
-apply-defaults==0.1.4 \
-    --hash=sha256:1ce26326a61d8773d38a9726a345c6525a91a6120d7333af79ad792dacb6246c
 async-timeout==3.0.1 \
     --hash=sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f \
     --hash=sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3
 attrs==19.3.0 \
     --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \
     --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72
-bitstring==3.1.6 \
+bitstring==3.1.7 \
-    --hash=sha256:7b60b0c300d0d3d0a24ec84abfda4b0eaed3dc56dc90f6cbfe497166c9ad8443 \
+    --hash=sha256:fdf3eb72b229d2864fb507f8f42b1b2c57af7ce5fec035972f9566de440a864a
-    --hash=sha256:c97a8e2a136e99b523b27da420736ae5cb68f83519d633794a6a11192f69f8bf \
+certifi==2020.4.5.2 \
-    --hash=sha256:e392819965e7e0246e3cf6a51d5a54e731890ae03ebbfa3cd0e4f74909072096
+    --hash=sha256:5ad7e9a056d25ffa5082862e36f119f7f7cec6457fa07ee2f8c339814b80c9b1 \
-certifi==2019.11.28 \
+    --hash=sha256:9cd41137dc19af6a5e03b630eefe7d1f458d964d406342dd3edf625839b944cc
-    --hash=sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3 \
-    --hash=sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f
 chardet==3.0.4 \
     --hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
     --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691
-click==6.7 \
-    --hash=sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d \
-    --hash=sha256:f15516df478d5a56180fbf80e68f206010e6d160fc39fa508b65e035fd75130b
 dnspython==1.16.0 \
     --hash=sha256:36c5e8e38d4369a08b6780b7f27d790a292b2b08eea01607865bf0936c558e01 \
     --hash=sha256:f69c21288a962f4da86e56c4905b49d11aba7938d3d740e80d9e366ee4f1632d
-ecdsa==0.14.1 \
+ecdsa==0.15 \
-    --hash=sha256:64c613005f13efec6541bb0a33290d0d03c27abab5f15fbab20fb0ee162bdd8e \
+    --hash=sha256:867ec9cf6df0b03addc8ef66b56359643cb5d0c1dc329df76ba7ecfe256c8061 \
-    --hash=sha256:e108a5fe92c67639abae3260e43561af914e7fd0d27bae6d2ec1312ae7934dfe
+    --hash=sha256:8f12ac317f8a1318efa75757ef0a651abe12e51fc1af8838fb91079445227277
-idna==2.8 \
+helpdev==0.7.1 \
-    --hash=sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407 \
+    --hash=sha256:779a761b18c2d96fb181aa699609f802347806125f2fee2f60dad875a625e38e \
-    --hash=sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c
+    --hash=sha256:bb62a79acbac141dadf42cadeb92bb7450dd18b9824a62043b6a0b149190db3d
+idna==2.9 \
+    --hash=sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb \
+    --hash=sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa
 idna_ssl==1.1.0 \
     --hash=sha256:a933e3bb13da54383f9e8f35dc4f9cb9eb9b3b78c6b36f311254d6d0d92c6c7c
-importlib-metadata==1.1.0 \
+importlib-metadata==1.6.1 \
-    --hash=sha256:b044f07694ef14a6683b097ba56bd081dbc7cdc7c7fe46011e499dfecc082f21 \
+    --hash=sha256:0505dd08068cfec00f53a74a0ad927676d7757da81b7436a6eefe4c7cf75c545 \
-    --hash=sha256:e6ac600a142cf2db707b1998382cc7fc3b02befb7273876e01b8ad10b9652742
+    --hash=sha256:15ec6c0fd909e893e3a08b3a7c76ecb149122fb14b7efe1199ddd4c7c57ea958
-jsonrpcclient==3.3.4 \
+multidict==4.7.6 \
-    --hash=sha256:c50860409b73af9f94b648439caae3b4af80d5ac937f2a8ac7783de3d1050ba9
+    --hash=sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a \
-jsonrpcserver==4.0.5 \
+    --hash=sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000 \
-    --hash=sha256:240c517f49b0fdd3bfa428c9a7cc581126a0c43eca60d29762da124017d9d9f4
+    --hash=sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2 \
-jsonschema==3.2.0 \
+    --hash=sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507 \
-    --hash=sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163 \
+    --hash=sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5 \
-    --hash=sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a
+    --hash=sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7 \
-more-itertools==8.0.0 \
+    --hash=sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d \
-    --hash=sha256:53ff73f186307d9c8ef17a9600309154a6ae27f25579e80af4db8f047ba14bc2 \
+    --hash=sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463 \
-    --hash=sha256:a0ea684c39bc4315ba7aae406596ef191fd84f873d2d2751f84d64e81a7a2d45
+    --hash=sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19 \
-multidict==4.6.1 \
+    --hash=sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3 \
-    --hash=sha256:07f9a6bf75ad675d53956b2c6a2d4ef2fa63132f33ecc99e9c24cf93beb0d10b \
+    --hash=sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b \
-    --hash=sha256:0ffe4d4d28cbe9801952bfb52a8095dd9ffecebd93f84bdf973c76300de783c5 \
+    --hash=sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c \
-    --hash=sha256:1b605272c558e4c659dbaf0fb32a53bfede44121bcf77b356e6e906867b958b7 \
+    --hash=sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87 \
-    --hash=sha256:205a011e636d885af6dd0029e41e3514a46e05bb2a43251a619a6e8348b96fc0 \
+    --hash=sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7 \
-    --hash=sha256:250632316295f2311e1ed43e6b26a63b0216b866b45c11441886ac1543ca96e1 \
+    --hash=sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430 \
-    --hash=sha256:2bc9c2579312c68a3552ee816311c8da76412e6f6a9cf33b15152e385a572d2a \
+    --hash=sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255 \
-    --hash=sha256:318aadf1cfb6741c555c7dd83d94f746dc95989f4f106b25b8a83dfb547f2756 \
+    --hash=sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d
-    --hash=sha256:42cdd649741a14b0602bf15985cad0dd4696a380081a3319cd1ead46fd0f0fab \
+pip==20.1.1 \
-    --hash=sha256:5159c4975931a1a78bf6602bbebaa366747fce0a56cb2111f44789d2c45e379f \
+    --hash=sha256:27f8dc29387dd83249e06e681ce087e6061826582198a425085e0bf4c1cf3a55 \
-    --hash=sha256:87e26d8b89127c25659e962c61a4c655ec7445d19150daea0759516884ecb8b4 \
+    --hash=sha256:b27c4dedae8c41aa59108f2fa38bf78e0890e590545bc8ece7cdceb4ba60f6e4
-    --hash=sha256:891b7e142885e17a894d9d22b0349b92bb2da4769b4e675665d0331c08719be5 \
+protobuf==3.12.2 \
-    --hash=sha256:8d919034420378132d074bf89df148d0193e9780c9fe7c0e495e895b8af4d8a2 \
+    --hash=sha256:304e08440c4a41a0f3592d2a38934aad6919d692bb0edfb355548786728f9a5e \
-    --hash=sha256:9c890978e2b37dd0dc1bd952da9a5d9f245d4807bee33e3517e4119c48d66f8c \
+    --hash=sha256:49ef8ab4c27812a89a76fa894fe7a08f42f2147078392c0dee51d4a444ef6df5 \
-    --hash=sha256:a37433ce8cdb35fc9e6e47e1606fa1bfd6d70440879038dca7d8dd023197eaa9 \
+    --hash=sha256:50b5fee674878b14baea73b4568dc478c46a31dd50157a5b5d2f71138243b1a9 \
-    --hash=sha256:c626029841ada34c030b94a00c573a0c7575fe66489cde148785b6535397d675 \
+    --hash=sha256:5524c7020eb1fb7319472cb75c4c3206ef18b34d6034d2ee420a60f99cddeb07 \
-    --hash=sha256:cfec9d001a83dc73580143f3c77e898cf7ad78b27bb5e64dbe9652668fcafec7 \
+    --hash=sha256:612bc97e42b22af10ba25e4140963fbaa4c5181487d163f4eb55b0b15b3dfcd2 \
-    --hash=sha256:efaf1b18ea6c1f577b1371c0159edbe4749558bfe983e13aa24d0a0c01e1ad7b
+    --hash=sha256:6f349adabf1c004aba53f7b4633459f8ca8a09654bf7e69b509c95a454755776 \
-pip==19.3.1 \
+    --hash=sha256:85b94d2653b0fdf6d879e39d51018bf5ccd86c81c04e18a98e9888694b98226f \
-    --hash=sha256:21207d76c1031e517668898a6b46a9fb1501c7a4710ef5dfd6a40ad9e6757ea7 \
+    --hash=sha256:87535dc2d2ef007b9d44e309d2b8ea27a03d2fa09556a72364d706fcb7090828 \
-    --hash=sha256:6917c65fc3769ecdc61405d3dfd97afdedd75808d200b2838d7d961cebc0c2c7
+    --hash=sha256:a7ab28a8f1f043c58d157bceb64f80e4d2f7f1b934bc7ff5e7f7a55a337ea8b0 \
-protobuf==3.11.1 \
+    --hash=sha256:a96f8fc625e9ff568838e556f6f6ae8eca8b4837cdfb3f90efcb7c00e342a2eb \
-    --hash=sha256:0265379852b9e1f76af6d3d3fe4b3c383a595cc937594bda8565cf69a96baabd \
+    --hash=sha256:b5a114ea9b7fc90c2cc4867a866512672a47f66b154c6d7ee7e48ddb68b68122 \
-    --hash=sha256:200b77e51f17fbc1d3049045f5835f60405dec3a00fe876b9b986592e46d908c \
+    --hash=sha256:be04fe14ceed7f8641e30f36077c1a654ff6f17d0c7a5283b699d057d150d82a \
-    --hash=sha256:29bd1ed46b2536ad8959401a2f02d2d7b5a309f8e97518e4f92ca6c5ba74dbed \
+    --hash=sha256:bff02030bab8b969f4de597543e55bd05e968567acb25c0a87495a31eb09e925 \
-    --hash=sha256:3175d45698edb9a07c1a78a1a4850e674ce8988f20596580158b1d0921d0f057 \
+    --hash=sha256:c9ca9f76805e5a637605f171f6c4772fc4a81eced4e2f708f79c75166a2c99ea \
-    --hash=sha256:34a7270940f86da7a28be466ac541c89b6dbf144a6348b9cf7ac6f56b71006ce \
+    --hash=sha256:e1464a4a2cf12f58f662c8e6421772c07947266293fb701cb39cd9c1e183f63c \
-    --hash=sha256:38cbc830a4a5ba9956763b0f37090bfd14dd74e72762be6225de2ceac55f4d03 \
+    --hash=sha256:e72736dd822748b0721f41f9aaaf6a5b6d5cfc78f6c8690263aef8bba4457f0e \
-    --hash=sha256:665194f5ad386511ac8d8a0bd57b9ab37b8dd2cd71969458777318e774b9cd46 \
+    --hash=sha256:eafe9fa19fcefef424ee089fb01ac7177ff3691af7cc2ae8791ae523eb6ca907 \
-    --hash=sha256:839bad7d115c77cdff29b488fae6a3ab503ce9a4192bd4c42302a6ea8e5d0f33 \
+    --hash=sha256:f4b73736108a416c76c17a8a09bc73af3d91edaa26c682aaa460ef91a47168d3
-    --hash=sha256:934a9869a7f3b0d84eca460e386fba1f7ba2a0c1a120a2648bc41fadf50efd1c \
-    --hash=sha256:aecdf12ef6dc7fd91713a6da93a86c2f2a8fe54840a3b1670853a2b7402e77c9 \
-    --hash=sha256:c4e90bc27c0691c76e09b5dc506133451e52caee1472b8b3c741b7c912ce43ef \
-    --hash=sha256:c65d135ea2d85d40309e268106dab02d3bea723db2db21c23ecad4163ced210b \
-    --hash=sha256:c98dea04a1ff41a70aff2489610f280004831798cb36a068013eed04c698903d \
-    --hash=sha256:d9049aa194378a426f0b2c784e2054565bf6f754d20fcafdee7102a6250556e8 \
-    --hash=sha256:e028fee51c96de4e81924484c77111dfdea14010ecfc906ea5b252209b0c4de6 \
-    --hash=sha256:e84ad26fb50091b1ea676403c0dd2bd47663099454aa6d88000b1dafecab0941 \
-    --hash=sha256:e88a924b591b06d0191620e9c8aa75297b3111066bb09d49a24bae1054a10c13
 pyaes==1.6.1 \
     --hash=sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f
-pycryptodomex==3.9.4 \
+QDarkStyle==2.8.1 \
-    --hash=sha256:0943b65fb41b7403a9def6214061fdd9ab9afd0bbc581e553c72eebe60bded36 \
+    --hash=sha256:7cead57817a8a1f38b48d76ef38986b6cc397d0315c0dd0431fcd06749556947 \
-    --hash=sha256:0a1dbb5c4d975a4ea568fb7686550aa225d94023191fb0cca8747dc5b5d77857 \
+    --hash=sha256:d53b0120bddd9e3efba9801731e22ef86ed798bb5fc6a802f5f7bb32dedf0321
-    --hash=sha256:0f43f1608518347fdcb9c8f443fa5cabedd33f94188b13e4196a3a7ba90d169c \
-    --hash=sha256:11ce5fec5990e34e3981ed14897ba601c83957b577d77d395f1f8f878a179f98 \
-    --hash=sha256:17a09e38fdc91e4857cf5a7ce82f3c0b229c3977490f2146513e366923fc256b \
-    --hash=sha256:22d970cee5c096b9123415e183ae03702b2cd4d3ba3f0ced25c4e1aba3967167 \
-    --hash=sha256:2a1793efcbae3a2264c5e0e492a2629eb10d895d6e5f17dbbd00eb8b489c6bda \
-    --hash=sha256:30a8a148a0fe482cec1aaf942bbd0ade56ec197c14fe058b2a94318c57e1f991 \
-    --hash=sha256:32fbbaf964c5184d3f3e349085b0536dd28184b02e2b014fc900f58bbc126339 \
-    --hash=sha256:347d67faee36d449dc9632da411cc318df52959079062627f1243001b10dc227 \
-    --hash=sha256:45f4b4e5461a041518baabc52340c249b60833aa84cea6377dc8016a2b33c666 \
-    --hash=sha256:4717daec0035034b002d31c42e55431c970e3e38a78211f43990e1b7eaf19e28 \
-    --hash=sha256:51a1ac9e7dda81da444fed8be558a60ec88dfc73b2aa4b0efa310e87acb75838 \
-    --hash=sha256:53e9dcc8f14783f6300b70da325a50ac1b0a3dbaee323bd9dc3f71d409c197a1 \
-    --hash=sha256:5519a2ed776e193688b7ddb61ab709303f6eb7d1237081e298283c72acc44271 \
-    --hash=sha256:583450e8e80a0885c453211ed2bd69ceea634d8c904f23ff8687f677fe810e95 \
-    --hash=sha256:60f862bd2a07133585a4fc2ce2b1a8ec24746b07ac44307d22ef2b767cb03435 \
-    --hash=sha256:612091f1d3c84e723bec7cb855cf77576e646045744794c9a3f75ba80737762f \
-    --hash=sha256:629a87b87c8203b8789ccefc7f2f2faecd2daaeb56bdd0b4e44cd89565f2db07 \
-    --hash=sha256:6e56ec4c8938fb388b6f250ddd5e21c15e8f25a76e0ad0e2abae9afee09e67b4 \
-    --hash=sha256:8e8092651844a11ec7fa534395f3dfe99256ce4edca06f128efc9d770d6e1dc1 \
-    --hash=sha256:8f5f260629876603e08f3ce95c8ccd9b6b83bf9a921c41409046796267f7adc5 \
-    --hash=sha256:9a6b74f38613f54c56bd759b411a352258f47489bbefd1d57c930a291498b35b \
-    --hash=sha256:a5a13ebb52c4cd065fb673d8c94f39f30823428a4de19e1f3f828b63a8882d1e \
-    --hash=sha256:a77ca778a476829876a3a70ae880073379160e4a465d057e3c4e1c79acdf1b8a \
-    --hash=sha256:a9f7be3d19f79429c2118fd61bc2ec4fa095e93b56fb3a5f3009822402c4380f \
-    --hash=sha256:dc15a467c4f9e4b43748ba2f97aea66f67812bfd581818284c47cadc81d4caec \
-    --hash=sha256:e13cdeea23059f7577c230fd580d2c8178e67ebe10e360041abe86c33c316f1c \
-    --hash=sha256:e45b85c8521bca6bdfaf57e4987743ade53e9f03529dd3adbc9524094c6d55c4 \
-    --hash=sha256:e87f17867b260f57c88487f943eb4d46c90532652bb37046e764842c3b66cbb1 \
-    --hash=sha256:ee40a5b156f6c1192bc3082e9d73d0479904433cdda83110546cd67f5a15a5be \
-    --hash=sha256:ef63ffde3b267043579af8830fc97fc3b9b8a526a24e3ba23af9989d4e9e689a
-pyrsistent==0.15.6 \
-    --hash=sha256:f3b280d030afb652f79d67c5586157c5c1355c9a58dfc7940566e28d28f3df1b
-QDarkStyle==2.6.8 \
-    --hash=sha256:037a54bf0aa5153f8055b65b8b36ac0d0f7648f2fd906c011a4da22eb0f582a2 \
-    --hash=sha256:fd1abae37d3a0a004089178da7c0b26ec5eb29f965b3e573853b8f280b614dea
 qrcode==6.1 \
     --hash=sha256:3996ee560fc39532910603704c82980ff6d4d5d629f9c3f25f34174ce8606cf5 \
     --hash=sha256:505253854f607f2abf4d16092c61d4e9d511a3b4392e60bff957a68592b04369
-setuptools==42.0.2 \
+QtPy==1.9.0 \
-    --hash=sha256:c5b372090d7c8709ce79a6a66872a91e518f7d65af97fca78135e1cb10d4b940 \
+    --hash=sha256:2db72c44b55d0fe1407be8fba35c838ad0d6d3bb81f23007886dc1fc0f459c8d \
-    --hash=sha256:c8abd0f3574bc23afd2f6fd2c415ba7d9e097c8a99b845473b0d957ba1e2dac6
+    --hash=sha256:fa0b8363b363e89b2a6f49eddc162a04c0699ae95e109a6be3bb145a913190ea
-six==1.13.0 \
+setuptools==46.4.0 \
-    --hash=sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd \
+    --hash=sha256:4334fc63121aafb1cc98fd5ae5dd47ea8ad4a38ad638b47af03a686deb14ef5b \
-    --hash=sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66
+    --hash=sha256:d05c2c47bbef97fd58632b63dd2b83426db38af18f65c180b2423fea4b67e6b8
-typing-extensions==3.7.4.1 \
+six==1.15.0 \
-    --hash=sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2 \
+    --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
-    --hash=sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d \
+    --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
-    --hash=sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575
+typing-extensions==3.7.4.2 \
-wheel==0.33.6 \
+    --hash=sha256:6e95524d8a547a91e08f404ae485bbb71962de46967e1b71a0cb89af24e761c5 \
-    --hash=sha256:10c9da68765315ed98850f8e048347c3eb06dd81822dc2ab1d4fde9dc9702646 \
+    --hash=sha256:79ee589a3caca649a9bfd2a8de4709837400dfa00b6cc81962a1e6a1815969ae \
-    --hash=sha256:f4da1763d3becf2e2cd92a14a7c920f0f00eca30fdde9ea992c836685b9faf28
+    --hash=sha256:f8d2bd89d25bc39dabe7d23df520442fa1d8969b82544370e03d88b5a591c392
-yarl==1.4.1 \
+wheel==0.34.2 \
-    --hash=sha256:031e8f56cf085d3b3df6b6bce756369ea7052b82d35ea07b6045f209c819e0e5 \
+    --hash=sha256:8788e9155fe14f54164c1b9eb0a319d98ef02c160725587ad60f14ddc57b6f96 \
-    --hash=sha256:074958fe4578ef3a3d0bdaf96bbc25e4c4db82b7ff523594776fcf3d3f16c531 \
+    --hash=sha256:df277cb51e61359aba502208d680f90c0493adec6f0e848af94948778aed386e
-    --hash=sha256:2db667ee21f620b446a54a793e467714fc5a446fcc82d93a47e8bde01d69afab \
+yarl==1.4.2 \
-    --hash=sha256:326f2dbaaa17b858ae86f261ae73a266fd820a561fc5142cee9d0fc58448fbd7 \
+    --hash=sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce \
-    --hash=sha256:32a3885f542f74d0f4f87057050c6b45529ebd79d0639f56582e741521575bfe \
+    --hash=sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6 \
-    --hash=sha256:56126ef061b913c3eefecace3404ca88917265d0550b8e32bbbeab29e5c830bf \
+    --hash=sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce \
-    --hash=sha256:589ac1e82add13fbdedc04eb0a83400db728e5f1af2bd273392088ca90de7062 \
+    --hash=sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae \
-    --hash=sha256:6076bce2ecc6ebf6c92919d77762f80f4c9c6ecc9c1fbaa16567ec59ad7d6f1d \
+    --hash=sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d \
-    --hash=sha256:63be649c535d18ab6230efbc06a07f7779cd4336a687672defe70c025349a47b \
+    --hash=sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f \
-    --hash=sha256:6642cbc92eaffa586180f669adc772f5c34977e9e849e93f33dc142351e98c9c \
+    --hash=sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b \
-    --hash=sha256:6fa05a25f2280e78a514041d4609d39962e7d51525f2439db9ad7a2ae7aac163 \
+    --hash=sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b \
-    --hash=sha256:7ed006a220422c33ff0889288be24db56ff0a3008ffe9eaead58a690715ad09b \
+    --hash=sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb \
-    --hash=sha256:80c9c213803b50899460cc355f47e66778c3c868f448b7b7de5b1f1858c82c2a \
+    --hash=sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462 \
-    --hash=sha256:8bae18e2129850e76969b57869dacc72a66cccdbeebce1a28d7f3d439c21a7a3 \
+    --hash=sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea \
-    --hash=sha256:ab112fba996a8f48f427e26969f2066d50080df0c24007a8cc6d7ae865e19013 \
+    --hash=sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70 \
-    --hash=sha256:b1c178ef813940c9a5cbad42ab7b8b76ac08b594b0a6bad91063c968e0466efc \
+    --hash=sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1 \
-    --hash=sha256:d6eff151c3b23a56a5e4f496805619bc3bdf4f749f63a7a95ad50e8267c17475
+    --hash=sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a \
-zipp==0.6.0 \
+    --hash=sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b \
-    --hash=sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e \
+    --hash=sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080 \
-    --hash=sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335
+    --hash=sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2
-colorama==0.4.1 \
+zipp==3.1.0 \
-    --hash=sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d \
+    --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \
-    --hash=sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48
+    --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96
+colorama==0.4.3 \
+    --hash=sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff \
+    --hash=sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1

contrib/freeze_packages.sh

@@ -6,13 +6,24 @@ set -e
 venv_dir=~/.electrum-venv
 contrib=$(dirname "$0")
 
-which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit 1; }
+# note: we should not use a higher version of python than what the binaries bundle
-python3 -m hashin -h > /dev/null 2>&1 || { python3 -m pip install hashin; }
+if [[ ! "$SYSTEM_PYTHON" ]] ; then
-other_python=$(which python3)
+    SYSTEM_PYTHON=$(which python3.6) || printf ""
+else
+    SYSTEM_PYTHON=$(which $SYSTEM_PYTHON) || printf ""
+fi
+if [[ ! "$SYSTEM_PYTHON" ]] ; then
+    echo "Please specify which python to use in \$SYSTEM_PYTHON" && exit 1;
+fi
 
-for i in '' '-hw' '-binaries' '-wine-build'; do
+which virtualenv > /dev/null 2>&1 || { echo "Please install virtualenv" && exit 1; }
+
+
+${SYSTEM_PYTHON} -m hashin -h > /dev/null 2>&1 || { ${SYSTEM_PYTHON} -m pip install hashin; }
+
+for i in '' '-hw' '-binaries' '-wine-build' '-mac-build' '-sdist-build'; do
     rm -rf "$venv_dir"
-    virtualenv -p $(which python3) $venv_dir
+    virtualenv -p ${SYSTEM_PYTHON} $venv_dir
 
     source $venv_dir/bin/activate
 
@@ -23,7 +34,7 @@ for i in '' '-hw' '-binaries' '-wine-build'; do
     echo "OK."
 
     requirements=$(pip freeze --all)
-    restricted=$(echo $requirements | $other_python $contrib/deterministic-build/find_restricted_dependencies.py)
+    restricted=$(echo $requirements | ${SYSTEM_PYTHON} $contrib/deterministic-build/find_restricted_dependencies.py)
     requirements="$requirements $restricted"
 
     echo "Generating package hashes..."
@@ -32,7 +43,7 @@ for i in '' '-hw' '-binaries' '-wine-build'; do
 
     for requirement in $requirements; do
         echo -e "\r  Hashing $requirement..."
-        $other_python -m hashin -r $contrib/deterministic-build/requirements${i}.txt ${requirement}
+        ${SYSTEM_PYTHON} -m hashin -r $contrib/deterministic-build/requirements${i}.txt ${requirement}
     done
 
     echo "OK."

contrib/make_libsecp256k1.sh

@@ -1,6 +1,16 @@
 #!/bin/bash
 
-LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
+# This script was tested on Linux and MacOS hosts, where it can be used
+# to build native libsecp256k1 binaries.
+#
+# It can also be used to cross-compile to Windows:
+# $ sudo apt-get install mingw-w64
+# For a Windows x86 (32-bit) target, run:
+# $ GCC_TRIPLET_HOST="i686-w64-mingw32" ./contrib/make_libsecp256k1.sh
+# Or for a Windows x86_64 (64-bit) target, run:
+# $ GCC_TRIPLET_HOST="x86_64-w64-mingw32" ./contrib/make_libsecp256k1.sh
+
+LIBSECP_VERSION="dbd41db16a0e91b2566820898a3ab2d7dad4fe00"
 
 set -e
 
@@ -19,9 +29,13 @@ info "Building $pkgname..."
         git clone https://github.com/bitcoin-core/secp256k1.git
     fi
     cd secp256k1
+    if ! $(git cat-file -e ${LIBSECP_VERSION}) ; then
+    info "Could not find requested version $LIBSECP_VERSION in local clone; fetching..."
+    git fetch --all
+    fi
     git reset --hard
     git clean -f -x -q
-    git checkout $LIBSECP_VERSION
+    git checkout "${LIBSECP_VERSION}^{commit}"
 
     if ! [ -x configure ] ; then
         echo "libsecp256k1_la_LDFLAGS = -no-undefined" >> Makefile.am
@@ -35,8 +49,9 @@ info "Building $pkgname..."
             --enable-module-recovery \
             --enable-experimental \
             --enable-module-ecdh \
-            --disable-jni \
+            --disable-benchmark \
             --disable-tests \
+            --disable-exhaustive-tests \
             --disable-static \
             --enable-shared || fail "Could not configure $pkgname. Please make sure you have a C compiler installed and try again."
     fi

contrib/make_packages

@@ -6,5 +6,5 @@ test -n "$CONTRIB" -a -d "$CONTRIB" || exit
 rm "$CONTRIB"/../packages/ -r
 
 #Install pure python modules in electrum directory
-python3 -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages
+python3 -m pip install --no-dependencies --no-binary :all: \
-
+    -r "$CONTRIB"/deterministic-build/requirements.txt -t "$CONTRIB"/../packages

contrib/osx/README.md

@@ -1,5 +1,5 @@
-Building Mac OS binaries
+Building macOS binaries
-========================
+=======================
 
 ✗ _This script does not produce reproducible output (yet!).
    Please help us remedy this._
@@ -7,36 +7,48 @@ Building Mac OS binaries
 This guide explains how to build Electrum binaries for macOS systems.
 
 
-## 1. Building the binary
+## Building the binary
 
-This needs to be done on a system running macOS or OS X. We use El Capitan (10.11.6) as building it
+This needs to be done on a system running macOS or OS X.
-on High Sierra (or later)
-makes the binaries [incompatible with older versions](https://github.com/pyinstaller/pyinstaller/issues/1191).
 
-Another factor for the minimum supported macOS version is the
+Notes about compatibility with different macOS versions:
-[bundled Qt version](https://github.com/spesmilo/electrum/issues/3685).
+- In general the binary is not guaranteed to run on an older version of macOS
+  than what the build machine has. This is due to bundling the compiled Python into
+  the [PyInstaller binary](https://github.com/pyinstaller/pyinstaller/issues/1191).
+- The [bundled version of Qt](https://github.com/spesmilo/electrum/issues/3685) also
+  imposes a minimum supported macOS version.
+- If you want to build binaries that conform to the macOS "Gatekeeper", so as to
+  minimise the warnings users get, the binaries need to be codesigned with a
+  certificate issued by Apple, and starting with macOS 10.15 the binaries also
+  need to be notarized by Apple's central server. The catch is that to be able to build
+  binaries that Apple will notarise (due to the requirements on the binaries themselves,
+  e.g. hardened runtime) the build machine needs at least macOS 10.14.
+  See [#6128](https://github.com/spesmilo/electrum/issues/6128).
+
+We currently build the release binaries on macOS 10.14.6, and these seem to run on
+10.13 or newer.
 
 Before starting, make sure that the Xcode command line tools are installed (e.g. you have `git`).
 
-#### 1.1a Get Xcode
+#### 1.a Get Xcode
 
 Building the QR scanner (CalinsQRReader) requires full Xcode (not just command line tools).
 
-The last Xcode version compatible with El Capitan is Xcode 8.2.1
-
 Get it from [here](https://developer.apple.com/download/more/).
 
 Unfortunately, you need an "Apple ID" account.
 
+(note: the last Xcode that runs on macOS 10.14.6 is Xcode 11.3.1)
+
 After downloading, uncompress it.
 
 Make sure it is the "selected" xcode (e.g.):
 
     sudo xcode-select -s $HOME/Downloads/Xcode.app/Contents/Developer/
 
-#### 1.1b Build QR scanner separately on newer Mac
+#### 1.b Build QR scanner separately on another Mac
 
-Alternatively, you can try building just the QR scanner on newer macOS.
+Alternatively, you can try building just the QR scanner on another Mac.
 
 On newer Mac, run:
 
@@ -46,27 +58,17 @@ On newer Mac, run:
 Move `prebuilt_qr` to El Capitan: `contrib/osx/CalinsQRReader/prebuilt_qr`.
 
 
-#### 1.2 Build Electrum
+#### 2. Build Electrum
 
     cd electrum
     ./contrib/osx/make_osx
 
 This creates both a folder named Electrum.app and the .dmg file.
 
+If you want the binaries codesigned for MacOS and notarised by Apple's central server,
+provide these env vars to the `make_osx` script:
 
-## 2. Building the image deterministically (WIP)
+    CODESIGN_CERT="Developer ID Application: Electrum Technologies GmbH (L6P37P7P56)" \
-The usual way to distribute macOS applications is to use image files containing the 
+    APPLE_ID_USER="me@email.com" \
-application. Although these images can be created on a Mac with the built-in `hdiutil`,
+    APPLE_ID_PASSWORD="1234" \
-they are not deterministic.
+    ./contrib/osx/make_osx
-
-Instead, we use the toolchain that Bitcoin uses: genisoimage and libdmg-hfsplus.
-These tools do not work on macOS, so you need a separate Linux machine (or VM).
-
-Copy the Electrum.app directory over and install the dependencies, e.g.:
-
-    apt install libcap-dev cmake make gcc faketime
-    
-Then you can just invoke `package.sh` with the path to the app:
-
-    cd electrum
-    ./contrib/osx/package.sh ~/Electrum.app/

contrib/osx/base.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-. $(dirname "$0")/../build_tools_util.sh
-
-
-function DoCodeSignMaybe { # ARGS: infoName fileOrDirName codesignIdentity
-    infoName="$1"
-    file="$2"
-    identity="$3"
-    deep=""
-    if [ -z "$identity" ]; then
-        # we are ok with them not passing anything; master script calls us unconditionally even if no identity is specified
-        return
-    fi
-    if [ -d "$file" ]; then
-        deep="--deep"
-    fi
-    if [ -z "$infoName" ] || [ -z "$file" ] || [ -z "$identity" ] || [ ! -e "$file" ]; then
-        fail "Argument error to internal function DoCodeSignMaybe()"
-    fi
-    info "Code signing ${infoName}..."
-    codesign -f -v $deep -s "$identity" "$file" || fail "Could not code sign ${infoName}"
-}

contrib/osx/entitlements.plist

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <!-- These are required for binaries built by PyInstaller -->
+    <!-- see pyinstaller/pyinstaller#4629 -->
+    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+    <true/>
+    <key>com.apple.security.cs.disable-library-validation</key>
+    <true/>
+
+    <!-- These are required for USB HID access (hw wallets). -->
+    <!-- see https://github.com/Electron-Cash/Electron-Cash/commit/5abec73eee0cdeb725e3c5a989621ec4ccfb92a0 -->
+    <key>com.apple.security.cs.allow-dyld-environment-variables</key>
+    <true/>
+    <key>com.apple.security.cs.allow-jit</key>
+    <true/>
+
+    <!-- Camera access, to read QR codes -->
+    <key>com.apple.security.device.camera</key>
+    <true/>
+</dict>
+</plist>

contrib/osx/make_osx

@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 
 # Parameterize
-PYTHON_VERSION=3.7.6
+PYTHON_VERSION=3.7.7
 BUILDDIR=/tmp/electrum-build
 PACKAGE=Electrum
 GIT_REPO=https://github.com/spesmilo/electrum
-LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
+PYTHON_VERSION=3.7.7
 
 export GCC_STRIP_BINARIES="1"
 
-. $(dirname "$0")/base.sh
+. $(dirname "$0")/../build_tools_util.sh
 
 CONTRIB_OSX="$(dirname "$(realpath "$0")")"
 CONTRIB="$CONTRIB_OSX/.."
@@ -24,26 +24,44 @@ which brew > /dev/null 2>&1 || fail "Please install brew from https://brew.sh/ t
 which xcodebuild > /dev/null 2>&1 || fail "Please install Xcode and xcode command line tools to continue"
 
 # Code Signing: See https://developer.apple.com/library/archive/documentation/Security/Conceptual/CodeSigningGuide/Procedures/Procedures.html
-APP_SIGN=""
+if [ -n "$CODESIGN_CERT" ]; then
-if [ -n "$1" ]; then
     # Test the identity is valid for signing by doing this hack. There is no other way to do this.
     cp -f /bin/ls ./CODESIGN_TEST
-    codesign -s "$1" --dryrun -f ./CODESIGN_TEST > /dev/null 2>&1
+    codesign -s "$CODESIGN_CERT" --dryrun -f ./CODESIGN_TEST > /dev/null 2>&1
     res=$?
     rm -f ./CODESIGN_TEST
     if ((res)); then
-        fail "Code signing identity \"$1\" appears to be invalid."
+        fail "Code signing identity \"$CODESIGN_CERT\" appears to be invalid."
     fi
     unset res
-    APP_SIGN="$1"
+    info "Code signing enabled using identity \"$CODESIGN_CERT\""
-    info "Code signing enabled using identity \"$APP_SIGN\""
 else
-    warn "Code signing DISABLED. Specify a valid macOS Developer identity installed on the system as the first argument to this script to enable signing."
+    warn "Code signing DISABLED. Specify a valid macOS Developer identity installed on the system to enable signing."
 fi
 
+function DoCodeSignMaybe { # ARGS: infoName fileOrDirName
+    infoName="$1"
+    file="$2"
+    deep=""
+    if [ -z "$CODESIGN_CERT" ]; then
+        # no cert -> we won't codesign
+        return
+    fi
+    if [ -d "$file" ]; then
+        deep="--deep"
+    fi
+    if [ -z "$infoName" ] || [ -z "$file" ] || [ ! -e "$file" ]; then
+        fail "Argument error to internal function DoCodeSignMaybe()"
+    fi
+    hardened_arg="--entitlements=${CONTRIB_OSX}/entitlements.plist -o runtime"
+
+    info "Code signing ${infoName}..."
+    codesign -f -v $deep -s "$CODESIGN_CERT" $hardened_arg "$file" || fail "Could not code sign ${infoName}"
+}
+
 info "Installing Python $PYTHON_VERSION"
 export PATH="~/.pyenv/bin:~/.pyenv/shims:~/Library/Python/3.7/bin:$PATH"
-if [ -d "~/.pyenv" ]; then
+if [ -d "${HOME}/.pyenv" ]; then
   pyenv update
 else
   curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash > /dev/null 2>&1
@@ -53,14 +71,9 @@ pyenv global $PYTHON_VERSION || \
 fail "Unable to use Python $PYTHON_VERSION"
 
 
-info "install dependencies specific to binaries"
+info "Installing build dependencies"
-# note that this also installs pinned versions of both pip and setuptools
+python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements-mac-build.txt --user \
-python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements-binaries.txt --user \
+    || fail "Could not install build dependencies"
-    || fail "Could not install pyinstaller"
-
-
-info "Installing pyinstaller"
-python3 -m pip install -I --user pyinstaller==3.6 || fail "Could not install pyinstaller"
 
 info "Using these versions for building $PACKAGE:"
 sw_vers
@@ -91,10 +104,10 @@ info "generating locale"
 
 
 info "Downloading libusb..."
-curl https://homebrew.bintray.com/bottles/libusb-1.0.22.el_capitan.bottle.tar.gz | \
+curl https://homebrew.bintray.com/bottles/libusb-1.0.23.high_sierra.bottle.tar.gz | \
 tar xz --directory $BUILDDIR
-cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/osx
+cp $BUILDDIR/libusb/1.0.23/lib/libusb-1.0.dylib contrib/osx
-echo "82c368dfd4da017ceb32b12ca885576f325503428a4966cc09302cbd62702493  contrib/osx/libusb-1.0.dylib" | \
+echo "caea266f3fc3982adc55d6cb8d9bad10f6e61f0c24ce5901aa1804618e08e14d  contrib/osx/libusb-1.0.dylib" | \
     shasum -a 256 -c || fail "libusb checksum mismatched"
 
 info "Preparing for building libsecp256k1"
@@ -109,16 +122,20 @@ rm -fr build
 # prefer building using xcode ourselves. otherwise fallback to prebuilt binary
 xcodebuild || cp -r prebuilt_qr build || fail "Could not build CalinsQRReader"
 popd
-DoCodeSignMaybe "CalinsQRReader.app" "${d}/build/Release/CalinsQRReader.app" "$APP_SIGN" # If APP_SIGN is empty will be a noop
+DoCodeSignMaybe "CalinsQRReader.app" "${d}/build/Release/CalinsQRReader.app"
 
 
 info "Installing requirements..."
-python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements.txt --user || \
+python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements.txt --user \
-fail "Could not install requirements"
+    || fail "Could not install requirements"
 
 info "Installing hardware wallet requirements..."
-python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements-hw.txt --user || \
+python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements-hw.txt --user \
-fail "Could not install hardware wallet requirements"
+    || fail "Could not install hardware wallet requirements"
+
+info "Installing dependencies specific to binaries..."
+python3 -m pip install --no-dependencies -Ir ./contrib/deterministic-build/requirements-binaries.txt --user \
+    || fail "Could not install dependencies specific to binaries"
 
 info "Building $PACKAGE..."
 python3 -m pip install --no-dependencies --user . > /dev/null || fail "Could not build $PACKAGE"
@@ -131,7 +148,7 @@ for d in ~/Library/Python/ ~/.pyenv .; do
 done
 
 info "Building binary"
-APP_SIGN="$APP_SIGN" pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/osx/osx.spec || fail "Could not build binary"
+APP_SIGN="$CODESIGN_CERT" pyinstaller --noconfirm --ascii --clean --name $VERSION contrib/osx/osx.spec || fail "Could not build binary"
 
 info "Adding bitcoin URI types to Info.plist"
 plutil -insert 'CFBundleURLTypes' \
@@ -139,14 +156,23 @@ plutil -insert 'CFBundleURLTypes' \
 	-- dist/$PACKAGE.app/Contents/Info.plist \
 	|| fail "Could not add keys to Info.plist. Make sure the program 'plutil' exists and is installed."
 
-DoCodeSignMaybe "app bundle" "dist/${PACKAGE}.app" "$APP_SIGN" # If APP_SIGN is empty will be a noop
+DoCodeSignMaybe "app bundle" "dist/${PACKAGE}.app"
+
+if [ ! -z "$CODESIGN_CERT" ]; then
+    if [ ! -z "$APPLE_ID_USER" ]; then
+        info "Notarizing .app with Apple's central server..."
+        "${CONTRIB_OSX}/notarize_app.sh" "dist/${PACKAGE}.app" || fail "Could not notarize binary."
+    else
+        warn "AppleID details not set! Skipping Apple notarization."
+    fi
+fi
 
 info "Creating .DMG"
 hdiutil create -fs HFS+ -volname $PACKAGE -srcfolder dist/$PACKAGE.app dist/electrum-$VERSION.dmg || fail "Could not create .DMG"
 
-DoCodeSignMaybe ".DMG" "dist/electrum-${VERSION}.dmg" "$APP_SIGN" # If APP_SIGN is empty will be a noop
+DoCodeSignMaybe ".DMG" "dist/electrum-${VERSION}.dmg"
 
-if [ -z "$APP_SIGN" ]; then
+if [ -z "$CODESIGN_CERT" ]; then
     warn "App was built successfully but was not code signed. Users may get security warnings from macOS."
-    warn "Specify a valid code signing identity as the first argument to this script to enable code signing."
+    warn "Specify a valid code signing identity to enable code signing."
 fi

contrib/osx/notarize_app.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# from https://github.com/metabrainz/picard/blob/e1354632d2db305b7a7624282701d34d73afa225/scripts/package/macos-notarize-app.sh
+
+
+if [ -z "$1" ]; then
+    echo "Specify app bundle as first parameter"
+    exit 1
+fi
+
+if [ -z "$APPLE_ID_USER" ] || [ -z "$APPLE_ID_PASSWORD" ]; then
+    echo "You need to set your Apple ID credentials with \$APPLE_ID_USER and \$APPLE_ID_PASSWORD."
+    exit 1
+fi
+
+APP_BUNDLE=$(basename "$1")
+APP_BUNDLE_DIR=$(dirname "$1")
+
+cd "$APP_BUNDLE_DIR" || exit 1
+
+# Package app for submission
+echo "Generating ZIP archive ${APP_BUNDLE}.zip..."
+ditto -c -k --rsrc --keepParent "$APP_BUNDLE" "${APP_BUNDLE}.zip"
+
+# Submit for notarization
+echo "Submitting $APP_BUNDLE for notarization..."
+RESULT=$(xcrun altool --notarize-app --type osx \
+  --file "${APP_BUNDLE}.zip" \
+  --primary-bundle-id org.electrum.electrum \
+  --username $APPLE_ID_USER \
+  --password @env:APPLE_ID_PASSWORD \
+  --output-format xml)
+
+if [ $? -ne 0 ]; then
+  echo "Submitting $APP_BUNDLE failed:"
+  echo "$RESULT"
+  exit 1
+fi
+
+REQUEST_UUID=$(echo "$RESULT" | xpath \
+  "//key[normalize-space(text()) = 'RequestUUID']/following-sibling::string[1]/text()" 2> /dev/null)
+
+if [ -z "$REQUEST_UUID" ]; then
+  echo "Submitting $APP_BUNDLE failed:"
+  echo "$RESULT"
+  exit 1
+fi
+
+echo "$(echo "$RESULT" | xpath \
+  "//key[normalize-space(text()) = 'success-message']/following-sibling::string[1]/text()" 2> /dev/null)"
+
+# Poll for notarization status
+echo "Submitted notarization request $REQUEST_UUID, waiting for response..."
+sleep 60
+while :
+do
+  RESULT=$(xcrun altool --notarization-info "$REQUEST_UUID" \
+    --username "$APPLE_ID_USER" \
+    --password @env:APPLE_ID_PASSWORD \
+    --output-format xml)
+  STATUS=$(echo "$RESULT" | xpath \
+    "//key[normalize-space(text()) = 'Status']/following-sibling::string[1]/text()" 2> /dev/null)
+
+  if [ "$STATUS" = "success" ]; then
+    echo "Notarization of $APP_BUNDLE succeeded!"
+    break
+  elif [ "$STATUS" = "in progress" ]; then
+    echo "Notarization in progress..."
+    sleep 20
+  else
+    echo "Notarization of $APP_BUNDLE failed:"
+    echo "$RESULT"
+    exit 1
+  fi
+done
+
+# Staple the notary ticket
+xcrun stapler staple "$APP_BUNDLE"

contrib/osx/osx.spec

@@ -59,21 +59,19 @@ block_cipher = None
 
 # see https://github.com/pyinstaller/pyinstaller/issues/2005
 hiddenimports = []
+hiddenimports += collect_submodules('pkg_resources')  # workaround for https://github.com/pypa/setuptools/issues/1963
 hiddenimports += collect_submodules('trezorlib')
 hiddenimports += collect_submodules('safetlib')
 hiddenimports += collect_submodules('btchip')
 hiddenimports += collect_submodules('keepkeylib')
 hiddenimports += collect_submodules('websocket')
 hiddenimports += collect_submodules('ckcc')
+hiddenimports += collect_submodules('bitbox02')
 hiddenimports += ['PyQt5.QtPrintSupport']  # needed by Revealer
 
-# safetlib imports PyQt5.Qt.  We use a local updated copy of pinmatrix.py until they
-# release a new version that includes https://github.com/archos-safe-t/python-safet/commit/b1eab3dba4c04fdfc1fcf17b66662c28c5f2380e
-hiddenimports.remove('safetlib.qt.pinmatrix')
-
-
 datas = [
     (electrum + PYPKG + '/*.json', PYPKG),
+    (electrum + PYPKG + '/lnwire/*.csv', PYPKG + '/lnwire'),
     (electrum + PYPKG + '/wordlist/english.txt', PYPKG + '/wordlist'),
     (electrum + PYPKG + '/locale', PYPKG + '/locale'),
     (electrum + PYPKG + '/plugins', PYPKG + '/plugins'),
@@ -84,8 +82,7 @@ datas += collect_data_files('safetlib')
 datas += collect_data_files('btchip')
 datas += collect_data_files('keepkeylib')
 datas += collect_data_files('ckcc')
-datas += collect_data_files('jsonrpcserver')
+datas += collect_data_files('bitbox02')
-datas += collect_data_files('jsonrpcclient')
 
 # Add the QR Scanner helper app
 datas += [(electrum + "contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app", "./contrib/osx/CalinsQRReader/build/Release/CalinsQRReader.app")]
@@ -142,18 +139,23 @@ if APP_SIGN:
 
 pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
 
-exe = EXE(pyz,
+exe = EXE(
+    pyz,
     a.scripts,
-          a.binaries,
+    exclude_binaries=True,
-          a.datas,
+    name=MAIN_SCRIPT,
-          name=PACKAGE,
     debug=False,
     strip=False,
     upx=True,
     icon=electrum+ICONS_FILE,
-          console=False)
+    console=False,
+)
 
-app = BUNDLE(exe,
+app = BUNDLE(
+    exe,
+    a.binaries,
+    a.zipfiles,
+    a.datas,
     version = VERSION,
     name=PACKAGE + '.app',
     icon=electrum+ICONS_FILE,
@@ -161,5 +163,5 @@ app = BUNDLE(exe,
     info_plist={
         'NSHighResolutionCapable': 'True',
         'NSSupportsAutomaticGraphicsSwitching': 'True'
-             }
+    },
 )

contrib/requirements/requirements-binaries.txt

@@ -1,2 +1,2 @@
-PyQt5<5.12
+PyQt5<5.15
-PyQt5-sip<=4.19.13
+pycryptodomex>=3.7

contrib/requirements/requirements-hw.txt

@@ -8,9 +8,10 @@
 # see https://github.com/spesmilo/electrum/issues/5859
 Cython>=0.27
 
-trezor[hidapi]>=0.11.5
+trezor[hidapi]>=0.12.0
-safet[hidapi]>=0.1.0
+safet>=0.1.5
 keepkey>=6.3.1
-btchip-python>=0.1.26
+btchip-python>=0.1.30
 ckcc-protocol>=0.7.7
+bitbox02>=4.0.0
 hidapi

contrib/requirements/requirements-mac-build.txt

@@ -0,0 +1,6 @@
+pip
+setuptools
+pyinstaller>=3.6
+
+# needed by pyinstaller:
+macholib

contrib/requirements/requirements-sdist-build.txt

@@ -0,0 +1,3 @@
+# need modern versions of pip (and maybe other build tools), the one in apt had issues
+pip
+setuptools

contrib/requirements/requirements-travis.txt

@@ -1,3 +1,3 @@
 tox
-python-coveralls
+coveralls
 tox-travis

contrib/requirements/requirements.txt

@@ -1,15 +1,12 @@
 pyaes>=0.1a1
 ecdsa>=0.14
 qrcode
-protobuf
+protobuf>=3.12
-dnspython
+dnspython<2.0
-qdarkstyle<2.7
+qdarkstyle<2.9
 aiorpcx>=0.18,<0.19
 aiohttp>=3.3.0,<4.0.0
-aiohttp_socks
+aiohttp_socks>=0.3
 certifi
 bitstring
-pycryptodomex>=3.7
+attrs>=19.2.0
-jsonrpcserver
-jsonrpcclient
-attrs

contrib/udev/53-hid-bitbox02.rules

@@ -0,0 +1 @@
+SUBSYSTEM=="usb", TAG+="uaccess", TAG+="udev-acl", SYMLINK+="bitbox02_%n", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2403"

contrib/udev/54-hid-bitbox02.rules

@@ -0,0 +1 @@
+KERNEL=="hidraw*", SUBSYSTEM=="hidraw", ATTRS{idVendor}=="03eb", ATTRS{idProduct}=="2403", TAG+="uaccess", TAG+="udev-acl", SYMLINK+="bitbox02-%n"

contrib/udev/README.md

@@ -6,7 +6,8 @@ These are necessary for the devices to be usable on Linux environments.
 
  - `20-hw1.rules` (Ledger): https://github.com/LedgerHQ/udev-rules/blob/master/20-hw1.rules
  - `51-coinkite.rules` (Coldcard): https://github.com/Coldcard/ckcc-protocol/blob/master/51-coinkite.rules
- - `51-hid-digitalbitbox.rules`, `52-hid-digitalbitbox.rules` (Digital Bitbox): https://shiftcrypto.ch/start_linux
+ - `51-hid-digitalbitbox.rules`, `52-hid-digitalbitbox.rules` (Digital Bitbox): https://github.com/digitalbitbox/bitbox-wallet-app/blob/master/frontends/qt/resources/deb-afterinstall.sh
+ - `53-hid-bitbox02.rules`, `54-hid-bitbox02.rules` (BitBox02): https://github.com/digitalbitbox/bitbox-wallet-app/blob/master/frontends/qt/resources/deb-afterinstall.sh
  - `51-trezor.rules` (Trezor): https://github.com/trezor/trezor-common/blob/master/udev/51-trezor.rules
  - `51-usb-keepkey.rules` (Keepkey): https://github.com/keepkey/udev-rules/blob/master/51-usb-keepkey.rules
  - `51-safe-t.rules` (Archos): https://github.com/archos-safe-t/safe-t-common/blob/master/udev/51-safe-t.rules

electrum/address_synchronizer.py

@@ -28,7 +28,7 @@ import itertools
 from collections import defaultdict
 from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple, NamedTuple, Sequence, List
 
-from . import bitcoin
+from . import bitcoin, util
 from .bitcoin import COINBASE_MATURITY
 from .util import profiler, bfh, TxMinedInfo
 from .transaction import Transaction, TxOutput, TxInput, PartialTxInput, TxOutpoint, PartialTransaction
@@ -70,13 +70,17 @@ class AddressSynchronizer(Logger):
     inherited by wallet
     """
 
+    network: Optional['Network']
+    synchronizer: Optional['Synchronizer']
+    verifier: Optional['SPV']
+
     def __init__(self, db: 'WalletDB'):
         self.db = db
-        self.network = None  # type: Network
+        self.network = None
         Logger.__init__(self)
         # verifier (SPV) and synchronizer are started in start_network
-        self.synchronizer = None  # type: Synchronizer
+        self.synchronizer = None
-        self.verifier = None  # type: SPV
+        self.verifier = None
         # locks: if you need to take multiple ones, acquire them in the order they are defined here!
         self.lock = threading.RLock()
         self.transaction_lock = threading.RLock()
@@ -156,17 +160,17 @@ class AddressSynchronizer(Logger):
                 # add it in case it was previously unconfirmed
                 self.add_unverified_tx(tx_hash, tx_height)
 
-    def start_network(self, network):
+    def start_network(self, network: Optional['Network']) -> None:
         self.network = network
         if self.network is not None:
             self.synchronizer = Synchronizer(self)
             self.verifier = SPV(self.network, self)
-            self.network.register_callback(self.on_blockchain_updated, ['blockchain_updated'])
+            util.register_callback(self.on_blockchain_updated, ['blockchain_updated'])
 
     def on_blockchain_updated(self, event, *args):
         self._get_addr_balance_cache = {}  # invalidate cache
 
-    def stop_threads(self):
+    def stop(self):
         if self.network:
             if self.synchronizer:
                 asyncio.run_coroutine_threadsafe(self.synchronizer.stop(), self.network.asyncio_loop)
@@ -174,7 +178,7 @@ class AddressSynchronizer(Logger):
             if self.verifier:
                 asyncio.run_coroutine_threadsafe(self.verifier.stop(), self.network.asyncio_loop)
                 self.verifier = None
-            self.network.unregister_callback(self.on_blockchain_updated)
+            util.unregister_callback(self.on_blockchain_updated)
             self.db.put('stored_height', self.get_local_height())
 
     def add_address(self, address):
@@ -345,7 +349,7 @@ class AddressSynchronizer(Logger):
                     prevout = TxOutpoint(bfh(tx_hash), idx)
                     self.db.remove_prevout_by_scripthash(scripthash, prevout=prevout, value=txo.value)
 
-    def get_depending_transactions(self, tx_hash):
+    def get_depending_transactions(self, tx_hash: str) -> Set[str]:
         """Returns all (grand-)children of tx_hash in this wallet."""
         with self.transaction_lock:
             children = set()
@@ -449,7 +453,7 @@ class AddressSynchronizer(Logger):
         domain = set(domain)
         # 1. Get the history of each address in the domain, maintain the
         #    delta of a tx as the sum of its deltas on domain addresses
-        tx_deltas = defaultdict(int)
+        tx_deltas = defaultdict(int)  # type: Dict[str, Optional[int]]
         for addr in domain:
             h = self.get_address_history(addr)
             for tx_hash, height in h:
@@ -546,7 +550,7 @@ class AddressSynchronizer(Logger):
             self.unverified_tx.pop(tx_hash, None)
             self.db.add_verified_tx(tx_hash, info)
         tx_mined_status = self.get_tx_height(tx_hash)
-        self.network.trigger_callback('verified', self, tx_hash, tx_mined_status)
+        util.trigger_callback('verified', self, tx_hash, tx_mined_status)
 
     def get_unverified_txs(self):
         '''Returns a map from tx hash to transaction height'''
@@ -584,13 +588,17 @@ class AddressSynchronizer(Logger):
             return cached_local_height
         return self.network.get_local_height() if self.network else self.db.get('stored_height', 0)
 
-    def add_future_tx(self, tx: Transaction, num_blocks: int) -> None:
+    def add_future_tx(self, tx: Transaction, num_blocks: int) -> bool:
         assert num_blocks > 0, num_blocks
         with self.lock:
-            self.add_transaction(tx)
+            tx_was_added = self.add_transaction(tx)
+            if tx_was_added:
                 self.future_tx[tx.txid()] = num_blocks
+            return tx_was_added
 
     def get_tx_height(self, tx_hash: str) -> TxMinedInfo:
+        if tx_hash is None:  # ugly backwards compat...
+            return TxMinedInfo(height=TX_HEIGHT_LOCAL, conf=0)
         with self.lock:
             verified_tx_mined_info = self.db.get_verified_tx(tx_hash)
             if verified_tx_mined_info:
@@ -609,9 +617,12 @@ class AddressSynchronizer(Logger):
 
     def set_up_to_date(self, up_to_date):
         with self.lock:
+            status_changed = self.up_to_date != up_to_date
             self.up_to_date = up_to_date
         if self.network:
             self.network.notify('status')
+        if status_changed:
+            self.logger.info(f'set_up_to_date: {up_to_date}')
 
     def is_up_to_date(self):
         with self.lock: return self.up_to_date
@@ -751,29 +762,34 @@ class AddressSynchronizer(Logger):
                     sent[txi] = height
         return received, sent
 
-    def get_addr_utxo(self, address: str) -> Dict[TxOutpoint, PartialTxInput]:
+    def get_addr_outputs(self, address: str) -> Dict[TxOutpoint, PartialTxInput]:
         coins, spent = self.get_addr_io(address)
-        for txi in spent:
-            coins.pop(txi)
         out = {}
         for prevout_str, v in coins.items():
             tx_height, value, is_cb = v
             prevout = TxOutpoint.from_str(prevout_str)
-            utxo = PartialTxInput(prevout=prevout,
+            utxo = PartialTxInput(prevout=prevout, is_coinbase_output=is_cb)
-                                  is_coinbase_output=is_cb)
             utxo._trusted_address = address
             utxo._trusted_value_sats = value
             utxo.block_height = tx_height
+            utxo.spent_height = spent.get(prevout_str, None)
             out[prevout] = utxo
         return out
 
+    def get_addr_utxo(self, address: str) -> Dict[TxOutpoint, PartialTxInput]:
+        out = self.get_addr_outputs(address)
+        for k, v in list(out.items()):
+            if v.spent_height is not None:
+                out.pop(k)
+        return out
+
     # return the total amount ever received by an address
     def get_addr_received(self, address):
         received, sent = self.get_addr_io(address)
         return sum([v for height, v, is_cb in received.values()])
 
     @with_local_height_cached
-    def get_addr_balance(self, address, *, excluded_coins: Set[str] = None):
+    def get_addr_balance(self, address, *, excluded_coins: Set[str] = None) -> Tuple[int, int, int]:
         """Return the balance of a bitcoin address:
         confirmed and matured, unconfirmed, unmatured
         """

electrum/base_crash_reporter.py

@@ -75,7 +75,7 @@ class BaseCrashReporter(Logger):
 
     async def do_post(self, proxy, url, data):
         async with make_aiohttp_session(proxy) as session:
-            async with session.post(url, data=data) as resp:
+            async with session.post(url, data=data, raise_for_status=True) as resp:
                 return await resp.text()
 
     def get_traceback_info(self):
@@ -121,15 +121,18 @@ class BaseCrashReporter(Logger):
             ['git', 'describe', '--always', '--dirty'], cwd=dir)
         return str(version, "utf8").strip()
 
+    def _get_traceback_str(self) -> str:
+        return "".join(traceback.format_exception(*self.exc_args))
+
     def get_report_string(self):
         info = self.get_additional_info()
-        info["traceback"] = "".join(traceback.format_exception(*self.exc_args))
+        info["traceback"] = self._get_traceback_str()
         return self.issue_template.format(**info)
 
     def get_user_description(self):
         raise NotImplementedError
 
-    def get_wallet_type(self):
+    def get_wallet_type(self) -> str:
         raise NotImplementedError
 
 

electrum/base_wizard.py

@@ -28,20 +28,19 @@ import sys
 import copy
 import traceback
 from functools import partial
-from typing import List, TYPE_CHECKING, Tuple, NamedTuple, Any, Dict, Optional
+from typing import List, TYPE_CHECKING, Tuple, NamedTuple, Any, Dict, Optional, Union
 
 from . import bitcoin
 from . import keystore
 from . import mnemonic
 from .bip32 import is_bip32_derivation, xpub_type, normalize_bip32_derivation, BIP32Node
-from .keystore import bip44_derivation, purpose48_derivation, Hardware_KeyStore, KeyStore
+from .keystore import bip44_derivation, purpose48_derivation, Hardware_KeyStore, KeyStore, bip39_to_seed
 from .wallet import (Imported_Wallet, Standard_Wallet, Multisig_Wallet,
                      wallet_types, Wallet, Abstract_Wallet)
-from .storage import (WalletStorage, StorageEncryptionVersion,
+from .storage import WalletStorage, StorageEncryptionVersion
-                      get_derivation_used_for_hw_device_encryption)
 from .wallet_db import WalletDB
 from .i18n import _
-from .util import UserCancelled, InvalidPassword, WalletFileException
+from .util import UserCancelled, InvalidPassword, WalletFileException, UserFacingException
 from .simple_config import SimpleConfig
 from .plugin import Plugins, HardwarePluginLibraryUnavailable
 from .logging import Logger
@@ -61,6 +60,12 @@ class ScriptTypeNotSupported(Exception): pass
 class GoBack(Exception): pass
 
 
+class ReRunDialog(Exception): pass
+
+
+class ChooseHwDeviceAgain(Exception): pass
+
+
 class WizardStackItem(NamedTuple):
     action: Any
     args: Any
@@ -114,18 +119,21 @@ class BaseWizard(Logger):
     def can_go_back(self):
         return len(self._stack) > 1
 
-    def go_back(self):
+    def go_back(self, *, rerun_previous: bool = True) -> None:
         if not self.can_go_back():
             return
         # pop 'current' frame
         self._stack.pop()
-        # pop 'previous' frame
+        prev_frame = self._stack[-1]
-        stack_item = self._stack.pop()
         # try to undo side effects since we last entered 'previous' frame
-        # FIXME only self.storage is properly restored
+        # FIXME only self.data is properly restored
-        self.data = copy.deepcopy(stack_item.db_data)
+        self.data = copy.deepcopy(prev_frame.db_data)
+
+        if rerun_previous:
+            # pop 'previous' frame
+            self._stack.pop()
             # rerun 'previous' frame
-        self.run(stack_item.action, *stack_item.args, **stack_item.kwargs)
+            self.run(prev_frame.action, *prev_frame.args, **prev_frame.kwargs)
 
     def reset_stack(self):
         self._stack = []
@@ -145,7 +153,7 @@ class BaseWizard(Logger):
         self.choice_dialog(title=title, message=message, choices=choices, run_next=self.on_wallet_type)
 
     def upgrade_db(self, storage, db):
-        exc = None
+        exc = None  # type: Optional[Exception]
         def on_finished():
             if exc is None:
                 self.terminate(storage=storage, db=db)
@@ -159,6 +167,15 @@ class BaseWizard(Logger):
                 exc = e
         self.waiting_dialog(do_upgrade, _('Upgrading wallet format...'), on_finished=on_finished)
 
+    def run_task_without_blocking_gui(self, task, *, msg: str = None) -> Any:
+        """Perform a task in a thread without blocking the GUI.
+        Returns the result of 'task', or raises the same exception.
+        This method blocks until 'task' is finished.
+        """
+        raise NotImplementedError()
+
+
+
     def load_2fa(self):
         self.data['wallet_type'] = '2fa'
         self.data['use_trustedcoin'] = True
@@ -254,7 +271,16 @@ class BaseWizard(Logger):
         k = keystore.from_master_key(text)
         self.on_keystore(k)
 
-    def choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage=None):
+    ef choose_hw_device(self, purpose=HWD_SETUP_NEW_WALLET, *, storage: WalletStorage = None):
+        while True:
+            try:
+                self._choose_hw_device(purpose=purpose, storage=storage)
+            except ChooseHwDeviceAgain:
+                pass
+            else:
+                break
+
+    def _choose_hw_device(self, *, purpose, storage: WalletStorage = None):
         title = _('Hardware Keystore')
         # check available plugins
         supported_plugins = self.plugins.get_hardware_support()
@@ -271,7 +297,8 @@ class BaseWizard(Logger):
 
         # scan devices
         try:
-            scanned_devices = devmgr.scan_devices()
+            scanned_devices = self.run_task_without_blocking_gui(task=devmgr.scan_devices,
+                                                     msg=_("Scanning devices..."))
         except BaseException as e:
             self.logger.info('error scanning devices: {}'.format(repr(e)))
             debug_msg = '  {}:\n    {}'.format(_('Error scanning devices'), e)
@@ -317,8 +344,8 @@ class BaseWizard(Logger):
             msg += '\n\n'
             msg += _('Debug message') + '\n' + debug_msg
             self.confirm_dialog(title=title, message=msg,
-                                run_next=lambda x: self.choose_hw_device(purpose, storage=storage))
+                                run_next=lambda x: None)
-            return
+raise ChooseHwDeviceAgain()
         # select device
         self.devices = devices
         choices = []
@@ -327,36 +354,37 @@ class BaseWizard(Logger):
             label = info.label or _("An unnamed {}").format(name)
             try: transport_str = info.device.transport_ui_string[:20]
             except: transport_str = 'unknown transport'
-            descr = f"{label} [{name}, {state}, {transport_str}]"
+            descr = f"{label} [{info.model_name or name}, {state}, {transport_str}]"
             choices.append(((name, info), descr))
         msg = _('Select a device') + ':'
         self.choice_dialog(title=title, message=msg, choices=choices,
                            run_next=lambda *args: self.on_device(*args, purpose=purpose, storage=storage))
 
-    def on_device(self, name, device_info, *, purpose, storage=None):
+    def on_device(self, name, device_info: 'DeviceInfo', *, purpose, storage: WalletStorage = None):
-        self.plugin = self.plugins.get_plugin(name)  # type: HW_PluginBase
+        self.plugin = self.plugins.get_plugin(name)
+        assert isinstance(self.plugin, HW_PluginBase)
+        devmgr = self.plugins.device_manager
         try:
-            self.plugin.setup_device(device_info, self, purpose)
+            client = self.plugin.setup_device(device_info, self, purpose)
         except OSError as e:
             self.show_error(_('We encountered an error while connecting to your device:')
                             + '\n' + str(e) + '\n'
                             + _('To try to fix this, we will now re-pair with your device.') + '\n'
                             + _('Please try again.'))
-            devmgr = self.plugins.device_manager
+
             devmgr.unpair_id(device_info.device.id_)
-            self.choose_hw_device(purpose, storage=storage)
+            raise ChooseHwDeviceAgain()
-            return
         except OutdatedHwFirmwareException as e:
             if self.question(e.text_ignore_old_fw_and_continue(), title=_("Outdated device firmware")):
                 self.plugin.set_ignore_outdated_fw()
                 # will need to re-pair
-                devmgr = self.plugins.device_manager
                 devmgr.unpair_id(device_info.device.id_)
-            self.choose_hw_device(purpose, storage=storage)
+            raise ChooseHwDeviceAgain()
-            return
         except (UserCancelled, GoBack):
-            self.choose_hw_device(purpose, storage=storage)
+            raise ChooseHwDeviceAgain()
-            return
+        except UserFacingException as e:
+            self.show_error(str(e))
+            raise ChooseHwDeviceAgain()
         except BaseException as e:
             self.logger.exception('')
             self.show_error(str(e))
@@ -368,22 +396,18 @@ class BaseWizard(Logger):
                 self.run('on_hw_derivation', name, device_info, derivation, script_type)
             self.derivation_and_script_type_dialog(f)
         elif purpose == HWD_SETUP_DECRYPT_WALLET:
-            derivation = get_derivation_used_for_hw_device_encryption()
+            password = client.get_password_for_storage_encryption()
-            xpub = self.plugin.get_xpub(device_info.device.id_, derivation, 'standard', self)
-            password = keystore.Xpub.get_pubkey_from_xpub(xpub, ()).hex()
             try:
                 storage.decrypt(password)
             except InvalidPassword:
                 # try to clear session so that user can type another passphrase
-                devmgr = self.plugins.device_manager
-                client = devmgr.client_by_id(device_info.device.id_)
                 if hasattr(client, 'clear_session'):  # FIXME not all hw wallet plugins have this
                     client.clear_session()
                 raise
         else:
             raise Exception('unknown purpose: %s' % purpose)
 
-    def derivation_and_script_type_dialog(self, f):
+    def derivation_and_script_type_dialog(self, f, *, get_account_xpub=None):
         message1 = _('Choose the type of addresses in your wallet.')
         message2 = ' '.join([
             _('You can override the suggested derivation path.'),
@@ -407,29 +431,32 @@ class BaseWizard(Logger):
             ]
         while True:
             try:
-                self.choice_and_line_dialog(
+                self.derivation_and_script_type_gui_specific_dialog(
                     run_next=f, title=_('Script type and Derivation path'), message1=message1,
                     message2=message2, choices=choices, test_text=is_bip32_derivation,
-                    default_choice_idx=default_choice_idx)
+                    default_choice_idx=default_choice_idx, get_account_xpub=get_account_xpub)
                 return
             except ScriptTypeNotSupported as e:
                 self.show_error(e)
                 # let the user choose again
 
-    def on_hw_derivation(self, name, device_info, derivation, xtype):
+    def on_hw_derivation(self, name, device_info: 'DeviceInfo', derivation, xtype):
         from .keystore import hardware_keystore
         devmgr = self.plugins.device_manager
+        assert isinstance(self.plugin, HW_PluginBase)
         try:
             xpub = self.plugin.get_xpub(device_info.device.id_, derivation, xtype, self)
-            client = devmgr.client_by_id(device_info.device.id_)
+            client = devmgr.client_by_id(device_info.device.id_, scan_now=False)
             if not client: raise Exception("failed to find client for device id")
             root_fingerprint = client.request_root_fingerprint_from_device()
+            label = client.label()  # use this as device_info.label might be outdated!
+            soft_device_id = client.get_soft_device_id()  # use this as device_info.device_id might be outdated!
         except ScriptTypeNotSupported:
             raise  # this is handled in derivation_dialog
         except BaseException as e:
             self.logger.exception('')
             self.show_error(e)
-            return
+            raise ChooseHwDeviceAgain()
         d = {
             'type': 'hardware',
             'hw_type': name,
@@ -466,7 +493,8 @@ class BaseWizard(Logger):
     def on_restore_seed(self, seed, is_bip39, is_ext):
         self.seed_type = 'bip39' if is_bip39 else mnemonic.seed_type(seed)
         if self.seed_type == 'bip39':
-            f = lambda passphrase: self.on_restore_bip39(seed, passphrase)
+            def f(passphrase):
+                self.on_restore_bip39(seed, passphrase)
             self.passphrase_dialog(run_next=f, is_restoring=True) if is_ext else f('')
         elif self.seed_type in ['standard', 'segwit']:
             f = lambda passphrase: self.run('create_keystore', seed, passphrase)
@@ -483,7 +511,13 @@ class BaseWizard(Logger):
         def f(derivation, script_type):
             derivation = normalize_bip32_derivation(derivation)
             self.run('on_bip43', seed, passphrase, derivation, script_type)
-        self.derivation_and_script_type_dialog(f)
+        def get_account_xpub(account_path):
+            root_seed = bip39_to_seed(seed, passphrase)
+            root_node = BIP32Node.from_rootseed(root_seed, xtype="standard")
+            account_node = root_node.subkey_at_private_derivation(account_path)
+            account_xpub = account_node.to_xpub()
+            return account_xpub
+        self.derivation_and_script_type_dialog(f, get_account_xpub=get_account_xpub)
 
     def create_keystore(self, seed, passphrase):
         k = keystore.from_seed(seed, passphrase, self.wallet_type == 'multisig')
@@ -520,12 +554,12 @@ class BaseWizard(Logger):
                     self.show_error(_('Cannot add this cosigner:') + '\n' + "Their key type is '%s', we are '%s'"%(t1, t2))
                     self.run('choose_keystore')
                     return
-            self.keystores.append(k)
+            if len(self.keystores) == 0:
-            if len(self.keystores) == 1:
                 xpub = k.get_master_public_key()
-                self.reset_stack()
                 self.run('show_xpub_and_add_cosigners', xpub)
-            elif len(self.keystores) < self.n:
+            self.reset_stack()
+            self.keystores.append(k)
+            if len(self.keystores) < self.n:
                 self.run('choose_keystore')
             else:
                 self.run('create_wallet')
@@ -537,18 +571,19 @@ class BaseWizard(Logger):
         if self.wallet_type == 'standard' and isinstance(self.keystores[0], Hardware_KeyStore):
             # offer encrypting with a pw derived from the hw device
             k = self.keystores[0]  # type: Hardware_KeyStore
+            assert isinstance(self.plugin, HW_PluginBase)
             try:
                 k.handler = self.plugin.create_handler(self)
                 password = k.get_password_for_storage_encryption()
             except UserCancelled:
                 devmgr = self.plugins.device_manager
                 devmgr.unpair_xpub(k.xpub)
-                self.choose_hw_device()
+                raise ChooseHwDeviceAgain()
-                return
+
             except BaseException as e:
                 self.logger.exception('')
                 self.show_error(str(e))
-                return
+                raise ChooseHwDeviceAgain()
             self.request_storage_encryption(
                 run_next=lambda encrypt_storage: self.on_password(
                     password,
@@ -593,11 +628,10 @@ class BaseWizard(Logger):
         self.terminate()
 
 
-    def create_storage(self, path):
+    def create_storage(self, path) -> Tuple[WalletStorage, WalletDB]:
         if os.path.exists(path):
             raise Exception('file already exists at path')
-        if not self.pw_args:
+        assert self.pw_args, f"pw_args not set?!"
-            return
         pw_args = self.pw_args
         self.pw_args = None  # clean-up so that it can get GC-ed
         storage = WalletStorage(path)
@@ -611,11 +645,13 @@ class BaseWizard(Logger):
         db.write(storage)
         return storage, db
 
-    def terminate(self, *, storage: Optional[WalletStorage], db: Optional[WalletDB] = None):
+    def terminate(self, *, storage: WalletStorage = None,
+              db: WalletDB = None,
+              aborted: bool = False) -> None:
         raise NotImplementedError()  # implemented by subclasses
 
     def show_xpub_and_add_cosigners(self, xpub):
-        self.show_xpub_dialog(xpub=xpub, run_next=lambda x: self.run('choose_keystore'))
+        self.show_xpub_dialog(xpub=xpub, run_next=lambda x: None)
 
     def choose_seed_type(self, message=None, choices=None):
         title = _('Choose Seed type')
@@ -666,3 +702,6 @@ class BaseWizard(Logger):
             self.line_dialog(run_next=f, title=title, message=message, default='', test=lambda x: x==passphrase)
         else:
             f('')
+
+    def show_error(self, msg: Union[str, BaseException]) -> None:
+        raise NotImplementedError()

electrum/bip32.py

@@ -401,3 +401,25 @@ def root_fp_and_der_prefix_from_xkey(xkey: str) -> Tuple[Optional[str], Optional
         derivation_prefix = convert_bip32_intpath_to_strpath([child_number_int])
         root_fingerprint = node.fingerprint.hex()
     return root_fingerprint, derivation_prefix
+
+def is_xkey_consistent_with_key_origin_info(xkey: str, *,
+                                            derivation_prefix: str = None,
+                                            root_fingerprint: str = None) -> bool:
+    bip32node = BIP32Node.from_xkey(xkey)
+    int_path = None
+    if derivation_prefix is not None:
+        int_path = convert_bip32_path_to_list_of_uint32(derivation_prefix)
+    if int_path is not None and len(int_path) != bip32node.depth:
+        return False
+    if bip32node.depth == 0:
+        if bfh(root_fingerprint) != bip32node.calc_fingerprint_of_this_node():
+            return False
+        if bip32node.child_number != bytes(4):
+            return False
+    if int_path is not None and bip32node.depth > 0:
+        if int.from_bytes(bip32node.child_number, 'big') != int_path[-1]:
+            return False
+    if bip32node.depth == 1:
+        if bfh(root_fingerprint) != bip32node.fingerprint:
+            return False
+    return True

electrum/bip39_recovery.py

@@ -0,0 +1,75 @@
+ Copyright (C) 2020 The Electrum developers
+# Distributed under the MIT software license, see the accompanying
+# file LICENCE or http://www.opensource.org/licenses/mit-license.php
+
+from typing import TYPE_CHECKING
+
+from aiorpcx import TaskGroup
+
+from . import bitcoin
+from .constants import BIP39_WALLET_FORMATS
+from .bip32 import BIP32_PRIME, BIP32Node
+from .bip32 import convert_bip32_path_to_list_of_uint32 as bip32_str_to_ints
+from .bip32 import convert_bip32_intpath_to_strpath as bip32_ints_to_str
+
+if TYPE_CHECKING:
+    from .network import Network
+
+
+async def account_discovery(network: 'Network', get_account_xpub):
+    async with TaskGroup() as group:
+        account_scan_tasks = []
+        for wallet_format in BIP39_WALLET_FORMATS:
+            account_scan = scan_for_active_accounts(network, get_account_xpub, wallet_format)
+            account_scan_tasks.append(await group.spawn(account_scan))
+    active_accounts = []
+    for task in account_scan_tasks:
+        active_accounts.extend(task.result())
+    return active_accounts
+
+
+async def scan_for_active_accounts(network: 'Network', get_account_xpub, wallet_format):
+    active_accounts = []
+    account_path = bip32_str_to_ints(wallet_format["derivation_path"])
+    while True:
+        account_xpub = get_account_xpub(account_path)
+        account_node = BIP32Node.from_xkey(account_xpub)
+        has_history = await account_has_history(network, account_node, wallet_format["script_type"])
+        if has_history:
+            account = format_account(wallet_format, account_path)
+            active_accounts.append(account)
+        if not has_history or not wallet_format["iterate_accounts"]:
+            break
+        account_path[-1] = account_path[-1] + 1
+    return active_accounts
+
+
+async def account_has_history(network: 'Network', account_node: BIP32Node, script_type: str) -> bool:
+    gap_limit = 20
+    async with TaskGroup() as group:
+        get_history_tasks = []
+        for address_index in range(gap_limit):
+            address_node = account_node.subkey_at_public_derivation("0/" + str(address_index))
+            pubkey = address_node.eckey.get_public_key_hex()
+            address = bitcoin.pubkey_to_address(script_type, pubkey)
+            script = bitcoin.address_to_script(address)
+            scripthash = bitcoin.script_to_scripthash(script)
+            get_history = network.get_history_for_scripthash(scripthash)
+            get_history_tasks.append(await group.spawn(get_history))
+    for task in get_history_tasks:
+        history = task.result()
+        if len(history) > 0:
+            return True
+    return False
+
+
+def format_account(wallet_format, account_path):
+    description = wallet_format["description"]
+    if wallet_format["iterate_accounts"]:
+        account_index = account_path[-1] % BIP32_PRIME
+        description = f'{description} (Account {account_index})'
+    return {
+        "description": description,
+        "derivation_path": bip32_ints_to_str(account_path),
+        "script_type": wallet_format["script_type"],
+    }

electrum/bip39_wallet_formats.json

@@ -0,0 +1,80 @@
+[
+    {
+        "description": "Standard BIP44 legacy",
+        "derivation_path": "m/44'/0'/0'",
+        "script_type": "p2pkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Standard BIP49 compatibility segwit",
+        "derivation_path": "m/49'/0'/0'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Standard BIP84 native segwit",
+        "derivation_path": "m/84'/0'/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard legacy",
+        "derivation_path": "m/0'",
+        "script_type": "p2pkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard compatibility segwit",
+        "derivation_path": "m/0'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Non-standard native segwit",
+        "derivation_path": "m/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Copay native segwit",
+        "derivation_path": "m/44'/0'/0'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": true
+    },
+    {
+        "description": "Samourai Bad Bank (toxic change)",
+        "derivation_path": "m/84'/0'/2147483644'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Whirlpool Pre Mix",
+        "derivation_path": "m/84'/0'/2147483645'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Whirlpool Post Mix",
+        "derivation_path": "m/84'/0'/2147483646'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet legacy",
+        "derivation_path": "m/44'/0'/2147483647'",
+        "script_type": "p2pkh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet compatibility segwit",
+        "derivation_path": "m/49'/0'/2147483647'",
+        "script_type": "p2wpkh-p2sh",
+        "iterate_accounts": false
+    },
+    {
+        "description": "Samourai Ricochet native segwit",
+        "derivation_path": "m/84'/0'/2147483647'",
+        "script_type": "p2wpkh",
+        "iterate_accounts": false
+    }
+]

electrum/bitcoin.py

@@ -25,7 +25,8 @@
 
 import hashlib
 from typing import List, Tuple, TYPE_CHECKING, Optional, Union
-from enum import IntEnum
+import enum
+from enum import IntEnum, Enum
 
 from .util import bfh, bh2u, BitcoinException, assert_bytes, to_bytes, inv_dict
 from . import version
@@ -44,6 +45,10 @@ COINBASE_MATURITY = 100
 COIN = 100000000
 TOTAL_COIN_SUPPLY_LIMIT_IN_BTC = 21000000
 
+NLOCKTIME_MIN = 0
+NLOCKTIME_BLOCKHEIGHT_MAX = 500_000_000 - 1
+NLOCKTIME_MAX = 2 ** 32 - 1
+
 # supported types of transaction outputs
 # TODO kill these with fire
 TYPE_ADDRESS = 0
@@ -295,19 +300,28 @@ def add_number_to_script(i: int) -> bytes:
 
 
 def relayfee(network: 'Network' = None) -> int:
+    """Returns feerate in sat/kbyte."""
     from .simple_config import FEERATE_DEFAULT_RELAY, FEERATE_MAX_RELAY
     if network and network.relay_fee is not None:
         fee = network.relay_fee
     else:
         fee = FEERATE_DEFAULT_RELAY
     fee = min(fee, FEERATE_MAX_RELAY)
-    fee = max(fee, 0)
+    fee = max(fee, FEERATE_DEFAULT_RELAY)
     return fee
 
 
-def dust_threshold(network: 'Network'=None) -> int:
+# see https://github.com/bitcoin/bitcoin/blob/a62f0ed64f8bbbdfe6467ac5ce92ef5b5222d1bd/src/policy/policy.cpp#L14
+DUST_LIMIT_DEFAULT_SAT_LEGACY = 546
+DUST_LIMIT_DEFAULT_SAT_SEGWIT = 294
+
+
+def dust_threshold(network: 'Network' = None) -> int:
+    """Returns the dust limit in satoshis."""
     # Change <= dust threshold is added to the tx fee
-    return 182 * 3 * relayfee(network) // 1000
+dust_lim = 182 * 3 * relayfee(network)  # in msat
+# convert to sat, but round up:
+return (dust_lim // 1000) + (dust_lim % 1000 > 0)
 
 
 def hash_encode(x: bytes) -> str:
@@ -423,6 +437,39 @@ def address_to_script(addr: str, *, net=None) -> str:
         raise BitcoinException(f'unknown address type: {addrtype}')
     return script
 
+class OnchainOutputType(Enum):
+    """Opaque types of scriptPubKeys.
+    In case of p2sh, p2wsh and similar, no knowledge of redeem script, etc.
+    """
+    P2PKH = enum.auto()
+    P2SH = enum.auto()
+    WITVER0_P2WPKH = enum.auto()
+    WITVER0_P2WSH = enum.auto()
+
+
+def address_to_hash(addr: str, *, net=None) -> Tuple[OnchainOutputType, bytes]:
+    """Return (type, pubkey hash / witness program) for an address."""
+    if net is None: net = constants.net
+    if not is_address(addr, net=net):
+        raise BitcoinException(f"invalid lbry address: {addr}")
+    witver, witprog = segwit_addr.decode(net.SEGWIT_HRP, addr)
+    if witprog is not None:
+        if witver != 0:
+            raise BitcoinException(f"not implemented handling for witver={witver}")
+        if len(witprog) == 20:
+            return OnchainOutputType.WITVER0_P2WPKH, bytes(witprog)
+        elif len(witprog) == 32:
+            return OnchainOutputType.WITVER0_P2WSH, bytes(witprog)
+        else:
+            raise BitcoinException(f"unexpected length for segwit witver=0 witprog: len={len(witprog)}")
+    addrtype, hash_160_ = b58_address_to_hash160(addr)
+    if addrtype == net.ADDRTYPE_P2PKH:
+        return OnchainOutputType.P2PKH, hash_160_
+    elif addrtype == net.ADDRTYPE_P2SH:
+        return OnchainOutputType.P2SH, hash_160_
+    raise BitcoinException(f"unknown address type: {addrtype}")
+
+
 def address_to_scripthash(addr: str) -> str:
     script = address_to_script(addr)
     return script_to_scripthash(script)
@@ -556,8 +603,8 @@ def is_segwit_script_type(txin_type: str) -> bool:
     return txin_type in ('p2wpkh', 'p2wpkh-p2sh', 'p2wsh', 'p2wsh-p2sh')
 
 
-def serialize_privkey(secret: bytes, compressed: bool, txin_type: str,
+def serialize_privkey(secret: bytes, compressed: bool, txin_type: str, *,
-                      internal_use: bool=False) -> str:
+                      internal_use: bool = False) -> str:
     # we only export secrets inside curve range
     secret = ecc.ECPrivkey.normalize_secret_bytes(secret)
     if internal_use:

electrum/blockchain.py

@@ -22,6 +22,7 @@
 # SOFTWARE.
 import os
 import threading
+import time
 from typing import Optional, Dict, Mapping, Sequence
 
 import hashlib
@@ -189,6 +190,19 @@ _CHAINWORK_CACHE = {
     "0000000000000000000000000000000000000000000000000000000000000000": 0,  # virtual block at height -1
 }  # type: Dict[str, int]
 
+def init_headers_file_for_best_chain():
+    b = get_best_chain()
+    filename = b.path()
+    length = HEADER_SIZE * len(constants.net.CHECKPOINTS) * 2016
+    if not os.path.exists(filename) or os.path.getsize(filename) < length:
+        with open(filename, 'wb') as f:
+            if length > 0:
+                f.seek(length - 1)
+                f.write(b'\x00')
+        util.ensure_sparse_file(filename)
+    with b.lock:
+        b.update_size()
+
 
 class Blockchain(Logger):
     """
@@ -503,6 +517,20 @@ class Blockchain(Logger):
         height = self.height()
         return self.read_header(height)
 
+    def is_tip_stale(self) -> bool:
+        STALE_DELAY = 8 * 60 * 60  # in seconds
+        header = self.header_at_tip()
+        if not header:
+            return True
+        # note: We check the timestamp only in the latest header.
+        #       The Bitcoin consensus has a lot of leeway here:
+        #       - needs to be greater than the median of the timestamps of the past 11 blocks, and
+        #       - up to at most 2 hours into the future compared to local clock
+        #       so there is ~2 hours of leeway in either direction
+        if header['timestamp'] + STALE_DELAY < time.time():
+            return True
+        return False
+
     def get_hash(self, height: int) -> str:
         def is_height_checkpoint():
             within_cp_range = height <= constants.net.max_checkpoint()
@@ -729,6 +757,14 @@ def can_connect(header: dict) -> Optional[Blockchain]:
             return b
     return None
 
+def get_chains_that_contain_header(height: int, header_hash: str) -> Sequence[Blockchain]:
+    """Returns a list of Blockchains that contain header, best chain first."""
+    with blockchains_lock: chains = list(blockchains.values())
+    chains = [chain for chain in chains
+              if chain.check_hash(height=height, header_hash=header_hash)]
+    chains = sorted(chains, key=lambda x: x.get_chainwork(), reverse=True)
+    return chains
+
 class ArithUint256:
     # https://github.com/bitcoin/bitcoin/blob/master/src/arith_uint256.cpp
 

electrum/channel_db.py

@@ -31,26 +31,22 @@ from typing import Sequence, List, Tuple, Optional, Dict, NamedTuple, TYPE_CHECK
 import binascii
 import base64
 import asyncio
+import threading
+from enum import IntEnum
 
 
 from .sql_db import SqlDB, sql
-from . import constants
+from . import constants, util
 from .util import bh2u, profiler, get_headers_dir, bfh, is_ip_address, list_enabled_bits
 from .logging import Logger
-from .lnutil import LN_GLOBAL_FEATURES_KNOWN_SET, LNPeerAddr, format_short_channel_id, ShortChannelID
+from .lnutil import (LNPeerAddr, format_short_channel_id, ShortChannelID,
+                     validate_features, IncompatibleOrInsaneFeatures)
 from .lnverifier import LNChannelVerifier, verify_sig_for_channel_update
+from .lnmsg import decode_msg
 
 if TYPE_CHECKING:
     from .network import Network
-
+    from .lnchannel import Channel
-
-class UnknownEvenFeatureBits(Exception): pass
-
-def validate_features(features : int):
-    enabled_features = list_enabled_bits(features)
-    for fbit in enabled_features:
-        if (1 << fbit) not in LN_GLOBAL_FEATURES_KNOWN_SET and fbit % 2 == 0:
-            raise UnknownEvenFeatureBits()
 
 
 FLAG_DISABLE   = 1 << 1
@@ -63,7 +59,7 @@ class ChannelInfo(NamedTuple):
     capacity_sat: Optional[int]
 
     @staticmethod
-    def from_msg(payload):
+    def from_msg(payload: dict) -> 'ChannelInfo':
         features = int.from_bytes(payload['features'], 'big')
         validate_features(features)
         channel_id = payload['short_channel_id']
@@ -79,6 +75,11 @@ class ChannelInfo(NamedTuple):
         )
 
 
+    @staticmethod
+    def from_raw_msg(raw: bytes) -> 'ChannelInfo':
+        payload_dict = decode_msg(raw)[1]
+        return ChannelInfo.from_msg(payload_dict)
+
 class Policy(NamedTuple):
     key: bytes
     cltv_expiry_delta: int
@@ -91,19 +92,25 @@ class Policy(NamedTuple):
     timestamp: int
 
     @staticmethod
-    def from_msg(payload):
+    def from_msg(payload: dict) -> 'Policy':
         return Policy(
             key                         = payload['short_channel_id'] + payload['start_node'],
-            cltv_expiry_delta           = int.from_bytes(payload['cltv_expiry_delta'], "big"),
+            cltv_expiry_delta           = payload['cltv_expiry_delta'],
-            htlc_minimum_msat           = int.from_bytes(payload['htlc_minimum_msat'], "big"),
+            htlc_minimum_msat           = payload['htlc_minimum_msat'],
-            htlc_maximum_msat           = int.from_bytes(payload['htlc_maximum_msat'], "big") if 'htlc_maximum_msat' in payload else None,
+            htlc_maximum_msat           = payload.get('htlc_maximum_msat', None),
-            fee_base_msat               = int.from_bytes(payload['fee_base_msat'], "big"),
+            fee_base_msat               = payload['fee_base_msat'],
-            fee_proportional_millionths = int.from_bytes(payload['fee_proportional_millionths'], "big"),
+            fee_proportional_millionths = payload['fee_proportional_millionths'],
             message_flags               = int.from_bytes(payload['message_flags'], "big"),
             channel_flags               = int.from_bytes(payload['channel_flags'], "big"),
-            timestamp                   = int.from_bytes(payload['timestamp'], "big")
+            timestamp                   = payload['timestamp'],
         )
 
+    @staticmethod
+    def from_raw_msg(key:bytes, raw: bytes) -> 'Policy':
+        payload = decode_msg(raw)[1]
+        payload['start_node'] = key[8:]
+        return Policy.from_msg(payload)
+
     def is_disabled(self):
         return self.channel_flags & FLAG_DISABLE
 
@@ -112,7 +119,7 @@ class Policy(NamedTuple):
         return ShortChannelID.normalize(self.key[0:8])
 
     @property
-    def start_node(self):
+    def start_node(self) -> bytes:
         return self.key[8:]
 
 
@@ -124,15 +131,30 @@ class NodeInfo(NamedTuple):
     alias: str
 
     @staticmethod
-    def from_msg(payload):
+    def from_msg(payload) -> Tuple['NodeInfo', Sequence['LNPeerAddr']]:
         node_id = payload['node_id']
         features = int.from_bytes(payload['features'], "big")
         validate_features(features)
         addresses = NodeInfo.parse_addresses_field(payload['addresses'])
+        peer_addrs = []
+        for host, port in addresses:
+            try:
+                peer_addrs.append(LNPeerAddr(host=host, port=port, pubkey=node_id))
+            except ValueError:
+                pass
         alias = payload['alias'].rstrip(b'\x00')
-        timestamp = int.from_bytes(payload['timestamp'], "big")
+        try:
-        return NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias), [
+            alias = alias.decode('utf8')
-            Address(host=host, port=port, node_id=node_id, last_connected_date=None) for host, port in addresses]
+        except:
+            alias = ''
+        timestamp = payload['timestamp']
+        node_info = NodeInfo(node_id=node_id, features=features, timestamp=timestamp, alias=alias)
+        return node_info, peer_addrs
+
+    @staticmethod
+    def from_raw_msg(raw: bytes) -> Tuple['NodeInfo', Sequence['LNPeerAddr']]:
+        payload_dict = decode_msg(raw)[1]
+        return NodeInfo.from_msg(payload_dict)
 
     @staticmethod
     def parse_addresses_field(addresses_field):
@@ -175,50 +197,41 @@ class NodeInfo(NamedTuple):
         return addresses
 
 
-class Address(NamedTuple):
+class UpdateStatus(IntEnum):
-    node_id: bytes
+    ORPHANED   = 0
-    host: str
+    EXPIRED    = 1
-    port: int
+    DEPRECATED = 2
-    last_connected_date: Optional[int]
+    UNCHANGED  = 3
+    GOOD       = 4
 
 
 class CategorizedChannelUpdates(NamedTuple):
     orphaned: List    # no channel announcement for channel update
     expired: List     # update older than two weeks
     deprecated: List  # update older than database entry
+    unchanged: List   # unchanged policies
     good: List        # good updates
-    to_delete: List   # database entries to delete
 
 
-# TODO It would make more sense to store the raw gossip messages in the db.
+
-#      That is pretty much a pre-requisite of actively participating in gossip.
 
 create_channel_info = """
 CREATE TABLE IF NOT EXISTS channel_info (
-short_channel_id VARCHAR(64),
+short_channel_id BLOB(8),
-node1_id VARCHAR(66),
+msg BLOB,
-node2_id VARCHAR(66),
-capacity_sat INTEGER,
 PRIMARY KEY(short_channel_id)
 )"""
 
 create_policy = """
 CREATE TABLE IF NOT EXISTS policy (
-key VARCHAR(66),
+key BLOB(41),
-cltv_expiry_delta INTEGER NOT NULL,
+msg BLOB,
-htlc_minimum_msat INTEGER NOT NULL,
-htlc_maximum_msat INTEGER,
-fee_base_msat INTEGER NOT NULL,
-fee_proportional_millionths INTEGER NOT NULL,
-channel_flags INTEGER NOT NULL,
-message_flags INTEGER NOT NULL,
-timestamp INTEGER NOT NULL,
 PRIMARY KEY(key)
 )"""
 
 create_address = """
 CREATE TABLE IF NOT EXISTS address (
-node_id VARCHAR(66),
+node_id BLOB(33),
 host STRING(256),
 port INTEGER NOT NULL,
 timestamp INTEGER,
@@ -227,10 +240,8 @@ PRIMARY KEY(node_id, host, port)
 
 create_node_info = """
 CREATE TABLE IF NOT EXISTS node_info (
-node_id VARCHAR(66),
+node_id BLOB(33),
-features INTEGER NOT NULL,
+msg BLOB,
-timestamp INTEGER NOT NULL,
-alias STRING(64),
 PRIMARY KEY(node_id)
 )"""
 
@@ -240,19 +251,26 @@ class ChannelDB(SqlDB):
     NUM_MAX_RECENT_PEERS = 20
 
     def __init__(self, network: 'Network'):
-        path = os.path.join(get_headers_dir(network.config), 'channel_db')
+        path = os.path.join(get_headers_dir(network.config), 'gossip_db')
-        super().__init__(network, path, commit_interval=100)
+        super().__init__(network.asyncio_loop, path, commit_interval=100)
+        self.lock = threading.RLock()
         self.num_nodes = 0
         self.num_channels = 0
         self._channel_updates_for_private_channels = {}  # type: Dict[Tuple[bytes, bytes], dict]
         self.ca_verifier = LNChannelVerifier(network, self)
+
         # initialized in load_data
-        self._channels = {}  # type: Dict[bytes, ChannelInfo]
+        # note: modify/iterate needs self.lock
-        self._policies = {}
+        self._channels = {}  # type: Dict[ShortChannelID, ChannelInfo]
-        self._nodes = {}
+        self._policies = {}  # type: Dict[Tuple[bytes, ShortChannelID], Policy]  # (node_id, scid) -> Policy
+        self._nodes = {}  # type: Dict[bytes, NodeInfo]  # node_id -> NodeInfo
         # node_id -> (host, port, ts)
         self._addresses = defaultdict(set)  # type: Dict[bytes, Set[Tuple[str, int, int]]]
-        self._channels_for_node = defaultdict(set)
+        self._channels_for_node = defaultdict(set)  # type: Dict[bytes, Set[ShortChannelID]]
+        self._recent_peers = []  # type: List[bytes]  # list of node_ids
+        self._chans_with_0_policies = set()  # type: Set[ShortChannelID]
+        self._chans_with_1_policies = set()  # type: Set[ShortChannelID]
+        self._chans_with_2_policies = set()  # type: Set[ShortChannelID]
         self.data_loaded = asyncio.Event()
         self.network = network # only for callback
 
@@ -260,18 +278,27 @@ class ChannelDB(SqlDB):
         self.num_nodes = len(self._nodes)
         self.num_channels = len(self._channels)
         self.num_policies = len(self._policies)
-        self.network.trigger_callback('channel_db', self.num_nodes, self.num_channels, self.num_policies)
+        util.trigger_callback('channel_db', self.num_nodes, self.num_channels, self.num_policies)
+        util.trigger_callback('ln_gossip_sync_progress')
 
     def get_channel_ids(self):
+        with self.lock:
             return set(self._channels.keys())
 
     def add_recent_peer(self, peer: LNPeerAddr):
         now = int(time.time())
         node_id = peer.pubkey
+        with self.lock:
             self._addresses[node_id].add((peer.host, peer.port, now))
-        self.save_node_address(node_id, peer, now)
+            # list is ordered
+            if node_id in self._recent_peers:
+                self._recent_peers.remove(node_id)
+            self._recent_peers.insert(0, node_id)
+            self._recent_peers = self._recent_peers[:self.NUM_MAX_RECENT_PEERS]
+        self._db_save_node_address(peer, now)
 
     def get_200_randomly_sorted_nodes_not_in(self, node_ids):
+        with self.lock:
             unshuffled = set(self._nodes.keys()) - node_ids
         return random.sample(unshuffled, min(200, len(unshuffled)))
 
@@ -287,16 +314,15 @@ class ChannelDB(SqlDB):
             return None
 
     def get_recent_peers(self):
-        assert self.data_loaded.is_set(), "channelDB load_data did not finish yet!"
+        if not self.data_loaded.is_set():
-        # FIXME this does not reliably return "recent" peers...
+            raise Exception("channelDB data not loaded yet!")
-        #       Also, the list() cast over the whole dict (thousands of elements),
+        with self.lock:
-        #       is really inefficient.
+            ret = [self.get_last_good_address(node_id)
-        r = [self.get_last_good_address(node_id)
+                   for node_id in self._recent_peers]
-             for node_id in list(self._addresses.keys())[-self.NUM_MAX_RECENT_PEERS:]]
+            return ret
-        return list(reversed(r))
 
     # note: currently channel announcements are trusted by default (trusted=True);
-    #       they are not verified. Verifying them would make the gossip sync
+     #       they are not SPV-verified. Verifying them would make the gossip sync
     #       even slower; especially as servers will start throttling us.
     #       It would probably put significant strain on servers if all clients
     #       verified the complete gossip.
@@ -313,8 +339,8 @@ class ChannelDB(SqlDB):
                 continue
             try:
                 channel_info = ChannelInfo.from_msg(msg)
-            except UnknownEvenFeatureBits:
+            except IncompatibleOrInsaneFeatures as e:
-                self.logger.info("unknown feature bits")
+                self.logger.info(f"unknown or insane feature bits: {e!r}")
                 continue
             if trusted:
                 added += 1
@@ -328,85 +354,112 @@ class ChannelDB(SqlDB):
     def add_verified_channel_info(self, msg: dict, *, capacity_sat: int = None) -> None:
         try:
             channel_info = ChannelInfo.from_msg(msg)
-        except UnknownEvenFeatureBits:
+        except IncompatibleOrInsaneFeatures:
             return
-        channel_info = channel_info._replace(capacity_sat=capacity_sat)
+            with self.lock:
                 self._channels[channel_info.short_channel_id] = channel_info
                 self._channels_for_node[channel_info.node1_id].add(channel_info.short_channel_id)
                 self._channels_for_node[channel_info.node2_id].add(channel_info.short_channel_id)
-        self.save_channel(channel_info)
+        self._update_num_policies_for_chan(channel_info.short_channel_id)
+        if 'raw' in msg:
+            self._db_save_channel(channel_info.short_channel_id, msg['raw'])
 
-    def print_change(self, old_policy: Policy, new_policy: Policy):
+    def policy_changed(self, old_policy: Policy, new_policy: Policy, verbose: bool) -> bool:
-        # print what changed between policies
+        changed = False
         if old_policy.cltv_expiry_delta != new_policy.cltv_expiry_delta:
+            changed |= True
+            if verbose:
                 self.logger.info(f'cltv_expiry_delta: {old_policy.cltv_expiry_delta} -> {new_policy.cltv_expiry_delta}')
         if old_policy.htlc_minimum_msat != new_policy.htlc_minimum_msat:
+            changed |= True
+            if verbose:
                 self.logger.info(f'htlc_minimum_msat: {old_policy.htlc_minimum_msat} -> {new_policy.htlc_minimum_msat}')
         if old_policy.htlc_maximum_msat != new_policy.htlc_maximum_msat:
+            changed |= True
+            if verbose:
                 self.logger.info(f'htlc_maximum_msat: {old_policy.htlc_maximum_msat} -> {new_policy.htlc_maximum_msat}')
         if old_policy.fee_base_msat != new_policy.fee_base_msat:
+            changed |= True
+            if verbose:
                 self.logger.info(f'fee_base_msat: {old_policy.fee_base_msat} -> {new_policy.fee_base_msat}')
         if old_policy.fee_proportional_millionths != new_policy.fee_proportional_millionths:
+            changed |= True
+            if verbose:
                 self.logger.info(f'fee_proportional_millionths: {old_policy.fee_proportional_millionths} -> {new_policy.fee_proportional_millionths}')
         if old_policy.channel_flags != new_policy.channel_flags:
+            changed |= True
+            if verbose:
                 self.logger.info(f'channel_flags: {old_policy.channel_flags} -> {new_policy.channel_flags}')
         if old_policy.message_flags != new_policy.message_flags:
+            changed |= True
+            if verbose:
                 self.logger.info(f'message_flags: {old_policy.message_flags} -> {new_policy.message_flags}')
+        if not changed and verbose:
+            self.logger.info(f'policy unchanged: {old_policy.timestamp} -> {new_policy.timestamp}')
+        return changed
 
-    def add_channel_updates(self, payloads, max_age=None, verify=True) -> CategorizedChannelUpdates:
+    def add_channel_update(self, payload, max_age=None, verify=False, verbose=True):
-        orphaned = []
-        expired = []
-        deprecated = []
-        good = []
-        to_delete = []
-        # filter orphaned and expired first
-        known = []
         now = int(time.time())
-        for payload in payloads:
         short_channel_id = ShortChannelID(payload['short_channel_id'])
-            timestamp = int.from_bytes(payload['timestamp'], "big")
+        timestamp = payload['timestamp']
         if max_age and now - timestamp > max_age:
-                expired.append(payload)
+            return UpdateStatus.EXPIRED
-                continue
+        if timestamp - now > 60:
+            return UpdateStatus.DEPRECATED
         channel_info = self._channels.get(short_channel_id)
         if not channel_info:
-                orphaned.append(payload)
+            return UpdateStatus.ORPHANED
-                continue
         flags = int.from_bytes(payload['channel_flags'], 'big')
         direction = flags & FLAG_DIRECTION
         start_node = channel_info.node1_id if direction == 0 else channel_info.node2_id
         payload['start_node'] = start_node
-            known.append(payload)
         # compare updates to existing database entries
-        for payload in known:
+        timestamp = payload['timestamp']
-            timestamp = int.from_bytes(payload['timestamp'], "big")
         start_node = payload['start_node']
         short_channel_id = ShortChannelID(payload['short_channel_id'])
         key = (start_node, short_channel_id)
         old_policy = self._policies.get(key)
-            if old_policy and timestamp <= old_policy.timestamp:
+        if old_policy and timestamp <= old_policy.timestamp + 60:
-                deprecated.append(payload)
+            return UpdateStatus.DEPRECATED
-                continue
-            good.append(payload)
         if verify:
             self.verify_channel_update(payload)
         policy = Policy.from_msg(payload)
+        with self.lock:
             self._policies[key] = policy
-            self.save_policy(policy)
+        self._update_num_policies_for_chan(short_channel_id)
-        #
+        if 'raw' in payload:
+            self._db_save_policy(policy.key, payload['raw'])
+        if old_policy and not self.policy_changed(old_policy, policy, verbose):
+            return UpdateStatus.UNCHANGED
+        else:
+            return UpdateStatus.GOOD
+
+    def add_channel_updates(self, payloads, max_age=None) -> CategorizedChannelUpdates:
+        orphaned = []
+        expired = []
+        deprecated = []
+        unchanged = []
+        good = []
+        for payload in payloads:
+            r = self.add_channel_update(payload, max_age=max_age, verbose=False)
+            if r == UpdateStatus.ORPHANED:
+                orphaned.append(payload)
+                continue
+            elif r == UpdateStatus.EXPIRED:
+                expired.append(payload)
+            elif r == UpdateStatus.DEPRECATED:
+                deprecated.append(payload)
+            elif r == UpdateStatus.UNCHANGED:
+                unchanged.append(payload)
+            elif r == UpdateStatus.GOOD:
+                good.append(payload)
         self.update_counts()
         return CategorizedChannelUpdates(
             orphaned=orphaned,
             expired=expired,
             deprecated=deprecated,
-            good=good,
+            unchanged=unchanged,
-            to_delete=to_delete,
+            good=good)
-        )
-
-    def add_channel_update(self, payload):
-        # called from add_own_channel
-        # the update may be categorized as deprecated because of caching
-        categorized_chan_upds = self.add_channel_updates([payload], verify=False)
 
     def create_database(self):
         c = self.conn.cursor()
@@ -417,44 +470,48 @@ class ChannelDB(SqlDB):
         self.conn.commit()
 
     @sql
-    def save_policy(self, policy):
+    def _db_save_policy(self, key: bytes, msg: bytes):
+        # 'msg' is a 'channel_update' message
         c = self.conn.cursor()
-        c.execute("""REPLACE INTO policy (key, cltv_expiry_delta, htlc_minimum_msat, htlc_maximum_msat, fee_base_msat, fee_proportional_millionths, channel_flags, message_flags, timestamp) VALUES (?,?,?,?,?,?,?,?,?)""", list(policy))
+        c.execute("""REPLACE INTO policy (key, msg) VALUES (?,?)""", [key, msg])
 
     @sql
-    def delete_policy(self, node_id, short_channel_id):
+    def _db_delete_policy(self, node_id: bytes, short_channel_id: ShortChannelID):
         key = short_channel_id + node_id
         c = self.conn.cursor()
         c.execute("""DELETE FROM policy WHERE key=?""", (key,))
 
     @sql
-    def save_channel(self, channel_info):
+    def _db_save_channel(self, short_channel_id: ShortChannelID, msg: bytes):
+    # 'msg' is a 'channel_announcement' message
         c = self.conn.cursor()
-        c.execute("REPLACE INTO channel_info (short_channel_id, node1_id, node2_id, capacity_sat) VALUES (?,?,?,?)", list(channel_info))
+        c.execute("REPLACE INTO channel_info (short_channel_id, msg) VALUES (?,?)", [short_channel_id, msg])
 
     @sql
-    def delete_channel(self, short_channel_id):
+    def _db_delete_channel(self, short_channel_id: ShortChannelID):
         c = self.conn.cursor()
         c.execute("""DELETE FROM channel_info WHERE short_channel_id=?""", (short_channel_id,))
 
     @sql
-    def save_node(self, node_info):
+    def _db_save_node_info(self, node_id: bytes, msg: bytes):
+    # 'msg' is a 'node_announcement' message
         c = self.conn.cursor()
-        c.execute("REPLACE INTO node_info (node_id, features, timestamp, alias) VALUES (?,?,?,?)", list(node_info))
+        c.execute("REPLACE INTO node_info (node_id, msg) VALUES (?,?)", [node_id, msg])
 
     @sql
-    def save_node_address(self, node_id, peer, now):
+    def _db_save_node_address(self, peer: LNPeerAddr, timestamp: int):
         c = self.conn.cursor()
-        c.execute("REPLACE INTO address (node_id, host, port, timestamp) VALUES (?,?,?,?)", (node_id, peer.host, peer.port, now))
+        c.execute("REPLACE INTO address (node_id, host, port, timestamp) VALUES (?,?,?,?)",
+                  (peer.pubkey, peer.host, peer.port, timestamp))
 
     @sql
-    def save_node_addresses(self, node_id, node_addresses):
+    def _db_save_node_addresses(self, node_addresses: Sequence[LNPeerAddr]):
         c = self.conn.cursor()
         for addr in node_addresses:
-            c.execute("SELECT * FROM address WHERE node_id=? AND host=? AND port=?", (addr.node_id, addr.host, addr.port))
+            c.execute("SELECT * FROM address WHERE node_id=? AND host=? AND port=?", (addr.pubkey, addr.host, addr.port))
             r = c.fetchall()
             if r == []:
-                c.execute("INSERT INTO address (node_id, host, port, timestamp) VALUES (?,?,?,?)", (addr.node_id, addr.host, addr.port, 0))
+                c.execute("INSERT INTO address (node_id, host, port, timestamp) VALUES (?,?,?,?)", (addr.pubkey, addr.host, addr.port, 0))
 
     def verify_channel_update(self, payload):
         short_channel_id = payload['short_channel_id']
@@ -463,16 +520,15 @@ class ChannelDB(SqlDB):
             raise Exception('wrong chain hash')
         if not verify_sig_for_channel_update(payload, payload['start_node']):
             raise Exception(f'failed verifying channel update for {short_channel_id}')
-
+# note: signatures have already been verified.
     def add_node_announcement(self, msg_payloads):
         if type(msg_payloads) is dict:
             msg_payloads = [msg_payloads]
-        old_addr = None
         new_nodes = {}
         for msg_payload in msg_payloads:
             try:
                 node_info, node_addresses = NodeInfo.from_msg(msg_payload)
-            except UnknownEvenFeatureBits:
+            except IncompatibleOrInsaneFeatures:
                 continue
             node_id = node_info.node_id
             # Ignore node if it has no associated channel (DoS protection)
@@ -486,50 +542,44 @@ class ChannelDB(SqlDB):
             if node and node.timestamp >= node_info.timestamp:
                 continue
             # save
+            with self.lock:
                 self._nodes[node_id] = node_info
-            self.save_node(node_info)
+            if 'raw' in msg_payload:
+                self._db_save_node_info(node_id, msg_payload['raw'])
+            with self.lock:
                 for addr in node_addresses:
                     self._addresses[node_id].add((addr.host, addr.port, 0))
-            self.save_node_addresses(node_id, node_addresses)
+            self._db_save_node_addresses(node_addresses)
 
         self.logger.debug("on_node_announcement: %d/%d"%(len(new_nodes), len(msg_payloads)))
         self.update_counts()
 
-    def get_routing_policy_for_channel(self, start_node_id: bytes,
+    def get_old_policies(self, delta) -> Sequence[Tuple[bytes, ShortChannelID]]:
-                                       short_channel_id: bytes) -> Optional[Policy]:
+        with self.lock:
-        if not start_node_id or not short_channel_id: return None
+            _policies = self._policies.copy()
-        channel_info = self.get_channel_info(short_channel_id)
-        if channel_info is not None:
-            return self.get_policy_for_node(short_channel_id, start_node_id)
-        msg = self._channel_updates_for_private_channels.get((start_node_id, short_channel_id))
-        if not msg:
-            return None
-        return Policy.from_msg(msg) # won't actually be written to DB
-
-    def get_old_policies(self, delta):
         now = int(time.time())
-        return list(k for k, v in list(self._policies.items()) if v.timestamp <= now - delta)
+        return list(k for k, v in _policies.items() if v.timestamp <= now - delta)
 
     def prune_old_policies(self, delta):
-        l = self.get_old_policies(delta)
+        old_policies = self.get_old_policies(delta)
-        if l:
+        if old_policies:
-            for k in l:
+            for key in old_policies:
-                self._policies.pop(k)
+                node_id, scid = key
-                self.delete_policy(*k)
+                with self.lock:
+                    self._policies.pop(key)
+                self._db_delete_policy(*key)
+                self._update_num_policies_for_chan(scid)
             self.update_counts()
-            self.logger.info(f'Deleting {len(l)} old policies')
+            self.logger.info(f'Deleting {len(old_policies)} old policies')
-
-    def get_orphaned_channels(self):
-        ids = set(x[1] for x in self._policies.keys())
-        return list(x for x in self._channels.keys() if x not in ids)
 
     def prune_orphaned_channels(self):
-        l = self.get_orphaned_channels()
+        with self.lock:
-        if l:
+            orphaned_chans = self._chans_with_0_policies.copy()
-            for short_channel_id in l:
+        if orphaned_chans:
+            for short_channel_id in orphaned_chans:
                 self.remove_channel(short_channel_id)
             self.update_counts()
-            self.logger.info(f'Deleting {len(l)} orphaned channels')
+            self.logger.info(f'Deleting {len(orphaned_chans)} orphaned channels')
 
     def add_channel_update_for_private_channel(self, msg_payload: dict, start_node_id: bytes):
         if not verify_sig_for_channel_update(msg_payload, start_node_id):
@@ -539,12 +589,15 @@ class ChannelDB(SqlDB):
         self._channel_updates_for_private_channels[(start_node_id, short_channel_id)] = msg_payload
 
     def remove_channel(self, short_channel_id: ShortChannelID):
+        # FIXME what about rm-ing policies?
+        with self.lock:
             channel_info = self._channels.pop(short_channel_id, None)
             if channel_info:
                 self._channels_for_node[channel_info.node1_id].remove(channel_info.short_channel_id)
                 self._channels_for_node[channel_info.node2_id].remove(channel_info.short_channel_id)
+        self._update_num_policies_for_chan(short_channel_id)
         # delete from database
-        self.delete_channel(short_channel_id)
+        self._db_delete_channel(short_channel_id)
 
     def get_node_addresses(self, node_id):
         return self._addresses.get(node_id)
@@ -557,42 +610,139 @@ class ChannelDB(SqlDB):
         for x in c:
             node_id, host, port, timestamp = x
             self._addresses[node_id].add((str(host), int(port), int(timestamp or 0)))
+    def newest_ts_for_node_id(node_id):
+        newest_ts = 0
+        for host, port, ts in self._addresses[node_id]:
+            newest_ts = max(newest_ts, ts)
+        return newest_ts
+    sorted_node_ids = sorted(self._addresses.keys(), key=newest_ts_for_node_id, reverse=True)
+    self._recent_peers = sorted_node_ids[:self.NUM_MAX_RECENT_PEERS]
         c.execute("""SELECT * FROM channel_info""")
-        for x in c:
+        for short_channel_id, msg in c:
-            x = (ShortChannelID.normalize(x[0]), *x[1:])
+            try:
-            ci = ChannelInfo(*x)
+                ci = ChannelInfo.from_raw_msg(msg)
-            self._channels[ci.short_channel_id] = ci
+            except IncompatibleOrInsaneFeatures:
+                continue
+            self._channels[ShortChannelID.normalize(short_channel_id)] = ci
         c.execute("""SELECT * FROM node_info""")
-        for x in c:
+        for node_id, msg in c:
-            ni = NodeInfo(*x)
+            try:
-            self._nodes[ni.node_id] = ni
+                node_info, node_addresses = NodeInfo.from_raw_msg(msg)
+            except IncompatibleOrInsaneFeatures:
+                continue
+            # don't load node_addresses because they dont have timestamps
+            self._nodes[node_id] = node_info
         c.execute("""SELECT * FROM policy""")
-        for x in c:
+        for key, msg in c:
-            p = Policy(*x)
+            p = Policy.from_raw_msg(key, msg)
             self._policies[(p.start_node, p.short_channel_id)] = p
         for channel_info in self._channels.values():
             self._channels_for_node[channel_info.node1_id].add(channel_info.short_channel_id)
             self._channels_for_node[channel_info.node2_id].add(channel_info.short_channel_id)
+            self._update_num_policies_for_chan(channel_info.short_channel_id)
         self.logger.info(f'load data {len(self._channels)} {len(self._policies)} {len(self._channels_for_node)}')
         self.update_counts()
-        self.count_incomplete_channels()
+        (nchans_with_0p, nchans_with_1p, nchans_with_2p) = self.get_num_channels_partitioned_by_policy_count()
+        self.logger.info(f'num_channels_partitioned_by_policy_count. '
+                         f'0p: {nchans_with_0p}, 1p: {nchans_with_1p}, 2p: {nchans_with_2p}')
         self.data_loaded.set()
+        util.trigger_callback('gossip_db_loaded')
 
-    def count_incomplete_channels(self):
+    def _update_num_policies_for_chan(self, short_channel_id: ShortChannelID) -> None:
-        out = set()
+        channel_info = self.get_channel_info(short_channel_id)
-        for short_channel_id, ci in self._channels.items():
+        if channel_info is None:
-            p1 = self.get_policy_for_node(short_channel_id, ci.node1_id)
+            with self.lock:
-            p2 = self.get_policy_for_node(short_channel_id, ci.node2_id)
+                self._chans_with_0_policies.discard(short_channel_id)
-            if p1 is None or p2 is not None:
+                self._chans_with_1_policies.discard(short_channel_id)
-                out.add(short_channel_id)
+                self._chans_with_2_policies.discard(short_channel_id)
-        self.logger.info(f'semi-orphaned: {len(out)}')
+            return
+        p1 = self.get_policy_for_node(short_channel_id, channel_info.node1_id)
+        p2 = self.get_policy_for_node(short_channel_id, channel_info.node2_id)
+        with self.lock:
+            self._chans_with_0_policies.discard(short_channel_id)
+            self._chans_with_1_policies.discard(short_channel_id)
+            self._chans_with_2_policies.discard(short_channel_id)
+            if p1 is not None and p2 is not None:
+                self._chans_with_2_policies.add(short_channel_id)
+            elif p1 is None and p2 is None:
+                self._chans_with_0_policies.add(short_channel_id)
+            else:
+                self._chans_with_1_policies.add(short_channel_id)
 
-    def get_policy_for_node(self, short_channel_id: bytes, node_id: bytes) -> Optional['Policy']:
+    def get_num_channels_partitioned_by_policy_count(self) -> Tuple[int, int, int]:
-        return self._policies.get((node_id, short_channel_id))
+        nchans_with_0p = len(self._chans_with_0_policies)
+        nchans_with_1p = len(self._chans_with_1_policies)
+        nchans_with_2p = len(self._chans_with_2_policies)
+        return nchans_with_0p, nchans_with_1p, nchans_with_2p
 
-    def get_channel_info(self, channel_id: bytes) -> ChannelInfo:
+        def get_policy_for_node(self, short_channel_id: bytes, node_id: bytes, *,
-        return self._channels.get(channel_id)
+                            my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Optional['Policy']:
+        channel_info = self.get_channel_info(short_channel_id)
+        if channel_info is not None:  # publicly announced channel
+            policy = self._policies.get((node_id, short_channel_id))
+            if policy:
+                return policy
+        else:  # private channel
+            chan_upd_dict = self._channel_updates_for_private_channels.get((node_id, short_channel_id))
+            if chan_upd_dict:
+                return Policy.from_msg(chan_upd_dict)
+        # check if it's one of our own channels
+        if not my_channels:
+            return
+        chan = my_channels.get(short_channel_id)  # type: Optional[Channel]
+        if not chan:
+            return
+        if node_id == chan.node_id:  # incoming direction (to us)
+            remote_update_raw = chan.get_remote_update()
+            if not remote_update_raw:
+                return
+            now = int(time.time())
+            remote_update_decoded = decode_msg(remote_update_raw)[1]
+            remote_update_decoded['timestamp'] = now
+            remote_update_decoded['start_node'] = node_id
+            return Policy.from_msg(remote_update_decoded)
+        elif node_id == chan.get_local_pubkey():  # outgoing direction (from us)
+            local_update_decoded = decode_msg(chan.get_outgoing_gossip_channel_update())[1]
+            local_update_decoded['start_node'] = node_id
+            return Policy.from_msg(local_update_decoded)
 
-    def get_channels_for_node(self, node_id) -> Set[bytes]:
+    def get_channel_info(self, short_channel_id: ShortChannelID, *,
-        """Returns the set of channels that have node_id as one of the endpoints."""
+                         my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Optional[ChannelInfo]:
-        return self._channels_for_node.get(node_id) or set()
+        ret = self._channels.get(short_channel_id)
+        if ret:
+            return ret
+        # check if it's one of our own channels
+        if not my_channels:
+            return
+        chan = my_channels.get(short_channel_id)  # type: Optional[Channel]
+        ci = ChannelInfo.from_raw_msg(chan.construct_channel_announcement_without_sigs())
+        return ci._replace(capacity_sat=chan.constraints.capacity)
+
+    def get_channels_for_node(self, node_id: bytes, *,
+                              my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Set[bytes]:
+        """Returns the set of short channel IDs where node_id is one of the channel participants."""
+        if not self.data_loaded.is_set():
+            raise Exception("channelDB data not loaded yet!")
+        relevant_channels = self._channels_for_node.get(node_id) or set()
+        relevant_channels = set(relevant_channels)  # copy
+        # add our own channels  # TODO maybe slow?
+        for chan in (my_channels.values() or []):
+            if node_id in (chan.node_id, chan.get_local_pubkey()):
+                relevant_channels.add(chan.short_channel_id)
+        return relevant_channels
+
+    def get_endnodes_for_chan(self, short_channel_id: ShortChannelID, *,
+                              my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Optional[Tuple[bytes, bytes]]:
+        channel_info = self.get_channel_info(short_channel_id)
+        if channel_info is not None:  # publicly announced channel
+            return channel_info.node1_id, channel_info.node2_id
+        # check if it's one of our own channels
+        if not my_channels:
+            return
+        chan = my_channels.get(short_channel_id)  # type: Optional[Channel]
+        if not chan:
+            return
+        return chan.get_local_pubkey(), chan.node_id
+
+    def get_node_info_for_node_id(self, node_id: bytes) -> Optional['NodeInfo']:
+        return self._nodes.get(node_id)

electrum/coinchooser.py

@@ -49,7 +49,7 @@ class PRNG:
             self.pool.extend(self.sha)
             self.sha = sha256(self.sha)
         result, self.pool = self.pool[:n], self.pool[n:]
-        return result
+        return bytes(result)
 
     def randint(self, start, end):
         # Returns random integer in [start, end)
@@ -103,10 +103,9 @@ def strip_unneeded(bkts: List[Bucket], sufficient_funds) -> List[Bucket]:
 
 class CoinChooserBase(Logger):
 
-    enable_output_value_rounding = False
+    def __init__(self, *, enable_output_value_rounding: bool):
-
-    def __init__(self):
         Logger.__init__(self)
+        self.enable_output_value_rounding = enable_output_value_rounding
 
     def keys(self, coins: Sequence[PartialTxInput]) -> Sequence[str]:
         raise NotImplementedError
@@ -485,6 +484,12 @@ def get_name(config):
 
 def get_coin_chooser(config):
     klass = COIN_CHOOSERS[get_name(config)]
-    coinchooser = klass()
+        # note: we enable enable_output_value_rounding by default as
-    coinchooser.enable_output_value_rounding = config.get('coin_chooser_output_rounding', False)
+    #       - for sacrificing a few satoshis
+    #       + it gives better privacy for the user re change output
+    #       + it also helps the network as a whole as fees will become noisier
+    #         (trying to counter the heuristic that "whole integer sat/byte feerates" are common)
+    coinchooser = klass(
+        enable_output_value_rounding=config.get('coin_chooser_output_rounding', True),
+    )
     return coinchooser

electrum/commands.py

@@ -47,17 +47,20 @@ from .bip32 import BIP32Node
 from .i18n import _
 from .transaction import (Transaction, multisig_script, TxOutput, PartialTransaction, PartialTxOutput,
                           tx_from_any, PartialTxInput, TxOutpoint)
-from .paymentrequest import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED
+from .invoices import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED
 from .synchronizer import Notifier
-from .wallet import Abstract_Wallet, create_new_wallet, restore_wallet_from_text
+from .wallet import Abstract_Wallet, create_new_wallet, restore_wallet_from_text, Deterministic_Wallet
 from .address_synchronizer import TX_HEIGHT_LOCAL
 from .mnemonic import Mnemonic
 from .lnutil import SENT, RECEIVED
+from .lnutil import LnFeatures
 from .lnutil import ln_dummy_address
 from .lnpeer import channel_id_from_funding_tx
 from .plugin import run_hook
 from .version import ELECTRUM_VERSION
 from .simple_config import SimpleConfig
+from .invoices import LNInvoice
+from . import submarine_swaps
 
 
 if TYPE_CHECKING:
@@ -68,10 +71,16 @@ if TYPE_CHECKING:
 known_commands = {}  # type: Dict[str, Command]
 
 
+class NotSynchronizedException(Exception):
+    pass
+
+
 def satoshis(amount):
     # satoshi conversion must not be performed by the parser
     return int(COIN*Decimal(amount)) if amount not in ['!', None] else amount
 
+def format_satoshis(x):
+    return str(Decimal(x)/COIN) if x is not None else None
 
 def json_normalize(x):
     # note: The return value of commands, when going through the JSON-RPC interface,
@@ -100,6 +109,15 @@ class Command:
             self.options = []
             self.defaults = []
 
+# sanity checks
+        if self.requires_password:
+            assert self.requires_wallet
+        for varname in ('wallet_path', 'wallet'):
+            if varname in varnames:
+                assert varname in self.options
+        assert not ('wallet_path' in varnames and 'wallet' in varnames)
+        if self.requires_wallet:
+            assert 'wallet' in varnames
 
 def command(s):
     def decorator(func):
@@ -113,18 +131,20 @@ def command(s):
             password = kwargs.get('password')
             daemon = cmd_runner.daemon
             if daemon:
-                if (cmd.requires_wallet or 'wallet_path' in cmd.options) and kwargs.get('wallet_path') is None:
+                if 'wallet_path' in cmd.options and kwargs.get('wallet_path') is None:
-                    # using JSON-RPC, sometimes the "wallet" kwarg needs to be used to specify a wallet
+                    kwargs['wallet_path'] = daemon.config.get_wallet_path()
-                    kwargs['wallet_path'] = kwargs.pop('wallet', None) or daemon.config.get_wallet_path()
+                if cmd.requires_wallet and kwargs.get('wallet') is None:
-                if cmd.requires_wallet:
+                    kwargs['wallet'] = daemon.config.get_wallet_path()
-                    wallet_path = kwargs.pop('wallet_path')
+                if 'wallet' in cmd.options:
+                    wallet_path = kwargs.get('wallet', None)
+                    if isinstance(wallet_path, str):
                         wallet = daemon.get_wallet(wallet_path)
                         if wallet is None:
                             raise Exception('wallet not loaded')
                         kwargs['wallet'] = wallet
-            else:
+            wallet = kwargs.get('wallet')  # type: Optional[Abstract_Wallet]
-                # we are offline. the wallet must have been passed if required
+            if cmd.requires_wallet and not wallet:
-                wallet = kwargs.get('wallet')
+                raise Exception('wallet not loaded')
             if cmd.requires_password and password is None and wallet.has_password():
                 raise Exception('Password required')
             return await func(*args, **kwargs)
@@ -181,7 +201,7 @@ class Commands:
         net_params = self.network.get_parameters()
         response = {
             'path': self.network.config.path,
-            'server': net_params.host,
+            'server': net_params.server.host,
             'blockchain_height': self.network.get_local_height(),
             'server_height': self.network.get_server_height(),
             'spv_nodes': len(self.network.get_interfaces()),
@@ -292,6 +312,12 @@ class Commands:
         self.config.set_key(key, value)
         return True
 
+    @command('')
+    async def get_ssl_domain(self):
+        """Check and return the SSL domain set in ssl_keyfile and ssl_certfile
+        """
+        return self.config.get_ssl_domain()
+
     @command('')
     async def make_seed(self, nbits=132, language=None, seed_type=None):
         """Create a seed"""
@@ -345,6 +371,9 @@ class Commands:
                 raise Exception("missing prevout for txin")
             txin = PartialTxInput(prevout=prevout)
             txin._trusted_value_sats = int(txin_dict['value'])
+            nsequence = txin_dict.get('nsequence', None)
+            if nsequence is not None:
+                txin.nsequence = nsequence
             sec = txin_dict.get('privkey')
             if sec:
                 txin_type, privkey, compressed = bitcoin.deserialize_privkey(sec)
@@ -414,6 +443,13 @@ class Commands:
         domain = address
         return [wallet.export_private_key(address, password) for address in domain]
 
+    @command('wp')
+    async def getprivatekeyforpath(self, path, password=None, wallet: Abstract_Wallet = None):
+        """Get private key corresponding to derivation path (address index).
+        'path' can be either a str such as "m/0/50", or a list of ints such as [0, 50].
+        """
+        return wallet.export_private_key_for_path(path, password)
+
     @command('w')
     async def ismine(self, address, wallet: Abstract_Wallet = None):
         """Check if address is in wallet. Return true if and only address is in wallet"""
@@ -467,7 +503,7 @@ class Commands:
 
     @command('n')
     async def getservers(self):
-        """Return the list of available servers"""
+        """Return the list of known servers (candidates for connecting)."""
         return self.network.get_servers()
 
     @command('')
@@ -553,56 +589,25 @@ class Commands:
         message = util.to_bytes(message)
         return ecc.verify_message_with_address(address, sig, message)
 
-    def _mktx(self, wallet: Abstract_Wallet, outputs, *, fee=None, feerate=None, change_addr=None, domain_addr=None, domain_coins=None,
-              nocheck=False, unsigned=False, rbf=None, password=None, locktime=None):
-        if fee is not None and feerate is not None:
-            raise Exception("Cannot specify both 'fee' and 'feerate' at the same time!")
-        self.nocheck = nocheck
-        change_addr = self._resolver(change_addr, wallet)
-        domain_addr = None if domain_addr is None else map(self._resolver, domain_addr, repeat(wallet))
-        final_outputs = []
-        for address, amount in outputs:
-            address = self._resolver(address, wallet)
-            amount = satoshis(amount)
-            final_outputs.append(PartialTxOutput.from_address_and_value(address, amount))
-
-        coins = wallet.get_spendable_coins(domain_addr)
-        if domain_coins is not None:
-            coins = [coin for coin in coins if (coin.prevout.to_str() in domain_coins)]
-        if feerate is not None:
-            fee_per_kb = 1000 * Decimal(feerate)
-            fee_estimator = partial(SimpleConfig.estimate_fee_for_feerate, fee_per_kb)
-        else:
-            fee_estimator = fee
-        tx = wallet.make_unsigned_transaction(coins=coins,
-                                              outputs=final_outputs,
-                                              fee=fee_estimator,
-                                              change_addr=change_addr)
-        if locktime is not None:
-            tx.locktime = locktime
-        if rbf is None:
-            rbf = self.config.get('use_rbf', True)
-        if rbf:
-            tx.set_rbf(True)
-        if not unsigned:
-            wallet.sign_transaction(tx, password)
-        return tx
-
     @command('wp')
     async def payto(self, destination, amount, fee=None, feerate=None, from_addr=None, from_coins=None, change_addr=None,
                     nocheck=False, unsigned=False, rbf=None, password=None, locktime=None, wallet: Abstract_Wallet = None):
         """Create a transaction. """
+        self.nocheck = nocheck
         tx_fee = satoshis(fee)
         domain_addr = from_addr.split(',') if from_addr else None
         domain_coins = from_coins.split(',') if from_coins else None
-        tx = self._mktx(wallet,
+        change_addr = self._resolver(change_addr, wallet)
-                        [(destination, amount)],
+        domain_addr = None if domain_addr is None else map(self._resolver, domain_addr, repeat(wallet))
+        amount_sat = satoshis(amount)
+        outputs = [PartialTxOutput.from_address_and_value(destination, amount_sat)]
+        tx = wallet.create_transaction(
+            outputs,
             fee=tx_fee,
             feerate=feerate,
             change_addr=change_addr,
             domain_addr=domain_addr,
             domain_coins=domain_coins,
-                        nocheck=nocheck,
             unsigned=unsigned,
             rbf=rbf,
             password=password,
@@ -613,17 +618,24 @@ class Commands:
     async def paytomany(self, outputs, fee=None, feerate=None, from_addr=None, from_coins=None, change_addr=None,
                         nocheck=False, unsigned=False, rbf=None, password=None, locktime=None, wallet: Abstract_Wallet = None):
         """Create a multi-output transaction. """
+        self.nocheck = nocheck
         tx_fee = satoshis(fee)
         domain_addr = from_addr.split(',') if from_addr else None
         domain_coins = from_coins.split(',') if from_coins else None
-        tx = self._mktx(wallet,
+        change_addr = self._resolver(change_addr, wallet)
-                        outputs,
+        domain_addr = None if domain_addr is None else map(self._resolver, domain_addr, repeat(wallet))
+        final_outputs = []
+        for address, amount in outputs:
+            address = self._resolver(address, wallet)
+            amount_sat = satoshis(amount)
+            final_outputs.append(PartialTxOutput.from_address_and_value(address, amount_sat))
+        tx = wallet.create_transaction(
+            final_outputs,
             fee=tx_fee,
             feerate=feerate,
             change_addr=change_addr,
             domain_addr=domain_addr,
             domain_coins=domain_coins,
-                        nocheck=nocheck,
             unsigned=unsigned,
             rbf=rbf,
             password=password,
@@ -754,19 +766,13 @@ class Commands:
         decrypted = wallet.decrypt_message(pubkey, encrypted, password)
         return decrypted.decode('utf-8')
 
-    def _format_request(self, out):
-        from .util import get_request_status
-        out['amount_BTC'] = format_satoshis(out.get('amount'))
-        out['status_str'] = get_request_status(out)
-        return out
-
     @command('w')
     async def getrequest(self, key, wallet: Abstract_Wallet = None):
         """Return a payment request"""
         r = wallet.get_request(key)
         if not r:
             raise Exception("Request not found")
-        return self._format_request(r)
+        return wallet.export_request(r)
 
     #@command('w')
     #async def ackrequest(self, serialized):
@@ -776,7 +782,7 @@ class Commands:
     @command('w')
     async def list_requests(self, pending=False, expired=False, paid=False, wallet: Abstract_Wallet = None):
         """List the payment requests you made."""
-        out = wallet.get_sorted_requests()
+
         if pending:
             f = PR_UNPAID
         elif expired:
@@ -785,9 +791,34 @@ class Commands:
             f = PR_PAID
         else:
             f = None
+        out = wallet.get_sorted_requests()
         if f is not None:
-            out = list(filter(lambda x: x.get('status')==f, out))
+            out = list(filter(lambda x: x.status==f, out))
-        return list(map(self._format_request, out))
+        return [wallet.export_request(x) for x in out]
+
+    @command('w')
+    async def changegaplimit(self, new_limit, iknowwhatimdoing=False, wallet: Abstract_Wallet = None):
+        """Change the gap limit of the wallet."""
+        if not iknowwhatimdoing:
+            raise Exception("WARNING: Are you SURE you want to change the gap limit?\n"
+                            "It makes recovering your wallet from seed difficult!\n"
+                            "Please do your research and make sure you understand the implications.\n"
+                            "Typically only merchants and power users might want to do this.\n"
+                            "To proceed, try again, with the --iknowwhatimdoing option.")
+        if not isinstance(wallet, Deterministic_Wallet):
+            raise Exception("This wallet is not deterministic.")
+        return wallet.change_gap_limit(new_limit)
+
+    @command('wn')
+    async def getminacceptablegap(self, wallet: Abstract_Wallet = None):
+        """Returns the minimum value for gap limit that would be sufficient to discover all
+        known addresses in the wallet.
+        """
+        if not isinstance(wallet, Deterministic_Wallet):
+            raise Exception("This wallet is not deterministic.")
+        if not wallet.is_up_to_date():
+            raise NotSynchronizedException("Wallet not fully synchronized.")
+        return wallet.min_acceptable_gap()
 
     @command('w')
     async def createnewaddress(self, wallet: Abstract_Wallet = None):
@@ -815,14 +846,13 @@ class Commands:
         expiration = int(expiration) if expiration else None
         req = wallet.make_payment_request(addr, amount, memo, expiration)
         wallet.add_payment_request(req)
-        out = wallet.get_request(addr)
+        return wallet.export_request(req)
-        return self._format_request(out)
 
     @command('wn')
     async def add_lightning_request(self, amount, memo='', expiration=3600, wallet: Abstract_Wallet = None):
         amount_sat = int(satoshis(amount))
         key = await wallet.lnworker._add_request_coro(amount_sat, memo, expiration)
-        return wallet.get_request(key)['invoice']
+        return wallet.get_formatted_request(key)
 
     @command('w')
     async def addtransaction(self, tx, wallet: Abstract_Wallet = None):
@@ -850,8 +880,8 @@ class Commands:
     @command('w')
     async def clear_requests(self, wallet: Abstract_Wallet = None):
         """Remove all payment requests"""
-        for k in list(wallet.receive_requests.keys()):
+        wallet.clear_requests()
-            wallet.remove_payment_request(k)
+        return True
 
     @command('w')
     async def clear_invoices(self, wallet: Abstract_Wallet = None):
@@ -860,11 +890,16 @@ class Commands:
         return True
 
     @command('n')
-    async def notify(self, address: str, URL: str):
+    async def notify(self, address: str, URL: Optional[str]):
-        """Watch an address. Every time the address changes, a http POST is sent to the URL."""
+        """Watch an address. Every time the address changes, a http POST is sent to the URL.
+        Call with an empty URL to stop watching an address.
+        """
         if not hasattr(self, "_notifier"):
             self._notifier = Notifier(self.network)
-        await self._notifier.start_watching_queue.put((address, URL))
+        if URL:
+            await self._notifier.start_watching_addr(address, URL)
+        else:
+            await self._notifier.stop_watching_addr(address)
         return True
 
     @command('wn')
@@ -928,10 +963,23 @@ class Commands:
 
     # lightning network commands
     @command('wn')
-    async def add_peer(self, connection_string, timeout=20, wallet: Abstract_Wallet = None):
+    async def add_peer(self, connection_string, timeout=20, gossip=False, wallet: Abstract_Wallet = None):
-        await wallet.lnworker.add_peer(connection_string)
+        lnworker = self.network.lngossip if gossip else wallet.lnworker
+        await lnworker.add_peer(connection_string)
         return True
 
+    @command('wn')
+    async def list_peers(self, gossip=False, wallet: Abstract_Wallet = None):
+        lnworker = self.network.lngossip if gossip else wallet.lnworker
+        return [{
+            'node_id':p.pubkey.hex(),
+            'address':p.transport.name(),
+            'initialized':p.is_initialized(),
+            'features': str(LnFeatures(p.features)),
+            'channels': [c.funding_outpoint.to_str() for c in p.channels.values()],
+        } for p in lnworker.peers.values()]
+
+
     @command('wpn')
     async def open_channel(self, connection_string, amount, push_amount=0, password=None, wallet: Abstract_Wallet = None):
         funding_sat = satoshis(amount)
@@ -945,9 +993,24 @@ class Commands:
                                                                          password=password)
         return chan.funding_outpoint.to_str()
 
+    command('')
+    async def decode_invoice(self, invoice: str):
+        invoice = LNInvoice.from_bech32(invoice)
+        return invoice.to_debug_json()
+
     @command('wn')
-    async def lnpay(self, invoice, attempts=1, timeout=10, wallet: Abstract_Wallet = None):
+    async def lnpay(self, invoice, attempts=1, timeout=30, wallet: Abstract_Wallet = None):
-        return await wallet.lnworker._pay(invoice, attempts=attempts)
+        lnworker = wallet.lnworker
+        lnaddr = lnworker._check_invoice(invoice)
+        payment_hash = lnaddr.paymenthash
+        wallet.save_invoice(LNInvoice.from_bech32(invoice))
+        success, log = await lnworker._pay(invoice, attempts=attempts)
+        return {
+            'payment_hash': payment_hash.hex(),
+            'success': success,
+            'preimage': lnworker.get_preimage(payment_hash).hex() if success else None,
+            'log': [x.formatted_tuple() for x in log]
+        }
 
     @command('w')
     async def nodeid(self, wallet: Abstract_Wallet = None):
@@ -956,7 +1019,27 @@ class Commands:
 
     @command('w')
     async def list_channels(self, wallet: Abstract_Wallet = None):
-        return list(wallet.lnworker.list_channels())
+                # we output the funding_outpoint instead of the channel_id because lnd uses channel_point (funding outpoint) to identify channels
+        from .lnutil import LOCAL, REMOTE, format_short_channel_id
+        l = list(wallet.lnworker.channels.items())
+        return [
+            {
+                'short_channel_id': format_short_channel_id(chan.short_channel_id) if chan.short_channel_id else None,
+                'channel_id': bh2u(chan.channel_id),
+                'channel_point': chan.funding_outpoint.to_str(),
+                'state': chan.get_state().name,
+                'peer_state': chan.peer_state.name,
+                'remote_pubkey': bh2u(chan.node_id),
+                'local_balance': chan.balance(LOCAL)//1000,
+                'remote_balance': chan.balance(REMOTE)//1000,
+                'local_reserve': chan.config[REMOTE].reserve_sat, # their config has our reserve
+                'remote_reserve': chan.config[LOCAL].reserve_sat,
+                'local_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(LOCAL, direction=SENT) // 1000,
+                'remote_unsettled_sent': chan.balance_tied_up_in_htlcs_by_direction(REMOTE, direction=SENT) // 1000,
+            } for channel_id, chan in l
+        ]
+
+
 
     @command('wn')
     async def dumpgraph(self, wallet: Abstract_Wallet = None):
@@ -968,17 +1051,19 @@ class Commands:
         self.network.config.fee_estimates = ast.literal_eval(fees)
         self.network.notify('fee')
 
+    @command('wn')
+    async def enable_htlc_settle(self, b: bool, wallet: Abstract_Wallet = None):
+        e = wallet.lnworker.enable_htlc_settle
+        e.set() if b else e.clear()
+
     @command('n')
     async def clear_ln_blacklist(self):
         self.network.path_finder.blacklist.clear()
 
     @command('w')
     async def list_invoices(self, wallet: Abstract_Wallet = None):
-        return wallet.get_invoices()
+        l = wallet.get_invoices()
-
+        return [wallet.export_invoice(x) for x in l]
-    @command('w')
-    async def lightning_history(self, wallet: Abstract_Wallet = None):
-        return wallet.lnworker.get_history()
 
     @command('wn')
     async def close_channel(self, channel_point, force=False, wallet: Abstract_Wallet = None):
@@ -987,9 +1072,22 @@ class Commands:
         coro = wallet.lnworker.force_close_channel(chan_id) if force else wallet.lnworker.close_channel(chan_id)
         return await coro
 
+    @command('w')
+    async def export_channel_backup(self, channel_point, wallet: Abstract_Wallet = None):
+        txid, index = channel_point.split(':')
+        chan_id, _ = channel_id_from_funding_tx(txid, int(index))
+        return wallet.lnworker.export_channel_backup(chan_id)
+
+    @command('w')
+    async def import_channel_backup(self, encrypted, wallet: Abstract_Wallet = None):
+        return wallet.lnbackups.import_channel_backup(encrypted)
+
     @command('wn')
-    async def get_channel_ctx(self, channel_point, wallet: Abstract_Wallet = None):
+    async def get_channel_ctx(self, channel_point, iknowwhatimdoing=False, wallet: Abstract_Wallet = None):
         """ return the current commitment transaction of a channel """
+        if not iknowwhatimdoing:
+            raise Exception("WARNING: this command is potentially unsafe.\n"
+                            "To proceed, try again, with the --iknowwhatimdoing option.")
         txid, index = channel_point.split(':')
         chan_id, _ = channel_id_from_funding_tx(txid, int(index))
         chan = wallet.lnworker.channels[chan_id]
@@ -1001,6 +1099,60 @@ class Commands:
         """ return the local watchtower's ctn of channel. used in regtests """
         return await self.network.local_watchtower.sweepstore.get_ctn(channel_point, None)
 
+    @command('wnp')
+    async def normal_swap(self, onchain_amount, lightning_amount, password=None, wallet: Abstract_Wallet = None):
+        """
+        Normal submarine swap: send on-chain BTC, receive on Lightning
+        Note that your funds will be locked for 24h if you do not have enough incoming capacity.
+        """
+        sm = wallet.lnworker.swap_manager
+        if lightning_amount == 'dryrun':
+            await sm.get_pairs()
+            onchain_amount_sat = satoshis(onchain_amount)
+            lightning_amount_sat = sm.get_recv_amount(onchain_amount_sat, is_reverse=False)
+            txid = None
+        elif onchain_amount == 'dryrun':
+            await sm.get_pairs()
+            lightning_amount_sat = satoshis(lightning_amount)
+            onchain_amount_sat = sm.get_send_amount(lightning_amount_sat, is_reverse=False)
+            txid = None
+        else:
+            lightning_amount_sat = satoshis(lightning_amount)
+            onchain_amount_sat = satoshis(onchain_amount)
+            txid = await wallet.lnworker.swap_manager.normal_swap(lightning_amount_sat, onchain_amount_sat, password)
+        return {
+            'txid': txid,
+            'lightning_amount': format_satoshis(lightning_amount_sat),
+            'onchain_amount': format_satoshis(onchain_amount_sat),
+        }
+
+    @command('wn')
+    async def reverse_swap(self, lightning_amount, onchain_amount, wallet: Abstract_Wallet = None):
+        """Reverse submarine swap: send on Lightning, receive on-chain
+        """
+        sm = wallet.lnworker.swap_manager
+        if onchain_amount == 'dryrun':
+            await sm.get_pairs()
+            lightning_amount_sat = satoshis(lightning_amount)
+            onchain_amount_sat = sm.get_recv_amount(lightning_amount_sat, is_reverse=True)
+            success = None
+        elif lightning_amount == 'dryrun':
+            await sm.get_pairs()
+            onchain_amount_sat = satoshis(onchain_amount)
+            lightning_amount_sat = sm.get_send_amount(onchain_amount_sat, is_reverse=True)
+            success = None
+        else:
+            lightning_amount_sat = satoshis(lightning_amount)
+            onchain_amount_sat = satoshis(onchain_amount)
+            success = await wallet.lnworker.swap_manager.reverse_swap(lightning_amount_sat, onchain_amount_sat)
+        return {
+            'success': success,
+            'lightning_amount': format_satoshis(lightning_amount_sat),
+            'onchain_amount': format_satoshis(onchain_amount_sat),
+        }
+
+
+
 
 def eval_bool(x: str) -> bool:
     if x == 'false': return False
@@ -1027,6 +1179,8 @@ param_descriptions = {
     'requested_amount': 'Requested amount (in LBC).',
     'outputs': 'list of ["address", amount]',
     'redeem_script': 'redeem script (hexadecimal)',
+    'lightning_amount': "Amount sent or received in a submarine swap. Set it to 'dryrun' to receive a value",
+    'onchain_amount': "Amount sent or received in a submarine swap. Set it to 'dryrun' to receive a value",
 }
 
 command_options = {
@@ -1073,6 +1227,8 @@ command_options = {
     'fee_level':   (None, "Float between 0.0 and 1.0, representing fee slider position"),
     'from_height': (None, "Only show transactions that confirmed after given block height"),
     'to_height':   (None, "Only show transactions that confirmed before given block height"),
+    'iknowwhatimdoing': (None, "Acknowledge that I understand the full implications of what I am about to do"),
+    'gossip':      (None, "Apply command to gossip node instead of wallet"),
 }
 
 
@@ -1099,6 +1255,7 @@ arg_types = {
     'encrypt_file': eval_bool,
     'rbf': eval_bool,
     'timeout': float,
+    'attempts': int,
 }
 
 config_variables = {
@@ -1166,11 +1323,13 @@ argparse._SubParsersAction.__call__ = subparser_call
 
 
 def add_network_options(parser):
+    parser.add_argument("-f", "--serverfingerprint", dest="serverfingerprint", default=None, help="only allow connecting to servers with a matching SSL certificate SHA256 fingerprint." + " " +
+                                                                                                  "To calculate this yourself: '$ openssl x509 -noout -fingerprint -sha256 -inform pem -in mycertfile.crt'. Enter as 64 hex chars.")
     parser.add_argument("-1", "--oneserver", action="store_true", dest="oneserver", default=None, help="connect to one server only")
     parser.add_argument("-s", "--server", dest="server", default=None, help="set server host:port:protocol, where protocol is either t (tcp) or s (ssl)")
-    parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port], where type is socks4,socks5 or http")
+    parser.add_argument("-p", "--proxy", dest="proxy", default=None, help="set proxy [type:]host[:port] (or 'none' to disable proxy), where type is socks4,socks5 or http")
     parser.add_argument("--noonion", action="store_true", dest="noonion", default=None, help="do not try to connect to onion servers")
-    parser.add_argument("--skipmerklecheck", action="store_true", dest="skipmerklecheck", default=False, help="Tolerate invalid merkle proofs from server")
+    parser.add_argument("--skipmerklecheck", action="store_true", dest="skipmerklecheck", default=None, help="Tolerate invalid merkle proofs from server")
 
 def add_global_options(parser):
     group = parser.add_argument_group('global options')
@@ -1185,6 +1344,7 @@ def add_global_options(parser):
 
 def add_wallet_option(parser):
     parser.add_argument("-w", "--wallet", dest="wallet_path", help="wallet path")
+    parser.add_argument("--forgetconfig", action="store_true", dest="forget_config", default=False, help="Forget config on exit")
 
 def get_parser():
     # create main parser

electrum/constants.py

@@ -43,6 +43,7 @@ def read_json(filename, default):
 
 GIT_REPO_URL = "https://github.com/spesmilo/electrum"
 GIT_REPO_ISSUES_URL = "https://github.com/spesmilo/electrum/issues"
+BIP39_WALLET_FORMATS = read_json('bip39_wallet_formats.json', [])
 
 
 class AbstractNet:
@@ -89,9 +90,9 @@ class BitcoinMainnet(AbstractNet):
     XPUB_HEADERS_INV = inv_dict(XPUB_HEADERS)
     BIP44_COIN_TYPE = 140
     LN_REALM_BYTE = 0
-    LN_DNS_SEEDS = [
+    LN_DNS_SEEDS = [  # TODO investigate this again
-        'nodes.lightning.directory.',
+        #'test.nodes.lightning.directory.',  # times out.
-        'lseed.bitcoinstats.com.',
+        #'lseed.bitcoinstats.com.',  # ignores REALM byte and returns mainnet peers...
     ]
 
 
@@ -125,9 +126,9 @@ class BitcoinTestnet(AbstractNet):
     XPUB_HEADERS_INV = inv_dict(XPUB_HEADERS)
     BIP44_COIN_TYPE = 1
     LN_REALM_BYTE = 1
-    LN_DNS_SEEDS = [
+    LN_DNS_SEEDS = [  # TODO investigate this again
-        'test.nodes.lightning.directory.',
+        #'test.nodes.lightning.directory.',  # times out.
-        'lseed.bitcoinstats.com.',
+        #'lseed.bitcoinstats.com.',  # ignores REALM byte and returns mainnet peers...
     ]
 
 

electrum/contacts.py

@@ -27,7 +27,7 @@ from dns.exception import DNSException
 
 from . import bitcoin
 from . import dnssec
-from .util import export_meta, import_meta, to_string
+from .util import read_json_file, write_json_file, to_string
 from .logging import Logger
 
 
@@ -52,14 +52,13 @@ class Contacts(dict, Logger):
         self.db.put('contacts', dict(self))
 
     def import_file(self, path):
-        import_meta(path, self._validate, self.load_meta)
+        data = read_json_file(path)
-
+        data = self._validate(data)
-    def load_meta(self, data):
         self.update(data)
         self.save()
 
-    def export_file(self, filename):
+    def export_file(self, path):
-        export_meta(self, filename)
+        write_json_file(path, self)
 
     def __setitem__(self, key, value):
         dict.__setitem__(self, key, value)

electrum/crypto.py

@@ -25,6 +25,7 @@
 
 import base64
 import os
+import sys
 import hashlib
 import hmac
 from typing import Union
@@ -34,11 +35,33 @@ import pyaes
 from .util import assert_bytes, InvalidPassword, to_bytes, to_string, WalletFileException
 from .i18n import _
 
-
+HAS_CRYPTODOME = False
 try:
-    from Cryptodome.Cipher import AES
+    from Cryptodome.Cipher import ChaCha20_Poly1305 as CD_ChaCha20_Poly1305
+    from Cryptodome.Cipher import ChaCha20 as CD_ChaCha20
+    from Cryptodome.Cipher import AES as CD_AES
 except:
-    AES = None
+    pass
+else:
+    HAS_CRYPTODOME = True
+
+HAS_CRYPTOGRAPHY = False
+try:
+    import cryptography
+    from cryptography import exceptions
+    from cryptography.hazmat.primitives.ciphers import Cipher as CG_Cipher
+    from cryptography.hazmat.primitives.ciphers import algorithms as CG_algorithms
+    from cryptography.hazmat.primitives.ciphers import modes as CG_modes
+    from cryptography.hazmat.backends import default_backend as CG_default_backend
+    import cryptography.hazmat.primitives.ciphers.aead as CG_aead
+except:
+    pass
+else:
+    HAS_CRYPTOGRAPHY = True
+
+
+if not (HAS_CRYPTODOME or HAS_CRYPTOGRAPHY):
+    sys.exit(f"Error: at least one of ('pycryptodomex', 'cryptography') needs to be installed.")
 
 
 class InvalidPadding(Exception):
@@ -67,8 +90,12 @@ def strip_PKCS7_padding(data: bytes) -> bytes:
 def aes_encrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
     assert_bytes(key, iv, data)
     data = append_PKCS7_padding(data)
-    if AES:
+    if HAS_CRYPTODOME:
-        e = AES.new(key, AES.MODE_CBC, iv).encrypt(data)
+        e = CD_AES.new(key, CD_AES.MODE_CBC, iv).encrypt(data)
+    elif HAS_CRYPTOGRAPHY:
+        cipher = CG_Cipher(CG_algorithms.AES(key), CG_modes.CBC(iv), backend=CG_default_backend())
+        encryptor = cipher.encryptor()
+        e = encryptor.update(data) + encryptor.finalize()
     else:
         aes_cbc = pyaes.AESModeOfOperationCBC(key, iv=iv)
         aes = pyaes.Encrypter(aes_cbc, padding=pyaes.PADDING_NONE)
@@ -78,9 +105,13 @@ def aes_encrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
 
 def aes_decrypt_with_iv(key: bytes, iv: bytes, data: bytes) -> bytes:
     assert_bytes(key, iv, data)
-    if AES:
+    if HAS_CRYPTODOME:
-        cipher = AES.new(key, AES.MODE_CBC, iv)
+        cipher = CD_AES.new(key, CD_AES.MODE_CBC, iv)
         data = cipher.decrypt(data)
+    elif HAS_CRYPTOGRAPHY:
+        cipher = CG_Cipher(CG_algorithms.AES(key), CG_modes.CBC(iv), backend=CG_default_backend())
+        decryptor = cipher.decryptor()
+        data = decryptor.update(data) + decryptor.finalize()
     else:
         aes_cbc = pyaes.AESModeOfOperationCBC(key, iv=iv)
         aes = pyaes.Decrypter(aes_cbc, padding=pyaes.PADDING_NONE)
@@ -157,34 +188,88 @@ def _hash_password(password: Union[bytes, str], *, version: int) -> bytes:
         raise UnexpectedPasswordHashVersion(version)
 
 
-def pw_encode(data: str, password: Union[bytes, str, None], *, version: int) -> str:
+def _pw_encode_raw(data: bytes, password: Union[bytes, str], *, version: int) -> bytes:
-    if not password:
-        return data
     if version not in KNOWN_PW_HASH_VERSIONS:
         raise UnexpectedPasswordHashVersion(version)
     # derive key from password
     secret = _hash_password(password, version=version)
     # encrypt given data
-    ciphertext = EncodeAES_bytes(secret, to_bytes(data, "utf8"))
+    ciphertext = EncodeAES_bytes(secret, data)
-    ciphertext_b64 = base64.b64encode(ciphertext)
+    return ciphertext
-    return ciphertext_b64.decode('utf8')
 
 
-def pw_decode(data: str, password: Union[bytes, str, None], *, version: int) -> str:
+def _pw_decode_raw(data_bytes: bytes, password: Union[bytes, str], *, version: int) -> bytes:
-    if password is None:
-        return data
     if version not in KNOWN_PW_HASH_VERSIONS:
         raise UnexpectedPasswordHashVersion(version)
-    data_bytes = bytes(base64.b64decode(data))
     # derive key from password
     secret = _hash_password(password, version=version)
     # decrypt given data
     try:
-        d = to_string(DecodeAES_bytes(secret, data_bytes), "utf8")
+        d = DecodeAES_bytes(secret, data_bytes)
     except Exception as e:
         raise InvalidPassword() from e
     return d
 
+def pw_encode_bytes(data: bytes, password: Union[bytes, str], *, version: int) -> str:
+    """plaintext bytes -> base64 ciphertext"""
+    ciphertext = _pw_encode_raw(data, password, version=version)
+    ciphertext_b64 = base64.b64encode(ciphertext)
+    return ciphertext_b64.decode('utf8')
+
+
+def pw_decode_bytes(data: str, password: Union[bytes, str], *, version:int) -> bytes:
+    """base64 ciphertext -> plaintext bytes"""
+    if version not in KNOWN_PW_HASH_VERSIONS:
+        raise UnexpectedPasswordHashVersion(version)
+    data_bytes = bytes(base64.b64decode(data))
+    return _pw_decode_raw(data_bytes, password, version=version)
+
+
+def pw_encode_with_version_and_mac(data: bytes, password: Union[bytes, str]) -> str:
+    """plaintext bytes -> base64 ciphertext"""
+    # https://crypto.stackexchange.com/questions/202/should-we-mac-then-encrypt-or-encrypt-then-mac
+    # Encrypt-and-MAC. The MAC will be used to detect invalid passwords
+    version = PW_HASH_VERSION_LATEST
+    mac = sha256(data)[0:4]
+    ciphertext = _pw_encode_raw(data, password, version=version)
+    ciphertext_b64 = base64.b64encode(bytes([version]) + ciphertext + mac)
+    return ciphertext_b64.decode('utf8')
+
+
+def pw_decode_with_version_and_mac(data: str, password: Union[bytes, str]) -> bytes:
+    """base64 ciphertext -> plaintext bytes"""
+    data_bytes = bytes(base64.b64decode(data))
+    version = int(data_bytes[0])
+    encrypted = data_bytes[1:-4]
+    mac = data_bytes[-4:]
+    if version not in KNOWN_PW_HASH_VERSIONS:
+        raise UnexpectedPasswordHashVersion(version)
+    decrypted = _pw_decode_raw(encrypted, password, version=version)
+    if sha256(decrypted)[0:4] != mac:
+        raise InvalidPassword()
+    return decrypted
+
+
+def pw_encode(data: str, password: Union[bytes, str, None], *, version: int) -> str:
+    """plaintext str -> base64 ciphertext"""
+    if not password:
+        return data
+    plaintext_bytes = to_bytes(data, "utf8")
+    return pw_encode_bytes(plaintext_bytes, password, version=version)
+
+
+def pw_decode(data: str, password: Union[bytes, str, None], *, version: int) -> str:
+    """base64 ciphertext -> plaintext str"""
+    if password is None:
+        return data
+    plaintext_bytes = pw_decode_bytes(data, password, version=version)
+    try:
+        plaintext_str = to_string(plaintext_bytes, "utf8")
+    except UnicodeDecodeError as e:
+        raise InvalidPassword() from e
+    return plaintext_str
+
+
 
 def sha256(x: Union[bytes, str]) -> bytes:
     x = to_bytes(x, 'utf8')
@@ -216,3 +301,70 @@ def hmac_oneshot(key: bytes, msg: bytes, digest) -> bytes:
         return hmac.digest(key, msg, digest)
     else:
         return hmac.new(key, msg, digest).digest()
+
+
+
+def chacha20_poly1305_encrypt(
+        *,
+        key: bytes,
+        nonce: bytes,
+        associated_data: bytes = None,
+        data: bytes
+) -> bytes:
+    assert isinstance(key, (bytes, bytearray))
+    assert isinstance(nonce, (bytes, bytearray))
+    assert isinstance(associated_data, (bytes, bytearray, type(None)))
+    assert isinstance(data, (bytes, bytearray))
+    if HAS_CRYPTODOME:
+        cipher = CD_ChaCha20_Poly1305.new(key=key, nonce=nonce)
+        if associated_data is not None:
+            cipher.update(associated_data)
+        ciphertext, mac = cipher.encrypt_and_digest(plaintext=data)
+        return ciphertext + mac
+    if HAS_CRYPTOGRAPHY:
+        a = CG_aead.ChaCha20Poly1305(key)
+        return a.encrypt(nonce, data, associated_data)
+    raise Exception("no chacha20 backend found")
+
+
+def chacha20_poly1305_decrypt(
+        *,
+        key: bytes,
+        nonce: bytes,
+        associated_data: bytes = None,
+        data: bytes
+) -> bytes:
+    assert isinstance(key, (bytes, bytearray))
+    assert isinstance(nonce, (bytes, bytearray))
+    assert isinstance(associated_data, (bytes, bytearray, type(None)))
+    assert isinstance(data, (bytes, bytearray))
+    if HAS_CRYPTODOME:
+        cipher = CD_ChaCha20_Poly1305.new(key=key, nonce=nonce)
+        if associated_data is not None:
+            cipher.update(associated_data)
+        # raises ValueError if not valid (e.g. incorrect MAC)
+        return cipher.decrypt_and_verify(ciphertext=data[:-16], received_mac_tag=data[-16:])
+    if HAS_CRYPTOGRAPHY:
+        a = CG_aead.ChaCha20Poly1305(key)
+        try:
+            return a.decrypt(nonce, data, associated_data)
+        except cryptography.exceptions.InvalidTag as e:
+            raise ValueError("invalid tag") from e
+    raise Exception("no chacha20 backend found")
+
+
+def chacha20_encrypt(*, key: bytes, nonce: bytes, data: bytes) -> bytes:
+    assert isinstance(key, (bytes, bytearray))
+    assert isinstance(nonce, (bytes, bytearray))
+    assert isinstance(data, (bytes, bytearray))
+    assert len(nonce) == 8, f"unexpected nonce size: {len(nonce)} (expected: 8)"
+    if HAS_CRYPTODOME:
+        cipher = CD_ChaCha20.new(key=key, nonce=nonce)
+        return cipher.encrypt(data)
+    if HAS_CRYPTOGRAPHY:
+        nonce = bytes(8) + nonce  # cryptography wants 16 byte nonces
+        algo = CG_algorithms.ChaCha20(key=key, nonce=nonce)
+        cipher = CG_Cipher(algo, mode=None, backend=CG_default_backend())
+        encryptor = cipher.encryptor()
+        return encryptor.update(data)
+    raise Exception("no chacha20 backend found")

electrum/daemon.py

@@ -29,21 +29,21 @@ import time
 import traceback
 import sys
 import threading
-from typing import Dict, Optional, Tuple, Iterable
+from typing import Dict, Optional, Tuple, Iterable, Callable, Union, Sequence, Mapping
 from base64 import b64decode, b64encode
 from collections import defaultdict
+import concurrent
+from concurrent import futures
+import json
 
 import aiohttp
 from aiohttp import web, client_exceptions
-import jsonrpcclient
-import jsonrpcserver
-from jsonrpcserver import response
-from jsonrpcclient.clients.aiohttp_client import AiohttpClient
 from aiorpcx import TaskGroup
 
+from . import util
 from .network import Network
 from .util import (json_decode, to_bytes, to_string, profiler, standardize_path, constant_time_compare)
-from .util import PR_PAID, PR_EXPIRED, get_request_status
+from .invoices import PR_PAID, PR_EXPIRED
 from .util import log_exceptions, ignore_exceptions, randrange
 from .wallet import Wallet, Abstract_Wallet
 from .storage import WalletStorage
@@ -104,10 +104,8 @@ def request(config: SimpleConfig, endpoint, args=(), timeout=60):
         loop = asyncio.get_event_loop()
         async def request_coroutine():
             async with aiohttp.ClientSession(auth=auth) as session:
-                server = AiohttpClient(session, server_url)
+                c = util.JsonRPCClient(session, server_url)
-                f = getattr(server, endpoint)
+                return await c.request(endpoint, *args)
-                response = await f(*args)
-                return response.data.result
         try:
             fut = asyncio.run_coroutine_threadsafe(request_coroutine(), loop)
             return fut.result(timeout=timeout)
@@ -137,127 +135,6 @@ def get_rpc_credentials(config: SimpleConfig) -> Tuple[str, str]:
     return rpc_user, rpc_password
 
 
-class WatchTowerServer(Logger):
-
-    def __init__(self, network):
-        Logger.__init__(self)
-        self.config = network.config
-        self.network = network
-        self.lnwatcher = network.local_watchtower
-        self.app = web.Application()
-        self.app.router.add_post("/", self.handle)
-        self.methods = jsonrpcserver.methods.Methods()
-        self.methods.add(self.get_ctn)
-        self.methods.add(self.add_sweep_tx)
-
-    async def handle(self, request):
-        request = await request.text()
-        self.logger.info(f'{request}')
-        response = await jsonrpcserver.async_dispatch(request, methods=self.methods)
-        if response.wanted:
-            return web.json_response(response.deserialized(), status=response.http_status)
-        else:
-            return web.Response()
-
-    async def run(self):
-        host = self.config.get('watchtower_host')
-        port = self.config.get('watchtower_port', 12345)
-        self.runner = web.AppRunner(self.app)
-        await self.runner.setup()
-        site = web.TCPSite(self.runner, host, port, ssl_context=self.config.get_ssl_context())
-        await site.start()
-
-    async def get_ctn(self, *args):
-        return await self.lnwatcher.sweepstore.get_ctn(*args)
-
-    async def add_sweep_tx(self, *args):
-        return await self.lnwatcher.sweepstore.add_sweep_tx(*args)
-
-
-class PayServer(Logger):
-
-    def __init__(self, daemon: 'Daemon'):
-        Logger.__init__(self)
-        self.daemon = daemon
-        self.config = daemon.config
-        self.pending = defaultdict(asyncio.Event)
-        self.daemon.network.register_callback(self.on_payment, ['payment_received'])
-
-    async def on_payment(self, evt, wallet, key, status):
-        if status == PR_PAID:
-            await self.pending[key].set()
-
-    @ignore_exceptions
-    @log_exceptions
-    async def run(self):
-        host = self.config.get('payserver_host', 'localhost')
-        port = self.config.get('payserver_port')
-        root = self.config.get('payserver_root', '/r')
-        app = web.Application()
-        app.add_routes([web.post('/api/create_invoice', self.create_request)])
-        app.add_routes([web.get('/api/get_invoice', self.get_request)])
-        app.add_routes([web.get('/api/get_status', self.get_status)])
-        app.add_routes([web.get('/bip70/{key}.bip70', self.get_bip70_request)])
-        app.add_routes([web.static(root, 'electrum/www')])
-        runner = web.AppRunner(app)
-        await runner.setup()
-        site = web.TCPSite(runner, port=port, host=host, ssl_context=self.config.get_ssl_context())
-        await site.start()
-
-    async def create_request(self, request):
-        params = await request.post()
-        wallet = self.daemon.wallet
-        if 'amount_sat' not in params or not params['amount_sat'].isdigit():
-            raise web.HTTPUnsupportedMediaType()
-        amount = int(params['amount_sat'])
-        message = params['message'] or "donation"
-        payment_hash = await wallet.lnworker._add_invoice_coro(amount, message, 3600)
-        key = payment_hash.hex()
-        raise web.HTTPFound(self.root + '/pay?id=' + key)
-
-    async def get_request(self, r):
-        key = r.query_string
-        request = self.daemon.wallet.get_request(key)
-        return web.json_response(request)
-
-    async def get_bip70_request(self, r):
-        from .paymentrequest import make_request
-        key = r.match_info['key']
-        request = self.daemon.wallet.get_request(key)
-        if not request:
-            return web.HTTPNotFound()
-        pr = make_request(self.config, request)
-        return web.Response(body=pr.SerializeToString(), content_type='application/bitcoin-paymentrequest')
-
-    async def get_status(self, request):
-        ws = web.WebSocketResponse()
-        await ws.prepare(request)
-        key = request.query_string
-        info = self.daemon.wallet.get_request(key)
-        if not info:
-            await ws.send_str('unknown invoice')
-            await ws.close()
-            return ws
-        if info.get('status') == PR_PAID:
-            await ws.send_str(f'paid')
-            await ws.close()
-            return ws
-        if info.get('status') == PR_EXPIRED:
-            await ws.send_str(f'expired')
-            await ws.close()
-            return ws
-        while True:
-            try:
-                await asyncio.wait_for(self.pending[key].wait(), 1)
-                break
-            except asyncio.TimeoutError:
-                # send data on the websocket, to keep it alive
-                await ws.send_str('waiting')
-        await ws.send_str('paid')
-        await ws.close()
-        return ws
-
-
 class AuthenticationError(Exception):
     pass
 
@@ -267,59 +144,18 @@ class AuthenticationInvalidOrMissing(AuthenticationError):
 class AuthenticationCredentialsInvalid(AuthenticationError):
     pass
 
-class Daemon(Logger):
+class AuthenticatedServer(Logger):
 
-    @profiler
+    def __init__(self, rpc_user, rpc_password):
-    def __init__(self, config: SimpleConfig, fd=None, *, listen_jsonrpc=True):
         Logger.__init__(self)
+        self.rpc_user = rpc_user
+        self.rpc_password = rpc_password
         self.auth_lock = asyncio.Lock()
-        self.running = False
+        self._methods = {}  # type: Dict[str, Callable]
-        self.running_lock = threading.Lock()
-        self.config = config
-        if fd is None and listen_jsonrpc:
-            fd = get_file_descriptor(config)
-            if fd is None:
-                raise Exception('failed to lock daemon; already running?')
-        self.asyncio_loop = asyncio.get_event_loop()
-        self.network = None
-        if not config.get('offline'):
-            self.network = Network(config, daemon=self)
-        self.fx = FxThread(config, self.network)
-        self.gui_object = None
-        # path -> wallet;   make sure path is standardized.
-        self._wallets = {}  # type: Dict[str, Abstract_Wallet]
-        daemon_jobs = []
-        # Setup JSONRPC server
-        if listen_jsonrpc:
-            daemon_jobs.append(self.start_jsonrpc(config, fd))
-        # request server
-        self.pay_server = None
-        if not config.get('offline') and self.config.get('run_payserver'):
-            self.pay_server = PayServer(self)
-            daemon_jobs.append(self.pay_server.run())
-        # server-side watchtower
-        self.watchtower = None
-        if not config.get('offline') and self.config.get('run_watchtower'):
-            self.watchtower = WatchTowerServer(self.network)
-            daemon_jobs.append(self.watchtower.run)
-        if self.network:
-            self.network.start(jobs=[self.fx.run])
 
-        self.taskgroup = TaskGroup()
+    def register_method(self, f):
-        asyncio.run_coroutine_threadsafe(self._run(jobs=daemon_jobs), self.asyncio_loop)
+        assert f.__name__ not in self._methods, f"name collision for {f.__name__}"
-
+        self._methods[f.__name__] = f
-    @log_exceptions
-    async def _run(self, jobs: Iterable = None):
-        if jobs is None:
-            jobs = []
-        try:
-            async with self.taskgroup as group:
-                [await group.spawn(job) for job in jobs]
-                await group.spawn(asyncio.Event().wait)  # run forever (until cancel)
-        except BaseException as e:
-            self.logger.exception('daemon.taskgroup died.')
-        finally:
-            self.logger.info("stopping daemon.taskgroup")
 
     async def authenticate(self, headers):
         if self.rpc_password == '':
@@ -348,46 +184,66 @@ class Daemon(Logger):
                                     text='Unauthorized', status=401)
             except AuthenticationCredentialsInvalid:
                 return web.Response(text='Forbidden', status=403)
+        try:
             request = await request.text()
-        response = await jsonrpcserver.async_dispatch(request, methods=self.methods)
+            request = json.loads(request)
-        if isinstance(response, jsonrpcserver.response.ExceptionResponse):
+            method = request['method']
-            self.logger.error(f"error handling request: {request}", exc_info=response.exc)
+            _id = request['id']
-            # this exposes the error message to the client
+            params = request.get('params', [])  # type: Union[Sequence, Mapping]
-            response.message = str(response.exc)
+            if method not in self._methods:
-        if response.wanted:
+                raise Exception(f"attempting to use unregistered method: {method}")
-            return web.json_response(response.deserialized(), status=response.http_status)
+            f = self._methods[method]
+        except Exception as e:
+            self.logger.exception("invalid request")
+            return web.Response(text='Invalid Request', status=500)
+        response = {'id': _id}
+        try:
+            if isinstance(params, dict):
+                response['result'] = await f(**params)
             else:
-            return web.Response()
+                response['result'] = await f(*params)
+        except BaseException as e:
+            self.logger.exception("internal error while executing RPC")
+            response['error'] = str(e)
+        return web.json_response(response)
 
-    async def start_jsonrpc(self, config: SimpleConfig, fd):
+
+class CommandsServer(AuthenticatedServer):
+
+    def __init__(self, daemon, fd):
+        rpc_user, rpc_password = get_rpc_credentials(daemon.config)
+        AuthenticatedServer.__init__(self, rpc_user, rpc_password)
+        self.daemon = daemon
+        self.fd = fd
+        self.config = daemon.config
+        self.host = self.config.get('rpchost', '127.0.0.1')
+        self.port = self.config.get('rpcport', 0)
         self.app = web.Application()
         self.app.router.add_post("/", self.handle)
-        self.rpc_user, self.rpc_password = get_rpc_credentials(config)
+        self.register_method(self.ping)
-        self.methods = jsonrpcserver.methods.Methods()
+        self.register_method(self.gui)
-        self.methods.add(self.ping)
+        self.cmd_runner = Commands(config=self.config, network=self.daemon.network, daemon=self.daemon)
-        self.methods.add(self.gui)
-        self.cmd_runner = Commands(config=self.config, network=self.network, daemon=self)
         for cmdname in known_commands:
-            self.methods.add(getattr(self.cmd_runner, cmdname))
+            self.register_method(getattr(self.cmd_runner, cmdname))
-        self.methods.add(self.run_cmdline)
+        self.register_method(self.run_cmdline)
-        self.host = config.get('rpchost', '127.0.0.1')
+
-        self.port = config.get('rpcport', 0)
+    async def run(self):
         self.runner = web.AppRunner(self.app)
         await self.runner.setup()
         site = web.TCPSite(self.runner, self.host, self.port)
         await site.start()
         socket = site._server.sockets[0]
-        os.write(fd, bytes(repr((socket.getsockname(), time.time())), 'utf8'))
+        os.write(self.fd, bytes(repr((socket.getsockname(), time.time())), 'utf8'))
-        os.close(fd)
+        os.close(self.fd)
 
     async def ping(self):
         return True
 
     async def gui(self, config_options):
-        if self.gui_object:
+        if self.daemon.gui_object:
-            if hasattr(self.gui_object, 'new_window'):
+            if hasattr(self.daemon.gui_object, 'new_window'):
                 path = self.config.get_wallet_path(use_gui_last_wallet=True)
-                self.gui_object.new_window(path, config_options.get('url'))
+                self.daemon.gui_object.new_window(path, config_options.get('url'))
                 response = "ok"
             else:
                 response = "error: current GUI does not support multiple windows"
@@ -395,6 +251,213 @@ class Daemon(Logger):
             response = "Error: Electrum is running in daemon mode. Please stop the daemon first."
         return response
 
+    async def run_cmdline(self, config_options):
+        cmdname = config_options['cmd']
+        cmd = known_commands[cmdname]
+        # arguments passed to function
+        args = [config_options.get(x) for x in cmd.params]
+        # decode json arguments
+        args = [json_decode(i) for i in args]
+        # options
+        kwargs = {}
+        for x in cmd.options:
+            kwargs[x] = config_options.get(x)
+        if 'wallet_path' in cmd.options:
+            kwargs['wallet_path'] = config_options.get('wallet_path')
+        elif 'wallet' in cmd.options:
+            kwargs['wallet'] = config_options.get('wallet_path')
+        func = getattr(self.cmd_runner, cmd.name)
+        # fixme: not sure how to retrieve message in jsonrpcclient
+        try:
+            result = await func(*args, **kwargs)
+        except Exception as e:
+            result = {'error':str(e)}
+        return result
+
+
+class WatchTowerServer(AuthenticatedServer):
+
+    def __init__(self, network, netaddress):
+        self.addr = netaddress
+        self.config = network.config
+        self.network = network
+        watchtower_user = self.config.get('watchtower_user', '')
+        watchtower_password = self.config.get('watchtower_password', '')
+        AuthenticatedServer.__init__(self, watchtower_user, watchtower_password)
+        self.lnwatcher = network.local_watchtower
+        self.app = web.Application()
+        self.app.router.add_post("/", self.handle)
+        self.register_method(self.get_ctn)
+        self.register_method(self.add_sweep_tx)
+
+    async def run(self):
+        self.runner = web.AppRunner(self.app)
+        await self.runner.setup()
+        site = web.TCPSite(self.runner, host=str(self.addr.host), port=self.addr.port, ssl_context=self.config.get_ssl_context())
+        await site.start()
+
+    async def get_ctn(self, *args):
+        return await self.lnwatcher.sweepstore.get_ctn(*args)
+
+    async def add_sweep_tx(self, *args):
+        return await self.lnwatcher.sweepstore.add_sweep_tx(*args)
+
+
+class PayServer(Logger):
+
+    def __init__(self, daemon: 'Daemon', netaddress):
+        Logger.__init__(self)
+        self.addr = netaddress
+        self.daemon = daemon
+        self.config = daemon.config
+        self.pending = defaultdict(asyncio.Event)
+        util.register_callback(self.on_payment, ['request_status'])
+
+@property
+def wallet(self):
+ # FIXME specify wallet somehow?
+ return list(self.daemon.get_wallets().values())[0]
+
+    async def on_payment(self, evt, key, status):
+        if status == PR_PAID:
+             self.pending[key].set()
+
+    @ignore_exceptions
+    @log_exceptions
+    async def run(self):
+        root = self.config.get('payserver_root', '/r')
+        app = web.Application()
+        app.add_routes([web.get('/api/get_invoice', self.get_request)])
+        app.add_routes([web.get('/api/get_status', self.get_status)])
+        app.add_routes([web.get('/bip70/{key}.bip70', self.get_bip70_request)])
+        app.add_routes([web.static(root, os.path.join(os.path.dirname(__file__), 'www'))])
+        if self.config.get('payserver_allow_create_invoice'):
+            app.add_routes([web.post('/api/create_invoice', self.create_request)])
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, host=str(self.addr.host), port=self.addr.port, ssl_context=self.config.get_ssl_context())
+        await site.start()
+
+    async def create_request(self, request):
+        params = await request.post()
+        wallet = self.wallet
+        if 'amount_sat' not in params or not params['amount_sat'].isdigit():
+            raise web.HTTPUnsupportedMediaType()
+        amount = int(params['amount_sat'])
+        message = params['message'] or "donation"
+        payment_hash = wallet.lnworker.add_request(
+            amount_sat=amount,
+            message=message,
+            expiry=3600)
+        key = payment_hash.hex()
+        raise web.HTTPFound(self.root + '/pay?id=' + key)
+
+    async def get_request(self, r):
+        key = r.query_string
+        request = self.wallet.get_formatted_request(key)
+        return web.json_response(request)
+
+    async def get_bip70_request(self, r):
+        from .paymentrequest import make_request
+        key = r.match_info['key']
+        request = self.wallet.get_request(key)
+        if not request:
+            return web.HTTPNotFound()
+        pr = make_request(self.config, request)
+        return web.Response(body=pr.SerializeToString(), content_type='application/bitcoin-paymentrequest')
+
+    async def get_status(self, request):
+        ws = web.WebSocketResponse()
+        await ws.prepare(request)
+        key = request.query_string
+        info = self.wallet.get_formatted_request(key)
+        if not info:
+            await ws.send_str('unknown invoice')
+            await ws.close()
+            return ws
+        if info.get('status') == PR_PAID:
+            await ws.send_str(f'paid')
+            await ws.close()
+            return ws
+        if info.get('status') == PR_EXPIRED:
+            await ws.send_str(f'expired')
+            await ws.close()
+            return ws
+        while True:
+            try:
+                await asyncio.wait_for(self.pending[key].wait(), 1)
+                break
+            except asyncio.TimeoutError:
+                # send data on the websocket, to keep it alive
+                await ws.send_str('waiting')
+        await ws.send_str('paid')
+        await ws.close()
+        return ws
+
+
+class Daemon(Logger):
+
+
+    network: Optional[Network]
+
+
+    @profiler
+    def __init__(self, config: SimpleConfig, fd=None, *, listen_jsonrpc=True):
+        Logger.__init__(self)
+        self.running = False
+        self.running_lock = threading.Lock()
+        self.config = config
+        if fd is None and listen_jsonrpc:
+            fd = get_file_descriptor(config)
+            if fd is None:
+                raise Exception('failed to lock daemon; already running?')
+        self.asyncio_loop = asyncio.get_event_loop()
+        self.network = None
+        if not config.get('offline'):
+            self.network = Network(config, daemon=self)
+        self.fx = FxThread(config, self.network)
+        self.gui_object = None
+        # path -> wallet;   make sure path is standardized.
+        self._wallets = {}  # type: Dict[str, Abstract_Wallet]
+        daemon_jobs = []
+        # Setup commands server
+        self.commands_server = None
+        if listen_jsonrpc:
+            self.commands_server = CommandsServer(self, fd)
+            daemon_jobs.append(self.commands_server.run())
+        # pay server
+        payserver_address = self.config.get_netaddress('payserver_address')
+        if not config.get('offline') and payserver_address:
+            self.pay_server = PayServer(self, payserver_address)
+            daemon_jobs.append(self.pay_server.run())
+        # server-side watchtower
+        self.watchtower = None
+        watchtower_address = self.config.get_netaddress('watchtower_address')
+        if not config.get('offline') and watchtower_address:
+            self.watchtower = WatchTowerServer(self.network, watchtower_address)
+            daemon_jobs.append(self.watchtower.run)
+        if self.network:
+            self.network.start(jobs=[self.fx.run])
+
+        self.taskgroup = TaskGroup()
+        asyncio.run_coroutine_threadsafe(self._run(jobs=daemon_jobs), self.asyncio_loop)
+
+    @log_exceptions
+    async def _run(self, jobs: Iterable = None):
+        if jobs is None:
+            jobs = []
+        self.logger.info("starting taskgroup.")
+        try:
+            async with self.taskgroup as group:
+                [await group.spawn(job) for job in jobs]
+                await group.spawn(asyncio.Event().wait)  # run forever (until cancel)
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            self.logger.exception("taskgroup died.")
+        finally:
+            self.logger.info("taskgroup stopped.")
+
     def load_wallet(self, path, password, *, manual_upgrades=True) -> Optional[Abstract_Wallet]:
         path = standardize_path(path)
         # wizard will be launched if we return
@@ -419,7 +482,6 @@ class Daemon(Logger):
         wallet = Wallet(db, storage, config=self.config)
         wallet.start_network(self.network)
         self._wallets[path] = wallet
-        self.wallet = wallet
         return wallet
 
     def add_wallet(self, wallet: Abstract_Wallet) -> None:
@@ -427,7 +489,7 @@ class Daemon(Logger):
         path = standardize_path(path)
         self._wallets[path] = wallet
 
-    def get_wallet(self, path: str) -> Abstract_Wallet:
+    def get_wallet(self, path: str) -> Optional[Abstract_Wallet]:
         path = standardize_path(path)
         return self._wallets.get(path)
 
@@ -447,30 +509,9 @@ class Daemon(Logger):
         wallet = self._wallets.pop(path, None)
         if not wallet:
             return False
-        wallet.stop_threads()
+        wallet.stop()
         return True
 
-    async def run_cmdline(self, config_options):
-        cmdname = config_options['cmd']
-        cmd = known_commands[cmdname]
-        # arguments passed to function
-        args = [config_options.get(x) for x in cmd.params]
-        # decode json arguments
-        args = [json_decode(i) for i in args]
-        # options
-        kwargs = {}
-        for x in cmd.options:
-            kwargs[x] = config_options.get(x)
-        if cmd.requires_wallet:
-            kwargs['wallet_path'] = config_options.get('wallet_path')
-        func = getattr(self.cmd_runner, cmd.name)
-        # fixme: not sure how to retrieve message in jsonrpcclient
-        try:
-            result = await func(*args, **kwargs)
-        except Exception as e:
-            result = {'error':str(e)}
-        return result
-
     def run_daemon(self):
         self.running = True
         try:
@@ -493,7 +534,7 @@ class Daemon(Logger):
             self.gui_object.stop()
         # stop network/wallets
         for k, wallet in self._wallets.items():
-            wallet.stop_threads()
+            wallet.stop()
         if self.network:
             self.logger.info("shutting down network")
             self.network.stop()
@@ -501,7 +542,7 @@ class Daemon(Logger):
         fut = asyncio.run_coroutine_threadsafe(self.taskgroup.cancel_remaining(), self.asyncio_loop)
         try:
             fut.result(timeout=2)
-        except (asyncio.TimeoutError, asyncio.CancelledError):
+        except (concurrent.futures.TimeoutError, concurrent.futures.CancelledError, asyncio.CancelledError):
             pass
         self.logger.info("removing lockfile")
         remove_lockfile(get_lockfile(self.config))
@@ -513,11 +554,13 @@ class Daemon(Logger):
         if gui_name in ['lite', 'classic']:
             gui_name = 'qt'
         self.logger.info(f'launching GUI: {gui_name}')
+        try:
             gui = __import__('electrum.gui.' + gui_name, fromlist=['electrum'])
             self.gui_object = gui.ElectrumGui(config, self, plugins)
-        try:
             self.gui_object.main()
         except BaseException as e:
-            self.logger.exception('')
+            self.logger.error(f'GUI raised exception: {repr(e)}. shutting down.')
+            raise
+        finally:
             # app will exit now
             self.on_stop()

electrum/dnssec.py

@@ -35,6 +35,7 @@
 # import sys
 import time
 import struct
+import hashlib
 
 
 import dns.name
@@ -165,11 +166,23 @@ def python_validate_rrsig(rrset, rrsig, keys, origin=None, now=None):
 
     raise ValidationFailure('verify failure')
 
+class PyCryptodomexHashAlike:
+    def __init__(self, hashlib_func):
+        self._hash = hashlib_func
+    def new(self):
+        return self._hash()
+
 
 # replace validate_rrsig
 dns.dnssec._validate_rrsig = python_validate_rrsig
 dns.dnssec.validate_rrsig = python_validate_rrsig
 dns.dnssec.validate = dns.dnssec._validate
+dns.dnssec._have_ecdsa = True
+dns.dnssec.MD5 = PyCryptodomexHashAlike(hashlib.md5)
+dns.dnssec.SHA1 = PyCryptodomexHashAlike(hashlib.sha1)
+dns.dnssec.SHA256 = PyCryptodomexHashAlike(hashlib.sha256)
+dns.dnssec.SHA384 = PyCryptodomexHashAlike(hashlib.sha384)
+dns.dnssec.SHA512 = PyCryptodomexHashAlike(hashlib.sha512)
 
 
 

electrum/ecc.py

@@ -115,6 +115,7 @@ def sig_string_from_r_and_s(r: int, s: int) -> bytes:
 
 
 def _x_and_y_from_pubkey_bytes(pubkey: bytes) -> Tuple[int, int]:
+    assert isinstance(pubkey, bytes), f'pubkey must be bytes, not {type(pubkey)}'
     pubkey_ptr = create_string_buffer(64)
     assert isinstance(pubkey, bytes), f'pubkey must be bytes, not {type(pubkey)}'
     ret = _libsecp256k1.secp256k1_ec_pubkey_parse(

electrum/ecc_fast.py

@@ -49,16 +49,17 @@ def load_library():
         library_paths = (os.path.join(os.path.dirname(__file__), 'libsecp256k1.so.0'),
                          'libsecp256k1.so.0')
 
+    exceptions = []
     secp256k1 = None
     for libpath in library_paths:
         try:
             secp256k1 = ctypes.cdll.LoadLibrary(libpath)
-        except:
+        except BaseException as e:
-            pass
+            exceptions.append(e)
         else:
             break
     if not secp256k1:
-        _logger.error('libsecp256k1 library failed to load')
+        _logger.error(f'libsecp256k1 library failed to load. exceptions: {repr(exceptions)}')
         return None
 
     try:

electrum/exchange_rate.py

@@ -452,12 +452,11 @@ def get_exchanges_by_ccy(history=True):
 
 class FxThread(ThreadJob):
 
-    def __init__(self, config: SimpleConfig, network: Network):
+    def __init__(self, config: SimpleConfig, network: Optional[Network]):
         ThreadJob.__init__(self)
         self.config = config
         self.network = network
-        if self.network:
+        util.register_callback(self.set_proxy, ['proxy_set'])
-            self.network.register_callback(self.set_proxy, ['proxy_set'])
         self.ccy = self.get_currency()
         self.history_used_spot = False
         self.ccy_combo = None
@@ -516,8 +515,11 @@ class FxThread(ThreadJob):
         self.config.set_key('use_exchange_rate', bool(b))
         self.trigger_update()
 
-    def get_history_config(self, *, default=False):
+    def get_history_config(self, *, allow_none=False):
-        return bool(self.config.get('history_rates', default))
+        val = self.config.get('history_rates', None)
+        if val is None and allow_none:
+            return None
+        return bool(val)
 
     def set_history_config(self, b):
         self.config.set_key('history_rates', bool(b))
@@ -567,12 +569,11 @@ class FxThread(ThreadJob):
         self.exchange.read_historical_rates(self.ccy, self.cache_dir)
 
     def on_quotes(self):
-        if self.network:
+        util.trigger_callback('on_quotes')
-            self.network.trigger_callback('on_quotes')
 
     def on_history(self):
         if self.network:
-            self.network.trigger_callback('on_history')
+        util.trigger_callback('on_history')
 
     def exchange_rate(self) -> Decimal:
         """Returns the exchange rate as a Decimal"""

electrum/gui/kivy/__init__.py

@@ -34,8 +34,8 @@ try:
     import kivy
 except ImportError:
     # This error ideally shouldn't be raised with pre-built packages
-    sys.exit("Error: Could not import kivy. Please install it using the" + \
+    sys.exit("Error: Could not import kivy. Please install it using the "
-             "instructions mentioned here `http://kivy.org/#download` .")
+             "instructions mentioned here `https://kivy.org/#download` .")
 
 # minimum required version for kivy
 kivy.require('1.8.0')

electrum/gui/kivy/tools/Dockerfile

@@ -1,6 +1,8 @@
 # based on https://github.com/kivy/python-for-android/blob/master/Dockerfile
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
 
 ENV ANDROID_HOME="/opt/android"
 
@@ -18,7 +20,7 @@ RUN apt -y update -qq \
 
 
 ENV ANDROID_NDK_HOME="${ANDROID_HOME}/android-ndk"
-ENV ANDROID_NDK_VERSION="19b"
+ENV ANDROID_NDK_VERSION="19c"
 ENV ANDROID_NDK_HOME_V="${ANDROID_NDK_HOME}-r${ANDROID_NDK_VERSION}"
 
 # get the latest version from https://developer.android.com/ndk/downloads/index.html
@@ -38,10 +40,11 @@ RUN curl --location --progress-bar \
 ENV ANDROID_SDK_HOME="${ANDROID_HOME}/android-sdk"
 
 # get the latest version from https://developer.android.com/studio/index.html
-ENV ANDROID_SDK_TOOLS_VERSION="4333796"
+ENV ANDROID_SDK_TOOLS_VERSION="6514223"
-ENV ANDROID_SDK_BUILD_TOOLS_VERSION="28.0.3"
+ENV ANDROID_SDK_BUILD_TOOLS_VERSION="29.0.3"
-ENV ANDROID_SDK_TOOLS_ARCHIVE="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
+ENV ANDROID_SDK_TOOLS_ARCHIVE="commandlinetools-linux-${ANDROID_SDK_TOOLS_VERSION}_latest.zip"
 ENV ANDROID_SDK_TOOLS_DL_URL="https://dl.google.com/android/repository/${ANDROID_SDK_TOOLS_ARCHIVE}"
+ENV ANDROID_SDK_MANAGER="${ANDROID_SDK_HOME}/tools/bin/sdkmanager --sdk_root=${ANDROID_SDK_HOME}"
 
 # download and install Android SDK
 RUN curl --location --progress-bar \
@@ -58,15 +61,15 @@ RUN mkdir --parents "${ANDROID_SDK_HOME}/.android/" \
 
 # accept Android licenses (JDK necessary!)
 RUN apt -y update -qq \
-    && apt -y install -qq --no-install-recommends openjdk-8-jdk \
+    && apt -y install -qq --no-install-recommends openjdk-13-jdk \
     && apt -y autoremove
-RUN yes | "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null
+RUN yes | ${ANDROID_SDK_MANAGER} --licenses > /dev/null
 
 # download platforms, API, build tools
-RUN "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-24" > /dev/null && \
+UN ${ANDROID_SDK_MANAGER} "platforms;android-24" > /dev/null && \
-    "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "platforms;android-28" > /dev/null && \
+    ${ANDROID_SDK_MANAGER} "platforms;android-28" > /dev/null && \
-    "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null && \
+    ${ANDROID_SDK_MANAGER} "build-tools;${ANDROID_SDK_BUILD_TOOLS_VERSION}" > /dev/null && \
-    "${ANDROID_SDK_HOME}/tools/bin/sdkmanager" "extras;android;m2repository" > /dev/null && \
+    ${ANDROID_SDK_MANAGER} "extras;android;m2repository" > /dev/null && \
     chmod +x "${ANDROID_SDK_HOME}/tools/bin/avdmanager"
 
 # download ANT
@@ -92,18 +95,10 @@ ENV WORK_DIR="${HOME_DIR}/wspace" \
 # install system dependencies
 RUN apt -y update -qq \
     && apt -y install -qq --no-install-recommends \
-        python3 virtualenv python3-pip python3-setuptools git wget lbzip2 patch sudo \
+    python3 python3-pip python3-setuptools git wget lbzip2 patch sudo \
-        software-properties-common \
+    software-properties-common libssl-dev \
     && apt -y autoremove
 
-# install kivy
-RUN add-apt-repository ppa:kivy-team/kivy \
-    && apt -y update -qq  \
-    && apt -y install -qq --no-install-recommends python3-kivy \
-    && apt -y autoremove \
-    && apt -y clean
-RUN python3 -m pip install image
-
 # build dependencies
 # https://buildozer.readthedocs.io/en/latest/installation.html#android-on-ubuntu-16-04-64bit
 RUN dpkg --add-architecture i386 \
@@ -111,7 +106,7 @@ RUN dpkg --add-architecture i386 \
     && apt -y install -qq --no-install-recommends \
         build-essential ccache git python3 python3-dev \
         libncurses5:i386 libstdc++6:i386 libgtk2.0-0:i386 \
-        libpangox-1.0-0:i386 libpangoxft-1.0-0:i386 libidn11:i386 \
+        libidn11:i386 \
         zip zlib1g-dev zlib1g:i386 \
     && apt -y autoremove \
     && apt -y clean
@@ -140,9 +135,12 @@ RUN chown ${USER} /opt
 USER ${USER}
 
 
-RUN python3 -m pip install --upgrade cython==0.28.6
+RUN python3 -m pip install --user --upgrade pip
-RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install --user --upgrade wheel
-RUN python3 -m pip install --user wheel
+RUN python3 -m pip install --user --upgrade cython==0.29.19
+RUN python3 -m pip install --user --pre kivy
+RUN python3 -m pip install --user image
+
 
 # prepare git
 RUN git config --global user.name "John Doe" \
@@ -154,7 +152,8 @@ RUN cd /opt \
     && cd buildozer \
     && git remote add sombernight https://github.com/SomberNight/buildozer \
     && git fetch --all \
-    && git checkout 7578fea609d4445b3fed1f441813ab4c86ef0086 \
+    # commit: kivy/buildozer "1.2.0" tag
+    && git checkout "94cfcb8d591c11d6ad0e11f129b08c1e27a161c5^{commit}" \
     && python3 -m pip install --user -e .
 
 # install python-for-android
@@ -163,7 +162,8 @@ RUN cd /opt \
     && cd python-for-android \
     && git remote add sombernight https://github.com/SomberNight/python-for-android \
     && git fetch --all \
-    && git checkout 9162ec6b4af464672960f6f9bb7c481af2d01802 \
+    # commit: from branch sombernight/electrum_20200703
+    && git checkout "0dd2ce87a8f380d20505ca5dc1e2d2357b4a08fc^{commit}" \
     && python3 -m pip install --user -e .
 
 # build env vars

electrum/gui/qt/__init__.py

@@ -28,7 +28,7 @@ import signal
 import sys
 import traceback
 import threading
-from typing import Optional, TYPE_CHECKING
+from typing import Optional, TYPE_CHECKING, List
 
 
 try:
@@ -92,6 +92,7 @@ class ElectrumGui(Logger):
     def __init__(self, config: 'SimpleConfig', daemon: 'Daemon', plugins: 'Plugins'):
         set_language(config.get('language', get_default_language()))
         Logger.__init__(self)
+        self.logger.info(f"Qt GUI starting up... Qt={QtCore.QT_VERSION_STR}, PyQt={QtCore.PYQT_VERSION_STR}")
         # Uncomment this call to verify objects are being properly
         # GC-ed when windows are closed
         #network.add_jobs([DebugMem([Abstract_Wallet, SPV, Synchronizer,
@@ -105,7 +106,7 @@ class ElectrumGui(Logger):
         self.config = config
         self.daemon = daemon
         self.plugins = plugins
-        self.windows = []
+        self.windows = []  # type: List[ElectrumWindow]
         self.efilter = OpenFileEventFilter(self.windows)
         self.app = QElectrumApplication(sys.argv)
         self.app.installEventFilter(self.efilter)
@@ -200,12 +201,15 @@ class ElectrumGui(Logger):
             self.lightning_dialog.close()
         if self.watchtower_dialog:
             self.watchtower_dialog.close()
+        self.app.quit()
 
     def new_window(self, path, uri=None):
         # Use a signal as can be called from daemon thread
         self.app.new_window_signal.emit(path, uri)
 
     def show_lightning_dialog(self):
+       if not self.daemon.network.is_lightning_running():
+            return
         if not self.lightning_dialog:
             self.lightning_dialog = LightningDialog(self)
         self.lightning_dialog.bring_to_top()
@@ -300,7 +304,7 @@ class ElectrumGui(Logger):
         return window
 
     def _start_wizard_to_select_or_create_wallet(self, path) -> Optional[Abstract_Wallet]:
-        wizard = InstallWizard(self.config, self.app, self.plugins)
+        wizard = InstallWizard(self.config, self.app, self.plugins, gui_object=self)
         try:
             path, storage = wizard.select_storage(path, self.daemon.get_wallet)
             # storage is None if file does not exist
@@ -339,7 +343,7 @@ class ElectrumGui(Logger):
         # Show network dialog if config does not exist
         if self.daemon.network:
             if self.config.get('auto_connect') is None:
-                wizard = InstallWizard(self.config, self.app, self.plugins)
+                wizard = InstallWizard(self.config, self.app, self.plugins, gui_object=self)
                 wizard.init_network(self.daemon.network)
                 wizard.terminate()
 

electrum/gui/qt/address_dialog.py

@@ -64,7 +64,7 @@ class AddressDialog(WindowModalDialog):
         vbox = QVBoxLayout()
         self.setLayout(vbox)
 
-        vbox.addWidget(QLabel(_("Address:")))
+        vbox.addWidget(QLabel(_("Address") + ":"))
         self.addr_e = ButtonsLineEdit(self.address)
         self.addr_e.addCopyButton(self.app)
         icon = "qrcode_white.png" if ColorScheme.dark_scheme else "qrcode.png"
@@ -98,6 +98,16 @@ class AddressDialog(WindowModalDialog):
             witness_e.addCopyButton(self.app)
             vbox.addWidget(witness_e)
 
+        address_path_str = self.wallet.get_address_path_str(address)
+        if address_path_str:
+            vbox.addWidget(QLabel(_("Derivation path") + ':'))
+            der_path_e = ButtonsLineEdit(address_path_str)
+            der_path_e.addCopyButton(self.app)
+            der_path_e.setReadOnly(True)
+            vbox.addWidget(der_path_e)
+
+
+
         vbox.addWidget(QLabel(_("History")))
         addr_hist_model = AddressHistoryModel(self.parent, self.address)
         self.hw = HistoryList(self.parent, addr_hist_model)

electrum/gui/qt/address_list.py

@@ -1,4 +1,7 @@
-#!/usr/bin/env python
+self.std_model = QStandardItemModel(self)
+self.proxy = MySortModel(self, sort_role=self.ROLE_SORT_ORDER)
+self.proxy.setSourceModel(self.std_model)
+self.setModel(self.proxy)#!/usr/bin/env python
 #
 # Electrum - lightweight Bitcoin client
 # Copyright (C) 2015 Thomas Voegtlin
@@ -35,7 +38,7 @@ from electrum.plugin import run_hook
 from electrum.bitcoin import is_address
 from electrum.wallet import InternalAddressCorruption
 
-from .util import MyTreeView, MONOSPACE_FONT, ColorScheme, webopen
+from .util import MyTreeView, MONOSPACE_FONT, ColorScheme, webopen, MySortModel
 
 
 class AddressUsageStateFilter(IntEnum):
@@ -78,6 +81,8 @@ class AddressList(MyTreeView):
 
     filter_columns = [Columns.TYPE, Columns.ADDRESS, Columns.LABEL, Columns.COIN_BALANCE]
 
+    ROLE_SORT_ORDER = Qt.UserRole + 1000
+
     def __init__(self, parent):
         super().__init__(parent, self.create_menu, stretch_column=self.Columns.LABEL)
         self.wallet = self.parent.wallet
@@ -93,8 +98,12 @@ class AddressList(MyTreeView):
         self.used_button.currentIndexChanged.connect(self.toggle_used)
         for addr_usage_state in AddressUsageStateFilter.__members__.values():  # type: AddressUsageStateFilter
             self.used_button.addItem(addr_usage_state.ui_text())
-        self.setModel(QStandardItemModel(self))
+        self.std_model = QStandardItemModel(self)
+        self.proxy = MySortModel(self, sort_role=self.ROLE_SORT_ORDER)
+        self.proxy.setSourceModel(self.std_model)
+        self.setModel(self.proxy)
         self.update()
+        self.sortByColumn(self.Columns.TYPE, Qt.AscendingOrder)
 
     def get_toolbar_buttons(self):
         return QLabel(_("Filter:")), self.change_button, self.used_button
@@ -146,10 +155,12 @@ class AddressList(MyTreeView):
             addr_list = self.wallet.get_change_addresses()
         else:
             addr_list = self.wallet.get_addresses()
-        self.model().clear()
+        self.proxy.setDynamicSortFilter(False)  # temp. disable re-sorting after every change
+        self.std_model.clear()
         self.refresh_headers()
         fx = self.parent.fx
         set_address = None
+        addresses_beyond_gap_limit = self.wallet.get_all_known_addresses_beyond_gap_limit()
         for address in addr_list:
             num = self.wallet.get_address_history_len(address)
             label = self.wallet.labels.get(address, '')
@@ -186,15 +197,21 @@ class AddressList(MyTreeView):
                 address_item[self.Columns.TYPE].setText(_('receiving'))
                 address_item[self.Columns.TYPE].setBackground(ColorScheme.GREEN.as_color(True))
             address_item[self.Columns.LABEL].setData(address, Qt.UserRole)
+            address_path = self.wallet.get_address_index(address)
+            address_item[self.Columns.TYPE].setData(address_path, self.ROLE_SORT_ORDER)
+            address_path_str = self.wallet.get_address_path_str(address)
+            if address_path_str is not None:
+                address_item[self.Columns.TYPE].setToolTip(address_path_str)
+            address_item[self.Columns.FIAT_BALANCE].setData(balance, self.ROLE_SORT_ORDER)
             # setup column 1
             if self.wallet.is_frozen_address(address):
                 address_item[self.Columns.ADDRESS].setBackground(ColorScheme.BLUE.as_color(True))
-            if self.wallet.is_beyond_limit(address):
+            if address in addresses_beyond_gap_limit:
                 address_item[self.Columns.ADDRESS].setBackground(ColorScheme.RED.as_color(True))
             # add item
-            count = self.model().rowCount()
+            count = self.std_model.rowCount()
-            self.model().insertRow(count, address_item)
+            self.std_model.insertRow(count, address_item)
-            address_idx = self.model().index(count, self.Columns.LABEL)
+            address_idx = self.std_model.index(count, self.Columns.LABEL)
             if address == current_address:
                 set_address = QPersistentModelIndex(address_idx)
         self.set_current_idx(set_address)
@@ -204,6 +221,7 @@ class AddressList(MyTreeView):
         else:
             self.hideColumn(self.Columns.FIAT_BALANCE)
         self.filter()
+        self.proxy.setDynamicSortFilter(True)
 
     def create_menu(self, position):
         from electrum.wallet import Multisig_Wallet
@@ -213,17 +231,17 @@ class AddressList(MyTreeView):
         if not selected:
             return
         multi_select = len(selected) > 1
-        addrs = [self.model().itemFromIndex(item).text() for item in selected]
+        addrs = [self.item_from_index(item).text() for item in selected]
         menu = QMenu()
         if not multi_select:
             idx = self.indexAt(position)
             if not idx.isValid():
                 return
-            item = self.model().itemFromIndex(idx)
+            item = self.item_from_index(idx)
             if not item:
                 return
             addr = addrs[0]
-            addr_column_title = self.model().horizontalHeaderItem(self.Columns.LABEL).text()
+            addr_column_title = self.std_model.horizontalHeaderItem(self.Columns.LABEL).text()
             addr_idx = idx.sibling(idx.row(), self.Columns.LABEL)
             self.add_copy_menu(menu, idx)
             menu.addAction(_('Details'), lambda: self.parent.show_address(addr))
@@ -256,7 +274,7 @@ class AddressList(MyTreeView):
     def place_text_on_clipboard(self, text: str, *, title: str = None) -> None:
         if is_address(text):
             try:
-                self.wallet.check_address(text)
+                self.wallet.check_address_for_corruption(text)
             except InternalAddressCorruption as e:
                 self.parent.show_error(str(e))
                 raise

electrum/gui/qt/amountedit.py

@@ -7,7 +7,7 @@ from PyQt5.QtCore import pyqtSignal, Qt
 from PyQt5.QtGui import QPalette, QPainter
 from PyQt5.QtWidgets import (QLineEdit, QStyle, QStyleOptionFrame)
 
-from .util import char_width_in_lineedit
+from .util import char_width_in_lineedit, ColorScheme
 
 from electrum.util import (format_satoshis_plain, decimal_point_to_base_unit_name,
                            FEERATE_PRECISION, quantize_feerate)
@@ -32,7 +32,6 @@ class AmountEdit(FreezableLineEdit):
         self.textChanged.connect(self.numbify)
         self.is_int = is_int
         self.is_shortcut = False
-        self.help_palette = QPalette()
         self.extra_precision = 0
 
     def decimal_point(self):
@@ -69,7 +68,7 @@ class AmountEdit(FreezableLineEdit):
             textRect = self.style().subElementRect(QStyle.SE_LineEditContents, panel, self)
             textRect.adjust(2, 0, -10, 0)
             painter = QPainter(self)
-            painter.setPen(self.help_palette.brush(QPalette.Disabled, QPalette.Text).color())
+            painter.setPen(ColorScheme.GRAY.as_color())
             painter.drawText(textRect, Qt.AlignRight | Qt.AlignVCenter, self.base_unit())
 
     def get_amount(self) -> Union[None, Decimal, int]:
@@ -106,11 +105,12 @@ class BTCAmountEdit(AmountEdit):
         amount = Decimal(max_prec_amount) / pow(10, self.max_precision()-self.decimal_point())
         return Decimal(amount) if not self.is_int else int(amount)
 
-    def setAmount(self, amount):
+    def setAmount(self, amount_sat):
-        if amount is None:
+        if amount_sat is None:
             self.setText(" ")  # Space forces repaint in case units changed
         else:
-            self.setText(format_satoshis_plain(amount, self.decimal_point()))
+            self.setText(format_satoshis_plain(amount_sat, decimal_point=self.decimal_point()))
+        self.repaint()  # macOS hack for #6269
 
 
 class FeerateEdit(BTCAmountEdit):

electrum/gui/qt/bip39_recovery_dialog.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2020 The Electrum developers
+# Distributed under the MIT software license, see the accompanying
+# file LICENCE or http://www.opensource.org/licenses/mit-license.php
+
+from PyQt5.QtCore import Qt
+from PyQt5.QtWidgets import QWidget, QVBoxLayout, QGridLayout, QLabel, QListWidget, QListWidgetItem
+
+from electrum.i18n import _
+from electrum.network import Network
+from electrum.bip39_recovery import account_discovery
+from electrum.logging import get_logger
+
+from .util import WindowModalDialog, MessageBoxMixin, TaskThread, Buttons, CancelButton, OkButton
+
+
+_logger = get_logger(__name__)
+
+
+class Bip39RecoveryDialog(WindowModalDialog):
+    def __init__(self, parent: QWidget, get_account_xpub, on_account_select):
+        self.get_account_xpub = get_account_xpub
+        self.on_account_select = on_account_select
+        WindowModalDialog.__init__(self, parent, _('BIP39 Recovery'))
+        self.setMinimumWidth(400)
+        vbox = QVBoxLayout(self)
+        self.content = QVBoxLayout()
+        self.content.addWidget(QLabel(_('Scanning common paths for existing accounts...')))
+        vbox.addLayout(self.content)
+        self.ok_button = OkButton(self)
+        self.ok_button.clicked.connect(self.on_ok_button_click)
+        self.ok_button.setEnabled(False)
+        vbox.addLayout(Buttons(CancelButton(self), self.ok_button))
+        self.finished.connect(self.on_finished)
+        self.show()
+        self.thread = TaskThread(self)
+        self.thread.finished.connect(self.deleteLater) # see #3956
+        self.thread.add(self.recovery, self.on_recovery_success, None, self.on_recovery_error)
+
+    def on_finished(self):
+        self.thread.stop()
+
+    def on_ok_button_click(self):
+        item = self.list.currentItem()
+        account = item.data(Qt.UserRole)
+        self.on_account_select(account)
+
+    def recovery(self):
+        network = Network.get_instance()
+        coroutine = account_discovery(network, self.get_account_xpub)
+        return network.run_from_another_thread(coroutine)
+
+    def on_recovery_success(self, accounts):
+        self.clear_content()
+        if len(accounts) == 0:
+            self.content.addWidget(QLabel(_('No existing accounts found.')))
+            return
+        self.content.addWidget(QLabel(_('Choose an account to restore.')))
+        self.list = QListWidget()
+        for account in accounts:
+            item = QListWidgetItem(account['description'])
+            item.setData(Qt.UserRole, account)
+            self.list.addItem(item)
+        self.list.clicked.connect(lambda: self.ok_button.setEnabled(True))
+        self.content.addWidget(self.list)
+
+    def on_recovery_error(self, exc_info):
+        self.clear_content()
+        self.content.addWidget(QLabel(_('Error: Account discovery failed.')))
+        _logger.error(f"recovery error", exc_info=exc_info)
+
+    def clear_content(self):
+        for i in reversed(range(self.content.count())):
+            self.content.itemAt(i).widget().setParent(None)

electrum/gui/qt/channel_details.py

@@ -3,13 +3,18 @@ from typing import TYPE_CHECKING
 import PyQt5.QtGui as QtGui
 import PyQt5.QtWidgets as QtWidgets
 import PyQt5.QtCore as QtCore
+from PyQt5.QtWidgets import QLabel, QLineEdit
 
+from electrum import util
 from electrum.i18n import _
 from electrum.util import bh2u, format_time
 from electrum.lnutil import format_short_channel_id, LOCAL, REMOTE, UpdateAddHtlc, Direction
-from electrum.lnchannel import htlcsum
+from electrum.lnchannel import htlcsum, Channel, AbstractChannel
 from electrum.lnaddr import LnAddr, lndecode
 from electrum.bitcoin import COIN
+from electrum.wallet import Abstract_Wallet
+
+from .util import Buttons, CloseButton, ButtonsLineEdit
 
 if TYPE_CHECKING:
     from .main_window import ElectrumWindow
@@ -32,7 +37,7 @@ class LinkedLabel(QtWidgets.QLabel):
 class ChannelDetailsDialog(QtWidgets.QDialog):
     def make_htlc_item(self, i: UpdateAddHtlc, direction: Direction) -> HTLCItem:
         it = HTLCItem(_('Sent HTLC with ID {}' if Direction.SENT == direction else 'Received HTLC with ID {}').format(i.htlc_id))
-        it.appendRow([HTLCItem(_('Amount')),HTLCItem(self.format(i.amount_msat))])
+        it.appendRow([HTLCItem(_('Amount')),HTLCItem(self.format_msat(i.amount_msat))])
         it.appendRow([HTLCItem(_('CLTV expiry')),HTLCItem(str(i.cltv_expiry))])
         it.appendRow([HTLCItem(_('Payment hash')),HTLCItem(bh2u(i.payment_hash))])
         return it
@@ -41,7 +46,11 @@ class ChannelDetailsDialog(QtWidgets.QDialog):
         model = QtGui.QStandardItemModel(0, 2)
         model.setHorizontalHeaderLabels(['HTLC', 'Property value'])
         parentItem = model.invisibleRootItem()
-        folder_types = {'settled': _('Fulfilled HTLCs'), 'inflight': _('HTLCs in current commitment transaction'), 'failed': _('Failed HTLCs')}
+        folder_types = {
+            'settled': _('Fulfilled HTLCs'),
+            'inflight': _('HTLCs in current commitment transaction'),
+            'failed': _('Failed HTLCs'),
+        }
         self.folders = {}
         self.keyname_rows = {}
 
@@ -54,8 +63,8 @@ class ChannelDetailsDialog(QtWidgets.QDialog):
             self.folders[keyname] = folder
             mapping = {}
             num = 0
-            for pay_hash, item in htlcs.items():
+           for item in htlcs:
-                chan_id, i, direction, status = item
+                pay_hash, chan_id, i, direction, status = item
                 if status != keyname:
                     continue
                 it = self.make_htlc_item(i, direction)
@@ -73,29 +82,49 @@ class ChannelDetailsDialog(QtWidgets.QDialog):
         dest_mapping = self.keyname_rows[to]
         dest_mapping[payment_hash] = len(dest_mapping)
 
-    ln_payment_completed = QtCore.pyqtSignal(str, float, Direction, UpdateAddHtlc, bytes, bytes)
+    ln_payment_completed = QtCore.pyqtSignal(str, bytes, bytes)
-    htlc_added = QtCore.pyqtSignal(str, UpdateAddHtlc, LnAddr, Direction)
+    ln_payment_failed = QtCore.pyqtSignal(str, bytes, bytes)
+    htlc_added = QtCore.pyqtSignal(str, Channel, UpdateAddHtlc, Direction)
+    state_changed = QtCore.pyqtSignal(str, Abstract_Wallet, AbstractChannel)
 
-    @QtCore.pyqtSlot(str, UpdateAddHtlc, LnAddr, Direction)
+    @QtCore.pyqtSlot(str, Abstract_Wallet, AbstractChannel)
-    def do_htlc_added(self, evtname, htlc, lnaddr, direction):
+    def do_state_changed(self, wallet, chan):
+        if wallet != self.wallet:
+            return
+        if chan == self.chan:
+            self.update()
+
+    @QtCore.pyqtSlot(str, Channel, UpdateAddHtlc, Direction)
+    def do_htlc_added(self, evtname, chan, htlc, direction):
+        if chan != self.chan:
+            return
         mapping = self.keyname_rows['inflight']
         mapping[htlc.payment_hash] = len(mapping)
         self.folders['inflight'].appendRow(self.make_htlc_item(htlc, direction))
 
-    @QtCore.pyqtSlot(str, float, Direction, UpdateAddHtlc, bytes, bytes)
+    @QtCore.pyqtSlot(str, bytes, bytes)
-    def do_ln_payment_completed(self, evtname, date, direction, htlc, preimage, chan_id):
+    def do_ln_payment_completed(self, evtname, payment_hash, chan_id):
         if chan_id != self.chan.channel_id:
             return
-        self.move('inflight', 'settled', htlc.payment_hash)
+        self.move('inflight', 'settled', payment_hash)
-        self.update_sent_received()
+        self.update()
 
-    def update_sent_received(self):
+    @QtCore.pyqtSlot(str, bytes, bytes)
-        self.sent_label.setText(str(self.chan.total_msat(Direction.SENT)))
+    def do_ln_payment_failed(self, evtname, payment_hash, chan_id):
-        self.received_label.setText(str(self.chan.total_msat(Direction.RECEIVED)))
+        if chan_id != self.chan.channel_id:
+            return
+        self.move('inflight', 'failed', payment_hash)
+        self.update()
+
+    def update(self):
+        self.can_send_label.setText(self.format_msat(self.chan.available_to_spend(LOCAL)))
+        self.can_receive_label.setText(self.format_msat(self.chan.available_to_spend(REMOTE)))
+        self.sent_label.setText(self.format_msat(self.chan.total_msat(Direction.SENT)))
+        self.received_label.setText(self.format_msat(self.chan.total_msat(Direction.RECEIVED)))
 
     @QtCore.pyqtSlot(str)
     def show_tx(self, link_text: str):
-        funding_tx = self.window.wallet.db.get_transaction(self.chan.funding_outpoint.txid)
+        funding_tx = self.wallet.db.get_transaction(self.chan.funding_outpoint.txid)
         self.window.show_transaction(funding_tx, tx_desc=_('Funding Transaction'))
 
     def __init__(self, window: 'ElectrumWindow', chan_id: bytes):
@@ -103,16 +132,21 @@ class ChannelDetailsDialog(QtWidgets.QDialog):
 
         # initialize instance fields
         self.window = window
+        self.wallet = window.wallet
         chan = self.chan = window.wallet.lnworker.channels[chan_id]
-        self.format = lambda msat: window.format_amount_and_units(msat / 1000)
+        self.format_msat = lambda msat: window.format_amount_and_units(msat / 1000)
 
         # connect signals with slots
         self.ln_payment_completed.connect(self.do_ln_payment_completed)
+        self.ln_payment_failed.connect(self.do_ln_payment_failed)
+        self.state_changed.connect(self.do_state_changed)
         self.htlc_added.connect(self.do_htlc_added)
 
         # register callbacks for updating
-        window.network.register_callback(self.ln_payment_completed.emit, ['ln_payment_completed'])
+        util.register_callback(self.ln_payment_completed.emit, ['ln_payment_completed'])
-        window.network.register_callback(self.htlc_added.emit, ['htlc_added'])
+        util.register_callback(self.ln_payment_failed.emit, ['ln_payment_failed'])
+        util.register_callback(self.htlc_added.emit, ['htlc_added'])
+        util.register_callback(self.state_changed.emit, ['channel'])
 
         # set attributes of QDialog
         self.setWindowTitle(_('Channel Details'))
@@ -120,37 +154,51 @@ class ChannelDetailsDialog(QtWidgets.QDialog):
 
         # add layouts
         vbox = QtWidgets.QVBoxLayout(self)
-        form_layout = QtWidgets.QFormLayout(None)
+        vbox.addWidget(QLabel(_('Remote Node ID:')))
-        vbox.addLayout(form_layout)
+        remote_id_e = ButtonsLineEdit(bh2u(chan.node_id))
-
+        remote_id_e.addCopyButton(self.window.app)
-        # add form content
+        remote_id_e.setReadOnly(True)
-        form_layout.addRow(_('Node ID:'), SelectableLabel(bh2u(chan.node_id)))
+        vbox.addWidget(remote_id_e)
-        form_layout.addRow(_('Channel ID:'), SelectableLabel(bh2u(chan.channel_id)))
         funding_label_text = f'<a href=click_destination>{chan.funding_outpoint.txid}</a>:{chan.funding_outpoint.output_index}'
-        form_layout.addRow(_('Funding Outpoint:'), LinkedLabel(funding_label_text, self.show_tx))
+        vbox.addWidget(QLabel(_('Funding Outpoint:')))
-        form_layout.addRow(_('Short Channel ID:'), SelectableLabel(format_short_channel_id(chan.short_channel_id)))
+        vbox.addWidget(LinkedLabel(funding_label_text, self.show_tx))
+
+        form_layout = QtWidgets.QFormLayout(None)
+        # add form content
+        form_layout.addRow(_('Channel ID:'), SelectableLabel(f"{chan.channel_id.hex()} (Short: {chan.short_channel_id})"))
+        form_layout.addRow(_('State:'), SelectableLabel(chan.get_state_for_GUI()))
+        self.initiator = 'Local' if chan.constraints.is_initiator else 'Remote'
+        form_layout.addRow(_('Initiator:'), SelectableLabel(self.initiator))
+        self.capacity = self.window.format_amount_and_units(chan.constraints.capacity)
+        form_layout.addRow(_('Capacity:'), SelectableLabel(self.capacity))
+        self.can_send_label = SelectableLabel()
+        self.can_receive_label = SelectableLabel()
+        form_layout.addRow(_('Can send:'), self.can_send_label)
+        form_layout.addRow(_('Can receive:'), self.can_receive_label)
         self.received_label = SelectableLabel()
-        form_layout.addRow(_('Received (mSAT):'), self.received_label)
+        form_layout.addRow(_('Received:'), self.received_label)
         self.sent_label = SelectableLabel()
-        form_layout.addRow(_('Sent (mSAT):'), self.sent_label)
+        form_layout.addRow(_('Sent:'), self.sent_label)
-        self.htlc_minimum_msat = SelectableLabel(str(chan.config[REMOTE].htlc_minimum_msat))
+        #self.htlc_minimum_msat = SelectableLabel(str(chan.config[REMOTE].htlc_minimum_msat))
-        form_layout.addRow(_('Minimum HTLC value accepted by peer (mSAT):'), self.htlc_minimum_msat)
+        #form_layout.addRow(_('Minimum HTLC value accepted by peer (mSAT):'), self.htlc_minimum_msat)
-        self.max_htlcs = SelectableLabel(str(chan.config[REMOTE].max_accepted_htlcs))
+        #self.max_htlcs = SelectableLabel(str(chan.config[REMOTE].max_accepted_htlcs))
-        form_layout.addRow(_('Maximum number of concurrent HTLCs accepted by peer:'), self.max_htlcs)
+        #form_layout.addRow(_('Maximum number of concurrent HTLCs accepted by peer:'), self.max_htlcs)
-        self.max_htlc_value = SelectableLabel(self.window.format_amount_and_units(chan.config[REMOTE].max_htlc_value_in_flight_msat / 1000))
+        #self.max_htlc_value = SelectableLabel(self.window.format_amount_and_units(chan.config[REMOTE].max_htlc_value_in_flight_msat / 1000))
-        form_layout.addRow(_('Maximum value of in-flight HTLCs accepted by peer:'), self.max_htlc_value)
+        #form_layout.addRow(_('Maximum value of in-flight HTLCs accepted by peer:'), self.max_htlc_value)
         self.dust_limit = SelectableLabel(self.window.format_amount_and_units(chan.config[REMOTE].dust_limit_sat))
         form_layout.addRow(_('Remote dust limit:'), self.dust_limit)
-        self.reserve = SelectableLabel(self.window.format_amount_and_units(chan.config[REMOTE].reserve_sat))
+        self.remote_reserve = self.window.format_amount_and_units(chan.config[REMOTE].reserve_sat)
-        form_layout.addRow(_('Remote channel reserve:'), self.reserve)
+        form_layout.addRow(_('Remote reserve:'), SelectableLabel(self.remote_reserve))
+        vbox.addLayout(form_layout)
 
         # add htlc tree view to vbox (wouldn't scale correctly in QFormLayout)
-        form_layout.addRow(_('Payments (HTLCs):'), None)
+        vbox.addWidget(QLabel(_('Payments (HTLCs):')))
         w = QtWidgets.QTreeView(self)
         htlc_dict = chan.get_payments()
         w.setModel(self.make_model(htlc_dict))
         w.header().setSectionResizeMode(0, QtWidgets.QHeaderView.ResizeToContents)
         vbox.addWidget(w)
 
+        vbox.addLayout(Buttons(CloseButton(self)))
         # initialize sent/received fields
-        self.update_sent_received()
+        self.update()

electrum/gui/qt/channels_list.py

@@ -1,20 +1,24 @@
 # -*- coding: utf-8 -*-
 import traceback
 from enum import IntEnum
+from typing import Sequence, Optional
 
 from PyQt5 import QtCore, QtGui
 from PyQt5.QtCore import Qt
-from PyQt5.QtWidgets import QMenu, QHBoxLayout, QLabel, QVBoxLayout, QGridLayout, QLineEdit
+from PyQt5.QtWidgets import (QMenu, QHBoxLayout, QLabel, QVBoxLayout, QGridLayout, QLineEdit,
+                             QPushButton, QAbstractItemView)
+from PyQt5.QtGui import QFont, QStandardItem, QBrush
 
 from electrum.util import bh2u, NotEnoughFunds, NoDynamicFeeEstimates
 from electrum.i18n import _
-from electrum.lnchannel import Channel
+from electrum.lnchannel import AbstractChannel, PeerState
 from electrum.wallet import Abstract_Wallet
 from electrum.lnutil import LOCAL, REMOTE, format_short_channel_id, LN_MAX_FUNDING_SAT
+from electrum.lnworker import LNWallet
 
-from .util import MyTreeView, WindowModalDialog, Buttons, OkButton, CancelButton, EnterButton, WaitingDialog
+from .util import (MyTreeView, WindowModalDialog, Buttons, OkButton, CancelButton,
+                   EnterButton, WaitingDialog, MONOSPACE_FONT, ColorScheme)
 from .amountedit import BTCAmountEdit, FreezableLineEdit
-from .channel_details import ChannelDetailsDialog
 
 
 ROLE_CHANNEL_ID = Qt.UserRole
@@ -22,32 +26,47 @@ ROLE_CHANNEL_ID = Qt.UserRole
 
 class ChannelsList(MyTreeView):
     update_rows = QtCore.pyqtSignal(Abstract_Wallet)
-    update_single_row = QtCore.pyqtSignal(Channel)
+    update_single_row = QtCore.pyqtSignal(Abstract_Wallet, AbstractChannel)
+    gossip_db_loaded = QtCore.pyqtSignal()
 
     class Columns(IntEnum):
         SHORT_CHANID = 0
-        NODE_ID = 1
+        NODE_ALIAS = 1
         LOCAL_BALANCE = 2
         REMOTE_BALANCE = 3
         CHANNEL_STATUS = 4
 
     headers = {
         Columns.SHORT_CHANID: _('Short Channel ID'),
-        Columns.NODE_ID: _('Node ID'),
+        Columns.NODE_ALIAS: _('Node alias'),
         Columns.LOCAL_BALANCE: _('Local'),
         Columns.REMOTE_BALANCE: _('Remote'),
         Columns.CHANNEL_STATUS: _('Status'),
     }
 
+    filter_columns = [
+        Columns.SHORT_CHANID,
+        Columns.NODE_ALIAS,
+        Columns.CHANNEL_STATUS,
+    ]
+
+    _default_item_bg_brush = None  # type: Optional[QBrush]
+
+
+
     def __init__(self, parent):
-        super().__init__(parent, self.create_menu, stretch_column=self.Columns.NODE_ID,
+        super().__init__(parent, self.create_menu, stretch_column=self.Columns.NODE_ALIAS,
                          editable_columns=[])
         self.setModel(QtGui.QStandardItemModel(self))
+        self.setSelectionMode(QAbstractItemView.ExtendedSelection)
         self.main_window = parent
+        self.gossip_db_loaded.connect(self.on_gossip_db)
         self.update_rows.connect(self.do_update_rows)
         self.update_single_row.connect(self.do_update_single_row)
         self.network = self.parent.network
         self.lnworker = self.parent.wallet.lnworker
+        self.lnbackups = self.parent.wallet.lnbackups
+        self.setSortingEnabled(True)
 
     def format_fields(self, chan):
         labels = {}
@@ -60,12 +79,18 @@ class ChannelsList(MyTreeView):
             if bal_other != bal_minus_htlcs_other:
                 label += ' (+' + self.parent.format_amount(bal_other - bal_minus_htlcs_other) + ')'
             labels[subject] = label
-        status = self.lnworker.get_channel_status(chan)
+        status = chan.get_state_for_GUI()
+        closed = chan.is_closed()
+        if self.parent.network.is_lightning_running():
+            node_info = self.parent.network.channel_db.get_node_info_for_node_id(chan.node_id)
+            node_alias = (node_info.alias if node_info else '') or ''
+        else:
+            node_alias = ''
         return [
-            format_short_channel_id(chan.short_channel_id),
+            chan.short_id_for_GUI(),
-            bh2u(chan.node_id),
+            node_alias,
-            labels[LOCAL],
+            '' if closed else labels[LOCAL],
-            labels[REMOTE],
+            '' if closed else labels[REMOTE],
             status
         ]
 
@@ -78,70 +103,200 @@ class ChannelsList(MyTreeView):
         self.main_window.show_error('Failed to close channel:\n{}'.format(repr(e)))
 
     def close_channel(self, channel_id):
+        msg = _('Close channel?')
+        if not self.parent.question(msg):
+            return
         def task():
             coro = self.lnworker.close_channel(channel_id)
             return self.network.run_from_another_thread(coro)
         WaitingDialog(self, 'please wait..', task, self.on_success, self.on_failure)
 
     def force_close(self, channel_id):
+        chan = self.lnworker.channels[channel_id]
+        to_self_delay = chan.config[REMOTE].to_self_delay
+        msg = _('Force-close channel?') + '\n\n'\
+              + _('Funds retrieved from this channel will not be available before {} blocks after forced closure.').format(to_self_delay) + ' '\
+              + _('After that delay, funds will be sent to an address derived from your wallet seed.') + '\n\n'\
+              + _('In the meantime, channel funds will not be recoverable from your seed, and might be lost if you lose your wallet.') + ' '\
+              + _('To prevent that, you should have a backup of this channel on another device.')
+        if self.parent.question(msg):
             def task():
                 coro = self.lnworker.force_close_channel(channel_id)
                 return self.network.run_from_another_thread(coro)
-        if self.parent.question('Force-close channel?\nReclaimed funds will not be immediately available.'):
             WaitingDialog(self, 'please wait..', task, self.on_success, self.on_failure)
 
     def remove_channel(self, channel_id):
         if self.main_window.question(_('Are you sure you want to delete this channel? This will purge associated transactions from your wallet history.')):
             self.lnworker.remove_channel(channel_id)
 
+    def remove_channel_backup(self, channel_id):
+        if self.main_window.question(_('Remove channel backup?')):
+            self.lnbackups.remove_channel_backup(channel_id)
+
+    def export_channel_backup(self, channel_id):
+        msg = ' '.join([
+            _("Channel backups can be imported in another instance of the same wallet, by scanning this QR code."),
+            _("Please note that channel backups cannot be used to restore your channels."),
+            _("If you lose your wallet file, the only thing you can do with a backup is to request your channel to be closed, so that your funds will be sent on-chain."),
+        ])
+        data = self.lnworker.export_channel_backup(channel_id)
+        self.main_window.show_qrcode(data, 'channel backup', help_text=msg,
+                                     show_copy_text_btn=True)
+
+    def request_force_close(self, channel_id):
+        def task():
+            coro = self.lnbackups.request_force_close(channel_id)
+            return self.network.run_from_another_thread(coro)
+        def on_success(b):
+            self.main_window.show_message('success')
+        WaitingDialog(self, 'please wait..', task, on_success, self.on_failure)
+
+
+
     def create_menu(self, position):
         menu = QMenu()
-        idx = self.selectionModel().currentIndex()
+        menu.setSeparatorsCollapsible(True)  # consecutive separators are merged together
+        selected = self.selected_in_column(self.Columns.NODE_ALIAS)
+        if not selected:
+            menu.addAction(_("Import channel backup"), lambda: self.parent.do_process_from_text_channel_backup())
+            menu.exec_(self.viewport().mapToGlobal(position))
+            return
+        multi_select = len(selected) > 1
+        if multi_select:
+            return
+        idx = self.indexAt(position)
+        if not idx.isValid():
+            return
         item = self.model().itemFromIndex(idx)
         if not item:
             return
-        channel_id = idx.sibling(idx.row(), self.Columns.NODE_ID).data(ROLE_CHANNEL_ID)
+        channel_id = idx.sibling(idx.row(), self.Columns.NODE_ALIAS).data(ROLE_CHANNEL_ID)
+        if channel_id in self.lnbackups.channel_backups:
+            menu.addAction(_("Request force-close"), lambda: self.request_force_close(channel_id))
+            menu.addAction(_("Delete"), lambda: self.remove_channel_backup(channel_id))
+            menu.exec_(self.viewport().mapToGlobal(position))
+            return
         chan = self.lnworker.channels[channel_id]
-        menu.addAction(_("Details..."), lambda: self.details(channel_id))
+        menu.addAction(_("Details..."), lambda: self.parent.show_channel(channel_id))
-        self.add_copy_menu(menu, idx)
+        cc = self.add_copy_menu(menu, idx)
+        cc.addAction(_("Node ID"), lambda: self.place_text_on_clipboard(
+            chan.node_id.hex(), title=_("Node ID")))
+        cc.addAction(_("Long Channel ID"), lambda: self.place_text_on_clipboard(
+            channel_id.hex(), title=_("Long Channel ID")))
         if not chan.is_closed():
+            if not chan.is_frozen_for_sending():
+                menu.addAction(_("Freeze (for sending)"), lambda: chan.set_frozen_for_sending(True))
+            else:
+                menu.addAction(_("Unfreeze (for sending)"), lambda: chan.set_frozen_for_sending(False))
+            if not chan.is_frozen_for_receiving():
+                menu.addAction(_("Freeze (for receiving)"), lambda: chan.set_frozen_for_receiving(True))
+            else:
+                menu.addAction(_("Unfreeze (for receiving)"), lambda: chan.set_frozen_for_receiving(False))
+
+        funding_tx = self.parent.wallet.db.get_transaction(chan.funding_outpoint.txid)
+        if funding_tx:
+            menu.addAction(_("View funding transaction"), lambda: self.parent.show_transaction(funding_tx))
+        if not chan.is_closed():
+            menu.addSeparator()
+            if chan.peer_state == PeerState.GOOD:
                 menu.addAction(_("Close channel"), lambda: self.close_channel(channel_id))
             menu.addAction(_("Force-close channel"), lambda: self.force_close(channel_id))
         else:
-            menu.addAction(_("Remove"), lambda: self.remove_channel(channel_id))
+            item = chan.get_closing_height()
+            if item:
+                txid, height, timestamp = item
+                closing_tx = self.lnworker.lnwatcher.db.get_transaction(txid)
+                if closing_tx:
+                    menu.addAction(_("View closing transaction"), lambda: self.parent.show_transaction(closing_tx))
+        menu.addSeparator()
+        menu.addAction(_("Export backup"), lambda: self.export_channel_backup(channel_id))
+        if chan.is_redeemed():
+            menu.addSeparator()
+            menu.addAction(_("Delete"), lambda: self.remove_channel(channel_id))
         menu.exec_(self.viewport().mapToGlobal(position))
 
-    def details(self, channel_id):
+    @QtCore.pyqtSlot(Abstract_Wallet, AbstractChannel)
-        assert self.parent.wallet
+    def do_update_single_row(self, wallet: Abstract_Wallet, chan: AbstractChannel):
-        ChannelDetailsDialog(self.parent, channel_id).show()
+        if wallet != self.parent.wallet:
-
+            return
-    @QtCore.pyqtSlot(Channel)
-    def do_update_single_row(self, chan):
         for row in range(self.model().rowCount()):
-            item = self.model().item(row, self.Columns.NODE_ID)
+            item = self.model().item(row, self.Columns.NODE_ALIAS)
-            if item.data(ROLE_CHANNEL_ID) == chan.channel_id:
+            if item.data(ROLE_CHANNEL_ID) != chan.channel_id:
+                continue
             for column, v in enumerate(self.format_fields(chan)):
                 self.model().item(row, column).setData(v, QtCore.Qt.DisplayRole)
+            items = [self.model().item(row, column) for column in self.Columns]
+            self._update_chan_frozen_bg(chan=chan, items=items)
+        if wallet.lnworker:
+            self.update_can_send(wallet.lnworker)
+
+    @QtCore.pyqtSlot()
+    def on_gossip_db(self):
+        self.do_update_rows(self.parent.wallet)
 
     @QtCore.pyqtSlot(Abstract_Wallet)
     def do_update_rows(self, wallet):
         if wallet != self.parent.wallet:
             return
-        lnworker = self.parent.wallet.lnworker
+        channels = list(wallet.lnworker.channels.values()) if wallet.lnworker else []
-        if not lnworker:
+        backups = list(wallet.lnbackups.channel_backups.values())
-            return
+        if wallet.lnworker:
+            self.update_can_send(wallet.lnworker)
         self.model().clear()
         self.update_headers(self.headers)
-        for chan in lnworker.channels.values():
+        for chan in channels + backups:
             items = [QtGui.QStandardItem(x) for x in self.format_fields(chan)]
             self.set_editability(items)
-            items[self.Columns.NODE_ID].setData(chan.channel_id, ROLE_CHANNEL_ID)
+            if self._default_item_bg_brush is None:
+                self._default_item_bg_brush = items[self.Columns.NODE_ALIAS].background()
+            items[self.Columns.NODE_ALIAS].setData(chan.channel_id, ROLE_CHANNEL_ID)
+            items[self.Columns.NODE_ALIAS].setFont(QFont(MONOSPACE_FONT))
+            items[self.Columns.LOCAL_BALANCE].setFont(QFont(MONOSPACE_FONT))
+            items[self.Columns.REMOTE_BALANCE].setFont(QFont(MONOSPACE_FONT))
+            self._update_chan_frozen_bg(chan=chan, items=items)
             self.model().insertRow(0, items)
 
+        self.sortByColumn(self.Columns.SHORT_CHANID, Qt.DescendingOrder)
+
+    def _update_chan_frozen_bg(self, *, chan: AbstractChannel, items: Sequence[QStandardItem]):
+        assert self._default_item_bg_brush is not None
+        # frozen for sending
+        item = items[self.Columns.LOCAL_BALANCE]
+        if chan.is_frozen_for_sending():
+            item.setBackground(ColorScheme.BLUE.as_color(True))
+            item.setToolTip(_("This channel is frozen for sending. It will not be used for outgoing payments."))
+        else:
+            item.setBackground(self._default_item_bg_brush)
+            item.setToolTip("")
+        # frozen for receiving
+        item = items[self.Columns.REMOTE_BALANCE]
+        if chan.is_frozen_for_receiving():
+            item.setBackground(ColorScheme.BLUE.as_color(True))
+            item.setToolTip(_("This channel is frozen for receiving. It will not be included in invoices."))
+        else:
+            item.setBackground(self._default_item_bg_brush)
+            item.setToolTip("")
+
+    def update_can_send(self, lnworker: LNWallet):
+        msg = _('Can send') + ' ' + self.parent.format_amount(lnworker.num_sats_can_send())\
+              + ' ' + self.parent.base_unit() + '; '\
+              + _('can receive') + ' ' + self.parent.format_amount(lnworker.num_sats_can_receive())\
+              + ' ' + self.parent.base_unit()
+        self.can_send_label.setText(msg)
+
+
+
     def get_toolbar(self):
         h = QHBoxLayout()
+        self.can_send_label = QLabel('')
+        h.addWidget(self.can_send_label)
         h.addStretch()
-        h.addWidget(EnterButton(_('Open Channel'), self.new_channel_dialog))
+        self.swap_button = EnterButton(_('Swap'), self.swap_dialog)
+        self.swap_button.setEnabled(self.parent.wallet.has_lightning())
+        self.new_channel_button = EnterButton(_('Open Channel'), self.new_channel_dialog)
+        self.new_channel_button.setEnabled(self.parent.wallet.has_lightning())
+        h.addWidget(self.new_channel_button)
+        h.addWidget(self.swap_button)
         return h
 
 
@@ -194,24 +349,35 @@ class ChannelsList(MyTreeView):
         max_button = EnterButton(_("Max"), spend_max)
         max_button.setFixedWidth(100)
         max_button.setCheckable(True)
+        suggest_button = QPushButton(d, text=_('Suggest'))
+        def on_suggest():
+            remote_nodeid.setText(bh2u(lnworker.suggest_peer() or b''))
+            remote_nodeid.repaint()  # macOS hack for #6269
+        suggest_button.clicked.connect(on_suggest)
+        clear_button = QPushButton(d, text=_('Clear'))
+        def on_clear():
+            amount_e.setText('')
+            amount_e.setFrozen(False)
+            amount_e.repaint()  # macOS hack for #6269
+            remote_nodeid.setText('')
+            remote_nodeid.repaint()  # macOS hack for #6269
+            max_button.setChecked(False)
+            max_button.repaint()  # macOS hack for #6269
+        clear_button.clicked.connect(on_clear)
         h = QGridLayout()
         h.addWidget(QLabel(_('Your Node ID')), 0, 0)
-        h.addWidget(local_nodeid, 0, 1)
+        h.addWidget(local_nodeid, 0, 1, 1, 3)
         h.addWidget(QLabel(_('Remote Node ID')), 1, 0)
-        h.addWidget(remote_nodeid, 1, 1)
+        h.addWidget(remote_nodeid, 1, 1, 1, 3)
-        h.addWidget(QLabel('Amount'), 2, 0)
+        h.addWidget(suggest_button, 2, 1)
-        hbox = QHBoxLayout()
+        h.addWidget(clear_button, 2, 2)
-        hbox.addWidget(amount_e)
+        h.addWidget(QLabel('Amount'), 3, 0)
-        hbox.addWidget(max_button)
+        h.addWidget(amount_e, 3, 1)
-        hbox.addStretch(1)
+        h.addWidget(max_button, 3, 2)
-        h.addLayout(hbox, 2, 1)
         vbox.addLayout(h)
         ok_button = OkButton(d)
         ok_button.setDefault(True)
         vbox.addLayout(Buttons(CancelButton(d), ok_button))
-        suggestion = lnworker.suggest_peer() or b''
-        remote_nodeid.setText(bh2u(suggestion))
-        remote_nodeid.setCursorPosition(0)
         if not d.exec_():
             return
         if max_button.isChecked() and amount_e.get_amount() < LN_MAX_FUNDING_SAT:
@@ -222,4 +388,12 @@ class ChannelsList(MyTreeView):
         else:
             funding_sat = amount_e.get_amount()
         connect_str = str(remote_nodeid.text()).strip()
+        if not connect_str or not funding_sat:
+            return
         self.parent.open_channel(connect_str, funding_sat, 0)
+
+
+    def swap_dialog(self):
+        from .swap_dialog import SwapDialog
+        d = SwapDialog(self.parent)
+        d.run()

electrum/gui/qt/confirm_tx_dialog.py

@@ -23,6 +23,7 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from decimal import Decimal
 from typing import TYPE_CHECKING, Optional, Union
 
 from PyQt5.QtWidgets import  QVBoxLayout, QLabel, QGridLayout, QPushButton, QLineEdit
@@ -31,12 +32,13 @@ from electrum.i18n import _
 from electrum.util import NotEnoughFunds, NoDynamicFeeEstimates
 from electrum.plugin import run_hook
 from electrum.transaction import Transaction, PartialTransaction
-from electrum.simple_config import FEERATE_WARNING_HIGH_FEE
+from electrum.simple_config import FEERATE_WARNING_HIGH_FEE, FEE_RATIO_HIGH_WARNING
 from electrum.wallet import InternalAddressCorruption
 
-from .util import WindowModalDialog, ColorScheme, HelpLabel, Buttons, CancelButton, BlockingWaitingDialog
+from .util import (WindowModalDialog, ColorScheme, HelpLabel, Buttons, CancelButton,
+                   BlockingWaitingDialog, PasswordLineEdit)
 
-from .fee_slider import FeeSlider
+from .fee_slider import FeeSlider, FeeComboBox
 
 if TYPE_CHECKING:
     from .main_window import ElectrumWindow
@@ -78,7 +80,7 @@ class TxEditor:
     def get_fee_estimator(self):
         return None
 
-    def update_tx(self):
+    def update_tx(self, *, fallback_to_zero_fee: bool = False):
         fee_estimator = self.get_fee_estimator()
         try:
             self.tx = self.make_tx(fee_estimator)
@@ -87,6 +89,12 @@ class TxEditor:
         except NotEnoughFunds:
             self.not_enough_funds = True
             self.tx = None
+            if fallback_to_zero_fee:
+                try:
+                    self.tx = self.make_tx(0)
+                except BaseException:
+                    return
+            else:
                 return
         except NoDynamicFeeEstimates:
             self.no_dynfee_estimates = True
@@ -103,7 +111,13 @@ class TxEditor:
         if use_rbf:
             self.tx.set_rbf(True)
 
-
+    def have_enough_funds_assuming_zero_fees(self) -> bool:
+        try:
+            tx = self.make_tx(0)
+        except NotEnoughFunds:
+            return False
+        else:
+            return True
 
 
 
@@ -138,14 +152,16 @@ class ConfirmTxDialog(TxEditor, WindowModalDialog):
         grid.addWidget(self.extra_fee_value, 2, 1)
 
         self.fee_slider = FeeSlider(self, self.config, self.fee_slider_callback)
+        self.fee_combo = FeeComboBox(self.fee_slider)
+        grid.addWidget(HelpLabel(_("Fee rate") + ": ", self.fee_combo.help_msg), 5, 0)
         grid.addWidget(self.fee_slider, 5, 1)
+        grid.addWidget(self.fee_combo, 5, 2)
 
         self.message_label = QLabel(self.default_message())
         grid.addWidget(self.message_label, 6, 0, 1, -1)
         self.pw_label = QLabel(_('Password'))
         self.pw_label.setVisible(self.password_required)
-        self.pw = QLineEdit()
+        self.pw = PasswordLineEdit()
-        self.pw.setEchoMode(2)
         self.pw.setVisible(self.password_required)
         grid.addWidget(self.pw_label, 8, 0)
         grid.addWidget(self.pw, 8, 1, 1, -1)
@@ -175,6 +191,7 @@ class ConfirmTxDialog(TxEditor, WindowModalDialog):
         password = self.pw.text() or None
         if self.password_required:
             if password is None:
+                self.main_window.show_error(_("Password required"), parent=self)
                 return
             try:
                 self.wallet.check_password(password)
@@ -184,22 +201,32 @@ class ConfirmTxDialog(TxEditor, WindowModalDialog):
         self.is_send = True
         self.accept()
 
-    def disable(self, reason):
+    def toggle_send_button(self, enable: bool, *, message: str = None):
-        self.message_label.setStyleSheet(ColorScheme.RED.as_stylesheet())
+        if message is None:
-        self.message_label.setText(reason)
-        self.pw.setEnabled(False)
-        self.send_button.setEnabled(False)
-
-    def enable(self):
             self.message_label.setStyleSheet(None)
             self.message_label.setText(self.default_message())
-        self.pw.setEnabled(True)
+        else:
-        self.send_button.setEnabled(True)
+            self.message_label.setStyleSheet(ColorScheme.RED.as_stylesheet())
+            self.message_label.setText(message)
+        self.pw.setEnabled(enable)
+        self.send_button.setEnabled(enable)
+
+    def _update_amount_label(self):
+        tx = self.tx
+        if self.output_value == '!':
+            if tx:
+                amount = tx.output_value()
+                amount_str = self.main_window.format_amount_and_units(amount)
+            else:
+                amount_str = "max"
+        else:
+            amount = self.output_value
+            amount_str = self.main_window.format_amount_and_units(amount)
+        self.amount_label.setText(amount_str)
 
     def update(self):
         tx = self.tx
-        amount = tx.output_value() if self.output_value == '!' else self.output_value
+        self._update_amount_label()
-        self.amount_label.setText(self.main_window.format_amount_and_units(amount))
 
         if self.not_enough_funds:
             text = _("Not enough funds")
@@ -208,7 +235,7 @@ class ConfirmTxDialog(TxEditor, WindowModalDialog):
                 text += " ({} {} {})".format(
                     self.main_window.format_amount(c + u + x).strip(), self.main_window.base_unit(), _("are frozen")
                 )
-            self.disable(text)
+            self.toggle_send_button(False, message=text)
             return
 
         if not tx:
@@ -223,16 +250,22 @@ class ConfirmTxDialog(TxEditor, WindowModalDialog):
             self.extra_fee_value.setVisible(True)
             self.extra_fee_value.setText(self.main_window.format_amount_and_units(x_fee_amount))
 
-        feerate_warning = FEERATE_WARNING_HIGH_FEE
+        amount = tx.output_value() if self.output_value == '!' else self.output_value
-        low_fee = fee < self.wallet.relayfee() * tx.estimated_size() / 1000
+        feerate = Decimal(fee) / tx.estimated_size()  # sat/byte
-        high_fee = fee > feerate_warning * tx.estimated_size() / 1000
+        fee_ratio = Decimal(fee) / amount if amount else 1
-        if low_fee:
+        if feerate < self.wallet.relayfee() / 1000:
             msg = '\n'.join([
                 _("This transaction requires a higher fee, or it will not be propagated by your current server"),
                 _("Try to raise your transaction fee, or use a server with a lower relay fee.")
             ])
-            self.disable(msg)
+            self.toggle_send_button(False, message=msg)
-        elif high_fee:
+        elif fee_ratio >= FEE_RATIO_HIGH_WARNING:
-            self.disable(_('Warning') + ': ' + _("The fee for this transaction seems unusually high."))
+            self.toggle_send_button(True,
+                                    message=_('Warning') + ': ' + _("The fee for this transaction seems unusually high.")
+                                            + f'\n({fee_ratio*100:.2f}% of amount)')
+        elif feerate > FEERATE_WARNING_HIGH_FEE / 1000:
+            self.toggle_send_button(True,
+                                    message=_('Warning') + ': ' + _("The fee for this transaction seems unusually high.")
+                                            + f'\n(feerate: {feerate:.2f} sat/byte)')
         else:
-            self.enable()
+            self.toggle_send_button(True)

electrum/gui/qt/console.py

@@ -44,7 +44,7 @@ class OverlayLabel(QtWidgets.QLabel):
 
 
 class Console(QtWidgets.QPlainTextEdit):
-    def __init__(self, prompt='>> ', startup_message='', parent=None):
+    def __init__(self, prompt='>>> ', parent=None):
         QtWidgets.QPlainTextEdit.__init__(self, parent)
 
         self.prompt = prompt
@@ -56,7 +56,6 @@ class Console(QtWidgets.QPlainTextEdit):
         self.setWordWrapMode(QtGui.QTextOption.WrapAnywhere)
         self.setUndoRedoEnabled(False)
         self.document().setDefaultFont(QtGui.QFont(MONOSPACE_FONT, 10, QtGui.QFont.Normal))
-        self.showMessage(startup_message)
 
         self.updateNamespace({'run':self.run_script})
         self.set_json(False)
@@ -91,13 +90,14 @@ class Console(QtWidgets.QPlainTextEdit):
 
     def showMessage(self, message):
         self.appendPlainText(message)
-        self.newPrompt()
+        self.newPrompt('')
 
     def clear(self):
+        curr_line = self.getCommand()
         self.setPlainText('')
-        self.newPrompt()
+        self.newPrompt(curr_line)
 
-    def newPrompt(self):
+    def newPrompt(self, curr_line):
         if self.construct:
             prompt = '.' * len(self.prompt)
         else:
@@ -178,7 +178,7 @@ class Console(QtWidgets.QPlainTextEdit):
     def getHistory(self):
         return self.history
 
-    def setHisory(self, history):
+    def setHistory(self, history):
         self.history = history
 
     def addToHistory(self, command):
@@ -244,7 +244,7 @@ class Console(QtWidgets.QPlainTextEdit):
             if type(self.namespace.get(command)) == type(lambda:None):
                 self.appendPlainText("'{}' is a function. Type '{}()' to use it in the Python console."
                                      .format(command, command))
-                self.newPrompt()
+                self.newPrompt('')
                 return
 
             sys.stdout = stdoutProxy(self.appendPlainText)
@@ -269,7 +269,7 @@ class Console(QtWidgets.QPlainTextEdit):
                     traceback_lines.pop(i)
                 self.appendPlainText('\n'.join(traceback_lines))
             sys.stdout = tmp_stdout
-        self.newPrompt()
+        self.newPrompt('')
         self.set_json(False)
 
 
@@ -346,17 +346,3 @@ class Console(QtWidgets.QPlainTextEdit):
                 self.setCommand(beginning + p)
             else:
                 self.show_completions(completions)
-
-
-welcome_message = '''
-   ---------------------------------------------------------------
-     Welcome to a primitive Python interpreter.
-   ---------------------------------------------------------------
-'''
-
-if __name__ == '__main__':
-    app = QtWidgets.QApplication(sys.argv)
-    console = Console(startup_message=welcome_message)
-    console.updateNamespace({'myVar1' : app, 'myVar2' : 1234})
-    console.show()
-    sys.exit(app.exec_())

electrum/gui/qt/contact_list.py

@@ -34,7 +34,7 @@ from electrum.bitcoin import is_address
 from electrum.util import block_explorer_URL
 from electrum.plugin import run_hook
 
-from .util import MyTreeView, import_meta_gui, export_meta_gui, webopen
+from .util import MyTreeView, webopen
 
 
 class ContactList(MyTreeView):
@@ -63,12 +63,6 @@ class ContactList(MyTreeView):
         self.parent.set_contact(text, user_role)
         self.update()
 
-    def import_contacts(self):
-        import_meta_gui(self.parent, _('contacts'), self.parent.contacts.import_file, self.update)
-
-    def export_contacts(self):
-        export_meta_gui(self.parent, _('contacts'), self.parent.contacts.export_file)
-
     def create_menu(self, position):
         menu = QMenu()
         idx = self.indexAt(position)

electrum/gui/qt/custom_model.py

@@ -0,0 +1,94 @@
+# loosely based on
+# http://trevorius.com/scrapbook/uncategorized/pyqt-custom-abstractitemmodel/
+
+from PyQt5 import QtCore, QtWidgets
+
+class CustomNode:
+
+    def __init__(self, model, data):
+        self.model = model
+        self._data = data
+        self._children = []
+        self._parent = None
+        self._row = 0
+
+    def get_data(self):
+        return self._data
+
+    def get_data_for_role(self, index, role):
+        # define in child class
+        raise NotImplementedError()
+
+    def childCount(self):
+        return len(self._children)
+
+    def child(self, row):
+        if row >= 0 and row < self.childCount():
+            return self._children[row]
+
+    def parent(self):
+        return self._parent
+
+    def row(self):
+        return self._row
+
+    def addChild(self, child):
+        child._parent = self
+        child._row = len(self._children)
+        self._children.append(child)
+
+
+
+class CustomModel(QtCore.QAbstractItemModel):
+
+    def __init__(self, parent, columncount):
+        QtCore.QAbstractItemModel.__init__(self, parent)
+        self._root = CustomNode(self, None)
+        self._columncount = columncount
+
+    def rowCount(self, index):
+        if index.isValid():
+            return index.internalPointer().childCount()
+        return self._root.childCount()
+
+    def columnCount(self, index):
+        return self._columncount
+
+    def addChild(self, node, _parent):
+        if not _parent or not _parent.isValid():
+            parent = self._root
+        else:
+            parent = _parent.internalPointer()
+        parent.addChild(self, node)
+
+    def index(self, row, column, _parent=None):
+        if not _parent or not _parent.isValid():
+            parent = self._root
+        else:
+            parent = _parent.internalPointer()
+
+        if not QtCore.QAbstractItemModel.hasIndex(self, row, column, _parent):
+            return QtCore.QModelIndex()
+
+        child = parent.child(row)
+        if child:
+            return QtCore.QAbstractItemModel.createIndex(self, row, column, child)
+        else:
+            return QtCore.QModelIndex()
+
+    def parent(self, index):
+        if index.isValid():
+            node = index.internalPointer()
+            if node:
+                p = node.parent()
+                if p:
+                    return QtCore.QAbstractItemModel.createIndex(self, p.row(), 0, p)
+            else:
+                return QtCore.QModelIndex()
+        return QtCore.QModelIndex()
+
+    def data(self, index, role):
+        if not index.isValid():
+            return None
+        node = index.internalPointer()
+        return node.get_data_for_role(index, role)

electrum/gui/qt/exception_window.py

@@ -22,6 +22,8 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 import sys
+import html
+from typing import TYPE_CHECKING, Optional, Set
 
 from PyQt5.QtCore import QObject
 import PyQt5.QtCore as QtCore
@@ -32,16 +34,24 @@ from electrum.i18n import _
 from electrum.base_crash_reporter import BaseCrashReporter
 from electrum.logging import Logger
 from electrum import constants
+from electrum.network import Network
 
 from .util import MessageBoxMixin, read_QIcon, WaitingDialog
 
+if TYPE_CHECKING:
+    from electrum.simple_config import SimpleConfig
+    from electrum.wallet import Abstract_Wallet
+
+
+
 
 class Exception_Window(BaseCrashReporter, QWidget, MessageBoxMixin, Logger):
     _active_window = None
 
-    def __init__(self, main_window, exctype, value, tb):
+    def __init__(self, config: 'SimpleConfig', exctype, value, tb):
         BaseCrashReporter.__init__(self, exctype, value, tb)
-        self.main_window = main_window
+        self.network = Network.get_instance()
+        self.config = config
 
         QWidget.__init__(self)
         self.setWindowTitle('Electrum - ' + _('An Error Occurred'))
@@ -109,13 +119,13 @@ class Exception_Window(BaseCrashReporter, QWidget, MessageBoxMixin, Logger):
             self.logger.error('There was a problem with the automatic reporting', exc_info=exc_info)
             self.show_critical(parent=self,
                                msg=(_('There was a problem with the automatic reporting:') + '<br/>' +
-                                    repr(e)[:120] + '<br/>' +
+                                    repr(e)[:120] + '<br/><br/>' +
                                     _("Please report this issue manually") +
                                     f' <a href="{constants.GIT_REPO_ISSUES_URL}">on GitHub</a>.'),
                                rich_text=True)
 
-        proxy = self.main_window.network.proxy
+        proxy = self.network.proxy
-        task = lambda: BaseCrashReporter.send_report(self, self.main_window.network.asyncio_loop, proxy)
+        task = lambda: BaseCrashReporter.send_report(self, self.network.asyncio_loop, proxy)
         msg = _('Sending crash report...')
         WaitingDialog(self, msg, task, on_success, on_failure)
 
@@ -124,7 +134,7 @@ class Exception_Window(BaseCrashReporter, QWidget, MessageBoxMixin, Logger):
         self.close()
 
     def show_never(self):
-        self.main_window.config.set_key(BaseCrashReporter.config_key, False)
+        self.config.set_key(BaseCrashReporter.config_key, False)
         self.close()
 
     def closeEvent(self, event):
@@ -135,7 +145,15 @@ class Exception_Window(BaseCrashReporter, QWidget, MessageBoxMixin, Logger):
         return self.description_textfield.toPlainText()
 
     def get_wallet_type(self):
-        return self.main_window.wallet.wallet_type
+        wallet_types = Exception_Hook._INSTANCE.wallet_types_seen
+        return ",".join(wallet_types)
+
+    def _get_traceback_str(self) -> str:
+        # The msg_box that shows the report uses rich_text=True, so
+        # if traceback contains special HTML characters, e.g. '<',
+        # they need to be escaped to avoid formatting issues.
+        traceback_str = super()._get_traceback_str()
+        return html.escape(traceback_str)
 
 
 def _show_window(*args):
@@ -146,15 +164,29 @@ def _show_window(*args):
 class Exception_Hook(QObject, Logger):
     _report_exception = QtCore.pyqtSignal(object, object, object, object)
 
-    def __init__(self, main_window, *args, **kwargs):
+    _INSTANCE = None  # type: Optional[Exception_Hook]  # singleton
-        QObject.__init__(self, *args, **kwargs)
+
+    def __init__(self, *, config: 'SimpleConfig'):
+        QObject.__init__(self)
         Logger.__init__(self)
-        if not main_window.config.get(BaseCrashReporter.config_key, default=True):
+        assert self._INSTANCE is None, "Exception_Hook is supposed to be a singleton"
-            return
+        self.config = config
-        self.main_window = main_window
+        self.wallet_types_seen = set()  # type: Set[str]
+
+
         sys.excepthook = self.handler
         self._report_exception.connect(_show_window)
 
+            @classmethod
+    def maybe_setup(cls, *, config: 'SimpleConfig', wallet: 'Abstract_Wallet') -> None:
+        if not config.get(BaseCrashReporter.config_key, default=True):
+            return
+        if not cls._INSTANCE:
+            cls._INSTANCE = Exception_Hook(config=config)
+        cls._INSTANCE.wallet_types_seen.add(wallet.wallet_type)
+
+
+
     def handler(self, *exc_info):
         self.logger.error('exception caught by crash reporter', exc_info=exc_info)
-        self._report_exception.emit(self.main_window, *exc_info)
+        self._report_exception.emit(self.config, *exc_info)

electrum/gui/qt/fee_slider.py

@@ -2,10 +2,32 @@ import threading
 
 from PyQt5.QtGui import QCursor
 from PyQt5.QtCore import Qt
-from PyQt5.QtWidgets import QSlider, QToolTip
+from PyQt5.QtWidgets import QSlider, QToolTip, QComboBox
 
 from electrum.i18n import _
 
+class FeeComboBox(QComboBox):
+
+    def __init__(self, fee_slider):
+        QComboBox.__init__(self)
+        self.config = fee_slider.config
+        self.fee_slider = fee_slider
+        self.addItems([_('Static'), _('ETA'), _('Mempool')])
+        self.setCurrentIndex((2 if self.config.use_mempool_fees() else 1) if self.config.is_dynfee() else 0)
+        self.currentIndexChanged.connect(self.on_fee_type)
+        self.help_msg = '\n'.join([
+            _('Static: the fee slider uses static values'),
+            _('ETA: fee rate is based on average confirmation time estimates'),
+            _('Mempool based: fee rate is targeting a depth in the memory pool')
+            ]
+        )
+
+    def on_fee_type(self, x):
+        self.config.set_key('mempool_fees', x==2)
+        self.config.set_key('dynamic_fees', x>0)
+        self.fee_slider.update()
+
+
 
 class FeeSlider(QSlider):
 

electrum/gui/qt/history_list.py

@@ -43,9 +43,10 @@ from electrum.address_synchronizer import TX_HEIGHT_LOCAL, TX_HEIGHT_FUTURE
 from electrum.i18n import _
 from electrum.util import (block_explorer_URL, profiler, TxMinedInfo,
                            OrderedDictWithIndex, timestamp_to_datetime,
-                           Satoshis, format_time)
+                           Satoshis, Fiat, format_time)
 from electrum.logging import get_logger, Logger
 
+from .custom_model import CustomNode, CustomModel
 from .util import (read_QIcon, MONOSPACE_FONT, Buttons, CancelButton, OkButton,
                    filename_field, MyTreeView, AcceptFileDragDrop, WindowModalDialog,
                    CloseButton, webopen)
@@ -106,42 +107,19 @@ class HistorySortModel(QSortFilterProxyModel):
 def get_item_key(tx_item):
     return tx_item.get('txid') or tx_item['payment_hash']
 
-class HistoryModel(QAbstractItemModel, Logger):
+class HistoryNode(CustomNode):
 
-    def __init__(self, parent: 'ElectrumWindow'):
+    def get_data_for_role(self, index: QModelIndex, role: Qt.ItemDataRole) -> QVariant:
-        QAbstractItemModel.__init__(self, parent)
-        Logger.__init__(self)
-        self.parent = parent
-        self.view = None  # type: HistoryList
-        self.transactions = OrderedDictWithIndex()
-        self.tx_status_cache = {}  # type: Dict[str, Tuple[int, str]]
-
-    def set_view(self, history_list: 'HistoryList'):
-        # FIXME HistoryModel and HistoryList mutually depend on each other.
-        # After constructing both, this method needs to be called.
-        self.view = history_list  # type: HistoryList
-        self.set_visibility_of_columns()
-
-    def columnCount(self, parent: QModelIndex):
-        return len(HistoryColumns)
-
-    def rowCount(self, parent: QModelIndex):
-        return len(self.transactions)
-
-    def index(self, row: int, column: int, parent: QModelIndex):
-        return self.createIndex(row, column)
-
-    def data(self, index: QModelIndex, role: Qt.ItemDataRole) -> QVariant:
         # note: this method is performance-critical.
         # it is called a lot, and so must run extremely fast.
         assert index.isValid()
         col = index.column()
-        tx_item = self.transactions.value_from_pos(index.row())
+        window = self.model.parent
+        tx_item = self.get_data()
         is_lightning = tx_item.get('lightning', False)
         timestamp = tx_item['timestamp']
         if is_lightning:
             status = 0
-            txpos = tx_item['txpos']
             if timestamp is None:
                 status_str = 'unconfirmed'
             else:
@@ -149,33 +127,25 @@ class HistoryModel(QAbstractItemModel, Logger):
         else:
             tx_hash = tx_item['txid']
             conf = tx_item['confirmations']
-            txpos = tx_item['txpos_in_block'] or 0
-            height = tx_item['height']
             try:
-                status, status_str = self.tx_status_cache[tx_hash]
+                status, status_str = self.model.tx_status_cache[tx_hash]
             except KeyError:
-                tx_mined_info = self.tx_mined_info_from_tx_item(tx_item)
+                tx_mined_info = self.model.tx_mined_info_from_tx_item(tx_item)
-                status, status_str = self.parent.wallet.get_tx_status(tx_hash, tx_mined_info)
+                status, status_str = window.wallet.get_tx_status(tx_hash, tx_mined_info)
-
-        # we sort by timestamp
-        if timestamp is None:
-            timestamp = float("inf")
 
         if role == Qt.UserRole:
             # for sorting
             d = {
                 HistoryColumns.STATUS:
-                    # height breaks ties for unverified txns
+                    # respect sort order of self.transactions (wallet.get_full_history)
-                    # txpos breaks ties for verified same block txns
+                    -index.row(),
-                    (-timestamp, conf, -status, -height, -txpos) if not is_lightning else (-timestamp, 0,0,0,-txpos),
                 HistoryColumns.DESCRIPTION:
                     tx_item['label'] if 'label' in tx_item else None,
                 HistoryColumns.AMOUNT:
                     (tx_item['bc_value'].value if 'bc_value' in tx_item else 0)\
                     + (tx_item['ln_value'].value if 'ln_value' in tx_item else 0),
                 HistoryColumns.BALANCE:
-                    (tx_item['balance'].value if 'balance' in tx_item else 0)\
+                    (tx_item['balance'].value if 'balance' in tx_item else 0),
-                    + (tx_item['balance_msat']//1000 if 'balance_msat'in tx_item else 0),
                 HistoryColumns.FIAT_VALUE:
                     tx_item['fiat_value'].value if 'fiat_value' in tx_item else None,
                 HistoryColumns.FIAT_ACQ_PRICE:
@@ -190,11 +160,20 @@ class HistoryModel(QAbstractItemModel, Logger):
                 icon = "lightning" if is_lightning else TX_ICONS[status]
                 return QVariant(read_QIcon(icon))
             elif col == HistoryColumns.STATUS and role == Qt.ToolTipRole:
-                msg = 'lightning transaction' if is_lightning else str(conf) + _(" confirmation" + ("s" if conf != 1 else ""))
+                if is_lightning:
+                    msg = 'lightning transaction'
+                else:  # on-chain
+                    if tx_item['height'] == TX_HEIGHT_LOCAL:
+                        # note: should we also explain double-spends?
+                        msg = _("This transaction is only available on your local machine.\n"
+                                "The currently connected server does not know about it.\n"
+                                "You can either broadcast it now, or simply remove it.")
+                    else:
+                        msg = str(conf) + _(" confirmation" + ("s" if conf != 1 else ""))
                 return QVariant(msg)
             elif col > HistoryColumns.DESCRIPTION and role == Qt.TextAlignmentRole:
                 return QVariant(Qt.AlignRight | Qt.AlignVCenter)
-            elif col != HistoryColumns.STATUS and role == Qt.FontRole:
+            elif col > HistoryColumns.DESCRIPTION and role == Qt.FontRole:
                 monospace_font = QFont(MONOSPACE_FONT)
                 return QVariant(monospace_font)
             #elif col == HistoryColumns.DESCRIPTION and role == Qt.DecorationRole and not is_lightning\
@@ -217,37 +196,47 @@ class HistoryModel(QAbstractItemModel, Logger):
             bc_value = tx_item['bc_value'].value if 'bc_value' in tx_item else 0
             ln_value = tx_item['ln_value'].value if 'ln_value' in tx_item else 0
             value = bc_value + ln_value
-            v_str = self.parent.format_amount(value, is_diff=True, whitespaces=True)
+            v_str = window.format_amount(value, is_diff=True, whitespaces=True)
             return QVariant(v_str)
         elif col == HistoryColumns.BALANCE:
             balance = tx_item['balance'].value
-            balance_str = self.parent.format_amount(balance, whitespaces=True)
+            balance_str = window.format_amount(balance, whitespaces=True)
             return QVariant(balance_str)
         elif col == HistoryColumns.FIAT_VALUE and 'fiat_value' in tx_item:
-            value_str = self.parent.fx.format_fiat(tx_item['fiat_value'].value)
+            value_str = window.fx.format_fiat(tx_item['fiat_value'].value)
             return QVariant(value_str)
         elif col == HistoryColumns.FIAT_ACQ_PRICE and \
                 tx_item['value'].value < 0 and 'acquisition_price' in tx_item:
             # fixme: should use is_mine
             acq = tx_item['acquisition_price'].value
-            return QVariant(self.parent.fx.format_fiat(acq))
+            return QVariant(window.fx.format_fiat(acq))
         elif col == HistoryColumns.FIAT_CAP_GAINS and 'capital_gain' in tx_item:
             cg = tx_item['capital_gain'].value
-            return QVariant(self.parent.fx.format_fiat(cg))
+            return QVariant(window.fx.format_fiat(cg))
         elif col == HistoryColumns.TXID:
             return QVariant(tx_hash) if not is_lightning else QVariant('')
         return QVariant()
 
-    def parent(self, index: QModelIndex):
+class HistoryModel(CustomModel, Logger):
-        return QModelIndex()
 
-    def hasChildren(self, index: QModelIndex):
+    def __init__(self, parent: 'ElectrumWindow'):
-        return not index.isValid()
+        CustomModel.__init__(self, parent, len(HistoryColumns))
+        Logger.__init__(self)
+        self.parent = parent
+        self.view = None  # type: HistoryList
+        self.transactions = OrderedDictWithIndex()
+        self.tx_status_cache = {}  # type: Dict[str, Tuple[int, str]]
 
-    def update_label(self, row):
+    def set_view(self, history_list: 'HistoryList'):
-        tx_item = self.transactions.value_from_pos(row)
+        # FIXME HistoryModel and HistoryList mutually depend on each other.
+        # After constructing both, this method needs to be called.
+        self.view = history_list  # type: HistoryList
+        self.set_visibility_of_columns()
+
+    def update_label(self, index):
+        tx_item = index.internalPointer().get_data()
         tx_item['label'] = self.parent.wallet.get_label(get_item_key(tx_item))
-        topLeft = bottomRight = self.createIndex(row, HistoryColumns.DESCRIPTION)
+        topLeft = bottomRight = self.createIndex(index.row(), HistoryColumns.DESCRIPTION)
         self.dataChanged.emit(topLeft, bottomRight, [Qt.DisplayRole])
         self.parent.utxo_list.update()
 
@@ -274,19 +263,64 @@ class HistoryModel(QAbstractItemModel, Logger):
         if fx: fx.history_used_spot = False
         wallet = self.parent.wallet
         self.set_visibility_of_columns()
-        transactions = wallet.get_full_history(self.parent.fx,
+        transactions = wallet.get_full_history(
+            self.parent.fx,
             onchain_domain=self.get_domain(),
             include_lightning=self.should_include_lightning_payments())
         if transactions == list(self.transactions.values()):
             return
-        old_length = len(self.transactions)
+        old_length = self._root.childCount()
         if old_length != 0:
             self.beginRemoveRows(QModelIndex(), 0, old_length)
             self.transactions.clear()
+            self._root = HistoryNode(self, None)
             self.endRemoveRows()
-        self.beginInsertRows(QModelIndex(), 0, len(transactions)-1)
+        parents = {}
+        for tx_item in transactions.values():
+            node = HistoryNode(self, tx_item)
+            group_id = tx_item.get('group_id')
+            if group_id is None:
+                self._root.addChild(node)
+            else:
+                parent = parents.get(group_id)
+                if parent is None:
+                    # create parent if it does not exist
+                    self._root.addChild(node)
+                    parents[group_id] = node
+                else:
+                    # if parent has no children, create two children
+                    if parent.childCount() == 0:
+                        child_data = dict(parent.get_data())
+                        node1 = HistoryNode(self, child_data)
+                        parent.addChild(node1)
+                        parent._data['label'] = child_data.get('group_label')
+                        parent._data['bc_value'] = child_data.get('bc_value', Satoshis(0))
+                        parent._data['ln_value'] = child_data.get('ln_value', Satoshis(0))
+                    # add child to parent
+                    parent.addChild(node)
+                    # update parent data
+                    parent._data['balance'] = tx_item['balance']
+                    parent._data['value'] += tx_item['value']
+                    if 'group_label' in tx_item:
+                        parent._data['label'] = tx_item['group_label']
+                    if 'bc_value' in tx_item:
+                        parent._data['bc_value'] += tx_item['bc_value']
+                    if 'ln_value' in tx_item:
+                        parent._data['ln_value'] += tx_item['ln_value']
+                    if 'fiat_value' in tx_item:
+                        parent._data['fiat_value'] += tx_item['fiat_value']
+                    if tx_item.get('txid') == group_id:
+                        parent._data['lightning'] = False
+                        parent._data['txid'] = tx_item['txid']
+                        parent._data['timestamp'] = tx_item['timestamp']
+                        parent._data['height'] = tx_item['height']
+                        parent._data['confirmations'] = tx_item['confirmations']
+
+        new_length = self._root.childCount()
+        self.beginInsertRows(QModelIndex(), 0, new_length-1)
         self.transactions = transactions
         self.endInsertRows()
+
         if selected_row:
             self.view.selectionModel().select(self.createIndex(selected_row, 0), QItemSelectionModel.Rows | QItemSelectionModel.SelectCurrent)
         self.view.filter()
@@ -318,8 +352,8 @@ class HistoryModel(QAbstractItemModel, Logger):
         set_visible(HistoryColumns.FIAT_ACQ_PRICE, history and cap_gains)
         set_visible(HistoryColumns.FIAT_CAP_GAINS, history and cap_gains)
 
-    def update_fiat(self, row, idx):
+    def update_fiat(self, idx):
-        tx_item = self.transactions.value_from_pos(row)
+        tx_item = idx.internalPointer().get_data()
         key = tx_item['txid']
         fee = tx_item.get('fee')
         value = tx_item['value'].value
@@ -398,7 +432,7 @@ class HistoryList(MyTreeView, AcceptFileDragDrop):
 
     def tx_item_from_proxy_row(self, proxy_row):
         hm_idx = self.model().mapToSource(self.model().index(proxy_row, 0))
-        return self.hm.transactions.value_from_pos(hm_idx.row())
+        return hm_idx.internalPointer().get_data()
 
     def should_hide(self, proxy_row):
         if self.start_timestamp and self.end_timestamp:
@@ -426,7 +460,7 @@ class HistoryList(MyTreeView, AcceptFileDragDrop):
         self.wallet = self.parent.wallet  # type: Abstract_Wallet
         self.sortByColumn(HistoryColumns.STATUS, Qt.AscendingOrder)
         self.editable_columns |= {HistoryColumns.FIAT_VALUE}
-
+        self.setRootIsDecorated(True)
         self.header().setStretchLastSection(False)
         for col in HistoryColumns:
             sm = QHeaderView.Stretch if col == self.stretch_column else QHeaderView.ResizeToContents
@@ -562,18 +596,18 @@ class HistoryList(MyTreeView, AcceptFileDragDrop):
 
     def on_edited(self, index, user_role, text):
         index = self.model().mapToSource(index)
-        row, column = index.row(), index.column()
+        tx_item = index.internalPointer().get_data()
-        tx_item = self.hm.transactions.value_from_pos(row)
+        column = index.column()
         key = get_item_key(tx_item)
         if column == HistoryColumns.DESCRIPTION:
             if self.wallet.set_label(key, text): #changed
-                self.hm.update_label(row)
+                self.hm.update_label(index)
                 self.parent.update_completions()
         elif column == HistoryColumns.FIAT_VALUE:
             self.wallet.set_fiat_value(key, self.parent.fx.ccy, text, self.parent.fx, tx_item['value'].value)
             value = tx_item['value'].value
             if value is not None:
-                self.hm.update_fiat(row, index)
+                self.hm.update_fiat(index)
         else:
             assert False
 
@@ -585,12 +619,15 @@ class HistoryList(MyTreeView, AcceptFileDragDrop):
         if self.hm.flags(self.model().mapToSource(idx)) & Qt.ItemIsEditable:
             super().mouseDoubleClickEvent(event)
         else:
-            self.show_transaction(tx_item)
-
-    def show_transaction(self, tx_item):
             if tx_item.get('lightning'):
+                if tx_item['type'] == 'payment':
+                    self.parent.show_lightning_transaction(tx_item)
                 return
             tx_hash = tx_item['txid']
+            self.show_transaction(tx_item, tx)
+
+    def show_transaction(self, tx_item, tx):
+        tx_hash = tx_item['txid']
         tx = self.wallet.db.get_transaction(tx_hash)
         if not tx:
             return
@@ -605,9 +642,11 @@ class HistoryList(MyTreeView, AcceptFileDragDrop):
             column_title = self.hm.headerData(column, Qt.Horizontal, Qt.DisplayRole)
             idx2 = idx.sibling(idx.row(), column)
             column_data = (self.hm.data(idx2, Qt.DisplayRole).value() or '').strip()
-            cc.addAction(column_title,
+            cc.addAction(
+                column_title,
                 lambda text=column_data, title=column_title:
                 self.place_text_on_clipboard(text, title=title))
+        return cc
 
     def create_menu(self, position: QPoint):
         org_idx: QModelIndex = self.indexAt(position)
@@ -615,48 +654,61 @@ class HistoryList(MyTreeView, AcceptFileDragDrop):
         if not idx.isValid():
             # can happen e.g. before list is populated for the first time
             return
-        tx_item = self.hm.transactions.value_from_pos(idx.row())
+        tx_item = idx.internalPointer().get_data()
-        if tx_item.get('lightning'):
+        if tx_item.get('lightning') and tx_item['type'] == 'payment':
+            menu = QMenu()
+            menu.addAction(_("View Payment"), lambda: self.parent.show_lightning_transaction(tx_item))
+            cc = self.add_copy_menu(menu, idx)
+            cc.addAction(_("Payment Hash"), lambda: self.place_text_on_clipboard(tx_item['payment_hash'], title="Payment Hash"))
+            cc.addAction(_("Preimage"), lambda: self.place_text_on_clipboard(tx_item['preimage'], title="Preimage"))
+            menu.exec_(self.viewport().mapToGlobal(position))
             return
         tx_hash = tx_item['txid']
         tx = self.wallet.db.get_transaction(tx_hash)
+        if tx_item.get('lightning'):
+            tx = self.wallet.lnworker.lnwatcher.db.get_transaction(tx_hash)
+        else:
+            tx = self.wallet.db.get_transaction(tx_hash)
         if not tx:
             return
         tx_URL = block_explorer_URL(self.config, 'tx', tx_hash)
-        height = self.wallet.get_tx_height(tx_hash).height
+        tx_details = self.wallet.get_tx_info(tx)
-        is_relevant, is_mine, v, fee = self.wallet.get_wallet_delta(tx)
+        is_unconfirmed = tx_details.tx_mined_status.height <= 0
-        is_unconfirmed = height <= 0
+        invoice_keys = self.wallet.get_relevant_invoice_keys_for_tx(tx)
-        invoice_keys = self.wallet._get_relevant_invoice_keys_for_tx(tx)
         menu = QMenu()
-        if height in [TX_HEIGHT_FUTURE, TX_HEIGHT_LOCAL]:
+        if tx_details.can_remove:
             menu.addAction(_("Remove"), lambda: self.remove_local_tx(tx_hash))
-        menu.addAction(_("Copy Transaction ID"), lambda: self.place_text_on_clipboard(tx_hash, title="TXID"))
+        cc = self.add_copy_menu(menu, idx)
-        self.add_copy_menu(menu, idx)
+        cc.addAction(_("Transaction ID"), lambda: self.place_text_on_clipboard(tx_hash, title="TXID"))
         for c in self.editable_columns:
             if self.isColumnHidden(c): continue
             label = self.hm.headerData(c, Qt.Horizontal, Qt.DisplayRole)
             # TODO use siblingAtColumn when min Qt version is >=5.11
             persistent = QPersistentModelIndex(org_idx.sibling(org_idx.row(), c))
             menu.addAction(_("Edit {}").format(label), lambda p=persistent: self.edit(QModelIndex(p)))
-        menu.addAction(_("Details"), lambda: self.show_transaction(tx_item))
+        menu.addAction(_("View Transaction"), lambda: self.show_transaction(tx_item, tx))
+        channel_id = tx_item.get('channel_id')
+        if channel_id:
+            menu.addAction(_("View Channel"), lambda: self.parent.show_channel(bytes.fromhex(channel_id)))
         if is_unconfirmed and tx:
             # note: the current implementation of RBF *needs* the old tx fee
-            rbf = is_mine and not tx.is_final() and fee is not None
+            if tx_details.can_bump and tx_details.fee is not None:
-            if rbf:
                 menu.addAction(_("Increase fee"), lambda: self.parent.bump_fee_dialog(tx))
             else:
                 child_tx = self.wallet.cpfp(tx, 0)
                 if child_tx:
                     menu.addAction(_("Child pays for parent"), lambda: self.parent.cpfp(tx, child_tx))
-        if invoice_keys:
+        for key in invoice_keys:
-           menu.addAction(read_QIcon("seal"), _("View invoice"), lambda: [self.parent.show_invoice(key) for key in invoice_keys])
+            invoice = self.parent.wallet.get_invoice(key)
+            if invoice:
+                menu.addAction(_("View invoice"), lambda: self.parent.show_onchain_invoice(invoice))
         if tx_URL:
             menu.addAction(_("View on block explorer"), lambda: webopen(tx_URL))
         menu.exec_(self.viewport().mapToGlobal(position))
 
-    def remove_local_tx(self, delete_tx):
+    def remove_local_tx(self, tx_hash: str):
-        to_delete = {delete_tx}
+        to_delete = {tx_hash}
-        to_delete |= self.wallet.get_depending_transactions(delete_tx)
+        to_delete |= self.wallet.get_depending_transactions(tx_hash)
         question = _("Are you sure you want to remove this transaction?")
         if len(to_delete) > 1:
             question = (_("Are you sure you want to remove this transaction and {} child transactions?")
@@ -739,7 +791,7 @@ class HistoryList(MyTreeView, AcceptFileDragDrop):
                 from electrum.util import json_encode
                 f.write(json_encode(txns))
 
-    def text_txid_from_coordinate(self, row, col):
+    def get_text_and_userrole_from_coordinate(self, row, col):
         idx = self.model().mapToSource(self.model().index(row, col))
-        tx_item = self.hm.transactions.value_from_pos(idx.row())
+        tx_item = idx.internalPointer().get_data()
         return self.hm.data(idx, Qt.DisplayRole).value(), get_item_key(tx_item)

electrum/gui/qt/installwizard.py

@@ -19,18 +19,22 @@ from PyQt5.QtWidgets import (QWidget, QDialog, QLabel, QHBoxLayout, QMessageBox,
 from electrum.wallet import Wallet, Abstract_Wallet
 from electrum.storage import WalletStorage, StorageReadWriteError
 from electrum.util import UserCancelled, InvalidPassword, WalletFileException, get_new_wallet_name
-from electrum.base_wizard import BaseWizard, HWD_SETUP_DECRYPT_WALLET, GoBack
+from electrum.base_wizard import BaseWizard, HWD_SETUP_DECRYPT_WALLET, GoBack, ReRunDialog
+from electrum.network import Network
 from electrum.i18n import _
 
 from .seed_dialog import SeedLayout, KeysLayout
 from .network_dialog import NetworkChoiceLayout
 from .util import (MessageBoxMixin, Buttons, icon_path, ChoicesLayout, WWLabel,
-                   InfoButton, char_width_in_lineedit)
+                   InfoButton, char_width_in_lineedit, PasswordLineEdit)
 from .password_dialog import PasswordLayout, PasswordLayoutForHW, PW_NEW
+from .bip39_recovery_dialog import Bip39RecoveryDialog
 from electrum.plugin import run_hook, Plugins
 
 if TYPE_CHECKING:
     from electrum.simple_config import SimpleConfig
+    from electrum.wallet_db import WalletDB
+    from . import ElectrumGui
 
 
 MSG_ENTER_PASSWORD = _("Choose a password to encrypt your wallet keys.") + '\n'\
@@ -94,19 +98,41 @@ def wizard_dialog(func):
     def func_wrapper(*args, **kwargs):
         run_next = kwargs['run_next']
         wizard = args[0]  # type: InstallWizard
+        while True:
+            #wizard.logger.debug(f"dialog stack. len: {len(wizard._stack)}. stack: {wizard._stack}")
             wizard.back_button.setText(_('Back') if wizard.can_go_back() else _('Cancel'))
+            # current dialog
             try:
                 out = func(*args, **kwargs)
                 if type(out) is not tuple:
                     out = (out,)
-            run_next(*out)
             except GoBack:
-            if wizard.can_go_back():
+                if not wizard.can_go_back():
-                wizard.go_back()
-                return
-            else:
                     wizard.close()
+                # to go back from the current dialog, we just let the caller unroll the stack:
                 raise
+            # next dialog
+            try:
+                while True:
+                    try:
+                        run_next(*out)
+                    except ReRunDialog:
+                        # restore state, and then let the loop re-run next
+                        wizard.go_back(rerun_previous=False)
+                    else:
+                        break
+            except GoBack as e:
+                # to go back from the next dialog, we ask the wizard to restore state
+                wizard.go_back(rerun_previous=False)
+                # and we re-run the current dialog
+                if wizard.can_go_back():
+                    # also rerun any calculations that might have populated the inputs to the current dialog,
+                    # by going back to just after the *previous* dialog finished
+                    raise ReRunDialog() from e
+                else:
+                    continue
+            else:
+                break
     return func_wrapper
 
 
@@ -121,12 +147,13 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
 
     accept_signal = pyqtSignal()
 
-    def __init__(self, config: 'SimpleConfig', app: QApplication, plugins: 'Plugins'):
+    def __init__(self, config: 'SimpleConfig', app: QApplication, plugins: 'Plugins', *, gui_object: 'ElectrumGui'):
         QDialog.__init__(self, None)
         BaseWizard.__init__(self, config, plugins)
         self.setWindowTitle('LBRY Vault  -  ' + _('Install Wizard'))
         self.app = app
         self.config = config
+        self.gui_thread = gui_object.gui_thread
         self.setMinimumSize(600, 400)
         self.accept_signal.connect(self.accept)
         self.title = QLabel()
@@ -176,21 +203,20 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         vbox = QVBoxLayout()
         hbox = QHBoxLayout()
         hbox.addWidget(QLabel(_('Wallet') + ':'))
-        self.name_e = QLineEdit()
+        name_e = QLineEdit()
-        hbox.addWidget(self.name_e)
+        hbox.addWidget(name_e)
         button = QPushButton(_('Choose...'))
         hbox.addWidget(button)
         vbox.addLayout(hbox)
 
-        self.msg_label = WWLabel('')
+        msg_label = WWLabel('')
-        vbox.addWidget(self.msg_label)
+        vbox.addWidget(msg_label)
         hbox2 = QHBoxLayout()
-        self.pw_e = QLineEdit('', self)
+        pw_e = PasswordLineEdit('', self)
-        self.pw_e.setFixedWidth(17 * char_width_in_lineedit())
+        pw_e.setFixedWidth(17 * char_width_in_lineedit())
-        self.pw_e.setEchoMode(2)
+        pw_label = QLabel(_('Password') + ':')
-        self.pw_label = QLabel(_('Password') + ':')
+        hbox2.addWidget(pw_label)
-        hbox2.addWidget(self.pw_label)
+        hbox2.addWidget(pw_e)
-        hbox2.addWidget(self.pw_e)
         hbox2.addStretch()
         vbox.addLayout(hbox2)
 
@@ -213,7 +239,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         def on_choose():
             path, __ = QFileDialog.getOpenFileName(self, "Select your wallet file", wallet_folder)
             if path:
-                self.name_e.setText(path)
+                name_e.setText(path)
 
         def on_filename(filename):
             # FIXME? "filename" might contain ".." (etc) and hence sketchy path traversals are possible
@@ -253,27 +279,28 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
                         + _("Press 'Next' to create/focus window.")
             if msg is None:
                 msg = _('Cannot read file')
-            self.msg_label.setText(msg)
+            msg_label.setText(msg)
             widget_create_new.setVisible(bool(temp_storage and temp_storage.file_exists()))
             if user_needs_to_enter_password:
-                self.pw_label.show()
+                pw_label.show()
-                self.pw_e.show()
+                pw_e.show()
-                self.pw_e.setFocus()
+                pw_e.setFocus()
             else:
-                self.pw_label.hide()
+                pw_label.hide()
-                self.pw_e.hide()
+                pw_e.hide()
 
         button.clicked.connect(on_choose)
         button_create_new.clicked.connect(
             partial(
-                self.name_e.setText,
+                name_e.setText,
                 get_new_wallet_name(wallet_folder)))
-        self.name_e.textChanged.connect(on_filename)
+        name_e.textChanged.connect(on_filename)
-        self.name_e.setText(os.path.basename(path))
+        name_e.setText(os.path.basename(path))
 
+        def run_user_interaction_loop():
             while True:
                 if self.loop.exec_() != 2:  # 2 = next
-                raise UserCancelled
+                    raise UserCancelled()
                 assert temp_storage
                 if temp_storage.file_exists() and not temp_storage.is_encrypted():
                     break
@@ -284,7 +311,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
                     raise WalletAlreadyOpenInMemory(wallet_from_memory)
                 if temp_storage.file_exists() and temp_storage.is_encrypted():
                     if temp_storage.is_encrypted_with_user_pw():
-                    password = self.pw_e.text()
+                        password = pw_e.text()
                         try:
                             temp_storage.decrypt(password)
                             break
@@ -304,6 +331,8 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
                                                   _('If you use a passphrase, make sure it is correct.'))
                             self.reset_stack()
                             return self.select_storage(path, get_wallet_from_daemon)
+                        except (UserCancelled, GoBack):
+                            raise
                         except BaseException as e:
                             self.logger.exception('')
                             self.show_message(title=_('Error'), msg=repr(e))
@@ -315,9 +344,17 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
                     else:
                         raise Exception('Unexpected encryption version')
 
+        try:
+            run_user_interaction_loop()
+        finally:
+            try:
+                pw_e.clear()
+            except RuntimeError:  # wrapped C/C++ object has been deleted.
+                pass              # happens when decrypting with hw device
+
         return temp_storage.path, (temp_storage if temp_storage.file_exists() else None)
 
-    def run_upgrades(self, storage, db):
+    def run_upgrades(self, storage: WalletStorage, db: 'WalletDB') -> None:
         path = storage.path
         if db.requires_split():
             self.hide()
@@ -356,12 +393,6 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         if db.requires_upgrade():
             self.upgrade_db(storage, db)
 
-        return db
-
-    def finished(self):
-        """Called in hardware client wrapper, in order to close popups."""
-        return
-
     def on_error(self, exc_info):
         if not isinstance(exc_info[1], UserCancelled):
             self.logger.error("on_error", exc_info=exc_info)
@@ -393,7 +424,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         self.set_layout(layout, title, next_enabled)
         result = self.loop.exec_()
         if not result and raise_on_cancel:
-            raise UserCancelled
+            raise UserCancelled()
         if result == 1:
             raise GoBack from None
         self.title.setVisible(False)
@@ -476,8 +507,11 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         playout = PasswordLayout(msg=msg, kind=kind, OK_button=self.next_button,
                                  force_disable_encrypt_cb=force_disable_encrypt_cb)
         playout.encrypt_cb.setChecked(True)
+        try:
             self.exec_layout(playout.layout())
             return playout.new_password(), playout.encrypt_cb.isChecked()
+        finally:
+            playout.clear_password_fields()
 
     @wizard_dialog
     def request_password(self, run_next, force_disable_encrypt_cb=False):
@@ -530,6 +564,29 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         if on_finished:
             on_finished()
 
+
+    def run_task_without_blocking_gui(self, task, *, msg=None):
+        assert self.gui_thread == threading.current_thread(), 'must be called from GUI thread'
+        if msg is None:
+            msg = _("Please wait...")
+
+        exc = None  # type: Optional[Exception]
+        res = None
+        def task_wrapper():
+            nonlocal exc
+            nonlocal res
+            try:
+                res = task()
+            except Exception as e:
+                exc = e
+        self.waiting_dialog(task_wrapper, msg=msg)
+        if exc is None:
+            return res
+        else:
+            raise exc
+
+
+
     @wizard_dialog
     def choice_dialog(self, title, message, choices, run_next):
         c_values = [x[0] for x in choices]
@@ -550,10 +607,34 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         return clayout.selected_index()
 
     @wizard_dialog
-    def choice_and_line_dialog(self, title: str, message1: str, choices: List[Tuple[str, str, str]],
+    def derivation_and_script_type_gui_specific_dialog(
-                               message2: str, test_text: Callable[[str], int],
+            self,
-                               run_next, default_choice_idx: int=0) -> Tuple[str, str]:
+            *,
+            title: str,
+            message1: str,
+            choices: List[Tuple[str, str, str]],
+            message2: str,
+            test_text: Callable[[str], int],
+            run_next,
+            default_choice_idx: int = 0,
+            get_account_xpub=None
+    ) -> Tuple[str, str]:
         vbox = QVBoxLayout()
+        if get_account_xpub:
+            button = QPushButton(_("Detect Existing Accounts"))
+            def on_account_select(account):
+                script_type = account["script_type"]
+                if script_type == "p2pkh":
+                    script_type = "standard"
+                button_index = c_values.index(script_type)
+                button = clayout.group.buttons()[button_index]
+                button.setChecked(True)
+                line.setText(account["derivation_path"])
+            button.clicked.connect(lambda: Bip39RecoveryDialog(self, get_account_xpub, on_account_select))
+            vbox.addWidget(button, alignment=Qt.AlignLeft)
+            vbox.addWidget(QLabel(_("Or")))
+
+
 
         c_values = [x[0] for x in choices]
         c_titles = [x[1] for x in choices]
@@ -565,7 +646,6 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
                                 checked_index=default_choice_idx)
         vbox.addLayout(clayout.layout())
 
-        vbox.addSpacing(50)
         vbox.addWidget(WWLabel(message2))
 
         line = QLineEdit()
@@ -622,7 +702,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         self.exec_layout(vbox, _('Master Public Key'))
         return None
 
-    def init_network(self, network):
+    def init_network(self, network: 'Network'):
         message = _("Electrum communicates with remote servers to get "
                   "information about your transactions and addresses. The "
                   "servers all fulfill the same purpose only differing in "
@@ -639,6 +719,7 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
             nlayout = NetworkChoiceLayout(network, self.config, wizard=True)
             if self.exec_layout(nlayout.layout()):
                 nlayout.accept()
+                self.config.set_key('auto_connect', network.auto_connect, True)
         else:
             network.auto_connect = True
             self.config.set_key('auto_connect', True, True)
@@ -664,18 +745,25 @@ class InstallWizard(QDialog, MessageBoxMixin, BaseWizard):
         def on_m(m):
             m_label.setText(_('Require {0} signatures').format(m))
             cw.set_m(m)
+            backup_warning_label.setVisible(cw.m != cw.n)
         def on_n(n):
             n_label.setText(_('From {0} cosigners').format(n))
             cw.set_n(n)
             m_edit.setMaximum(n)
+            backup_warning_label.setVisible(cw.m != cw.n)
         n_edit.valueChanged.connect(on_n)
         m_edit.valueChanged.connect(on_m)
-        on_n(2)
-        on_m(2)
         vbox = QVBoxLayout()
         vbox.addWidget(cw)
         vbox.addWidget(WWLabel(_("Choose the number of signatures needed to unlock funds in your wallet:")))
         vbox.addLayout(grid)
+        vbox.addSpacing(2 * char_width_in_lineedit())
+        backup_warning_label = WWLabel(_("Warning: to be able to restore a multisig wallet, "
+                                         "you should include the master public key for each cosigner "
+                                         "in all of your backups."))
+        vbox.addWidget(backup_warning_label)
+        on_n(2)
+        on_m(2)
         self.exec_layout(vbox, _("Multi-Signature Wallet"))
         m = int(m_edit.value())
         n = int(n_edit.value())

electrum/gui/qt/invoice_list.py

@@ -29,16 +29,14 @@ from typing import Sequence
 from PyQt5.QtCore import Qt, QItemSelectionModel
 from PyQt5.QtGui import QStandardItemModel, QStandardItem
 from PyQt5.QtWidgets import QAbstractItemView
-from PyQt5.QtWidgets import QMenu, QVBoxLayout, QTreeWidget, QTreeWidgetItem
+from PyQt5.QtWidgets import QMenu, QVBoxLayout, QTreeWidget, QTreeWidgetItem, QHeaderView
 
 from electrum.i18n import _
-from electrum.util import format_time, PR_UNPAID, PR_PAID, PR_INFLIGHT
+from electrum.util import format_time
-from electrum.util import get_request_status
+from electrum.invoices import Invoice, PR_UNPAID, PR_PAID, PR_INFLIGHT, PR_FAILED, PR_TYPE_ONCHAIN, PR_TYPE_LN
-from electrum.util import PR_TYPE_ONCHAIN, PR_TYPE_LN
 from electrum.lnutil import PaymentAttemptLog
 
-from .util import (MyTreeView, read_QIcon,
+from .util import MyTreeView, read_QIcon, MySortModel, pr_icons
-                   import_meta_gui, export_meta_gui, pr_icons)
 from .util import CloseButton, Buttons
 from .util import WindowModalDialog
 
@@ -46,6 +44,7 @@ from .util import WindowModalDialog
 
 ROLE_REQUEST_TYPE = Qt.UserRole
 ROLE_REQUEST_ID = Qt.UserRole + 1
+ROLE_SORT_ORDER = Qt.UserRole + 2
 
 
 class InvoiceList(MyTreeView):
@@ -68,16 +67,16 @@ class InvoiceList(MyTreeView):
         super().__init__(parent, self.create_menu,
                          stretch_column=self.Columns.DESCRIPTION,
                          editable_columns=[])
+        self.std_model = QStandardItemModel(self)
+        self.proxy = MySortModel(self, sort_role=ROLE_SORT_ORDER)
+        self.proxy.setSourceModel(self.std_model)
+        self.setModel(self.proxy)
         self.setSortingEnabled(True)
-        self.setModel(QStandardItemModel(self))
         self.setSelectionMode(QAbstractItemView.ExtendedSelection)
         self.update()
 
-    def update_item(self, key, status):
+    def update_item(self, key, invoice: Invoice):
-        req = self.parent.wallet.get_invoice(key)
+        model = self.std_model
-        if req is None:
-            return
-        model = self.model()
         for row in range(0, model.rowCount()):
             item = model.item(row, 0)
             if item.data(ROLE_REQUEST_ID) == key:
@@ -85,7 +84,8 @@ class InvoiceList(MyTreeView):
         else:
             return
         status_item = model.item(row, self.Columns.STATUS)
-        status, status_str = get_request_status(req)
+        status = self.parent.wallet.get_invoice_status(invoice)
+        status_str = invoice.get_status_str(status)
         if self.parent.wallet.lnworker:
             log = self.parent.wallet.lnworker.logs.get(key)
             if log and status == PR_INFLIGHT:
@@ -95,29 +95,23 @@ class InvoiceList(MyTreeView):
 
     def update(self):
         # not calling maybe_defer_update() as it interferes with conditional-visibility
-        _list = self.parent.wallet.get_invoices()
+        self.proxy.setDynamicSortFilter(False)  # temp. disable re-sorting after every change
-        # filter out paid invoices unless we have the log
+        self.std_model.clear()
-        lnworker_logs = self.parent.wallet.lnworker.logs if self.parent.wallet.lnworker else {}
-        _list = [x for x in _list
-                 if x and (x.get('status') != PR_PAID or x.get('rhash') in lnworker_logs)]
-        self.model().clear()
         self.update_headers(self.__class__.headers)
-        for idx, item in enumerate(_list):
+        for idx, item in enumerate(self.parent.wallet.get_invoices()):
-            invoice_type = item['type']
+            if item.is_lightning():
-            if invoice_type == PR_TYPE_LN:
+                key = item.rhash
-                key = item['rhash']
                 icon_name = 'lightning.png'
-            elif invoice_type == PR_TYPE_ONCHAIN:
-                key = item['id']
-                icon_name = 'bitcoin.png'
-                if item.get('bip70'):
-                    icon_name = 'seal.png'
             else:
-                raise Exception('Unsupported type')
+                key = item.id
-            status, status_str = get_request_status(item)
+                icon_name = 'bitcoin.png'
-            message = item['message']
+                if item.bip70:
-            amount = item['amount']
+                    icon_name = 'seal.png'
-            timestamp = item.get('time', 0)
+            status = self.parent.wallet.get_invoice_status(item)
+            status_str = item.get_status_str(status)
+            message = item.message
+            amount = item.get_amount_sat()
+            timestamp = item.time or 0
             date_str = format_time(timestamp) if timestamp else _('Unknown')
             amount_str = self.parent.format_amount(amount, whitespaces=True)
             labels = [date_str, message, amount_str, status_str]
@@ -126,84 +120,70 @@ class InvoiceList(MyTreeView):
             items[self.Columns.DATE].setIcon(read_QIcon(icon_name))
             items[self.Columns.STATUS].setIcon(read_QIcon(pr_icons.get(status)))
             items[self.Columns.DATE].setData(key, role=ROLE_REQUEST_ID)
-            items[self.Columns.DATE].setData(invoice_type, role=ROLE_REQUEST_TYPE)
+            items[self.Columns.DATE].setData(item.type, role=ROLE_REQUEST_TYPE)
-            self.model().insertRow(idx, items)
+            items[self.Columns.DATE].setData(timestamp, role=ROLE_SORT_ORDER)
-
+            self.std_model.insertRow(idx, items)
-        self.selectionModel().select(self.model().index(0,0), QItemSelectionModel.SelectCurrent)
+        self.filter()
+        self.proxy.setDynamicSortFilter(True)
         # sort requests by date
-        self.sortByColumn(self.Columns.DATE, Qt.AscendingOrder)
+        self.sortByColumn(self.Columns.DATE, Qt.DescendingOrder)
         # hide list if empty
         if self.parent.isVisible():
-            b = self.model().rowCount() > 0
+            b = self.std_model.rowCount() > 0
             self.setVisible(b)
             self.parent.invoices_label.setVisible(b)
-        self.filter()
-
-    def import_invoices(self):
-        import_meta_gui(self.parent, _('invoices'), self.parent.invoices.import_file, self.update)
-
-    def export_invoices(self):
-        export_meta_gui(self.parent, _('invoices'), self.parent.invoices.export_file)
 
     def create_menu(self, position):
+        wallet = self.parent.wallet
         items = self.selected_in_column(0)
         if len(items)>1:
             keys = [ item.data(ROLE_REQUEST_ID)  for item in items]
-            invoices = [ self.parent.wallet.get_invoice(key) for key in keys]
+            invoices = [ wallet.invoices.get(key) for key in keys]
-            invoices = [ invoice for invoice in invoices if invoice['status'] == PR_UNPAID and invoice['type'] == PR_TYPE_ONCHAIN]
+            can_batch_pay = all([i.type == PR_TYPE_ONCHAIN and wallet.get_invoice_status(i) == PR_UNPAID for i in invoices])
-            if len(invoices) > 1:
             menu = QMenu(self)
-                menu.addAction(_("Pay multiple invoices"), lambda: self.parent.pay_multiple_invoices(invoices))
+            if can_batch_pay:
+                menu.addAction(_("Batch pay invoices"), lambda: self.parent.pay_multiple_invoices(invoices))
+            menu.addAction(_("Delete invoices"), lambda: self.parent.delete_invoices(keys))
             menu.exec_(self.viewport().mapToGlobal(position))
             return
         idx = self.indexAt(position)
-        item = self.model().itemFromIndex(idx)
+        item = self.item_from_index(idx)
-        item_col0 = self.model().itemFromIndex(idx.sibling(idx.row(), self.Columns.DATE))
+        item_col0 = self.item_from_index(idx.sibling(idx.row(), self.Columns.DATE))
         if not item or not item_col0:
             return
         key = item_col0.data(ROLE_REQUEST_ID)
-        request_type = item_col0.data(ROLE_REQUEST_TYPE)
+       invoice = self.parent.wallet.get_invoice(key)
         menu = QMenu(self)
         self.add_copy_menu(menu, idx)
-        invoice = self.parent.wallet.get_invoice(key)
+        if invoice.is_lightning():
-        menu.addAction(_("Details"), lambda: self.parent.show_invoice(key))
+            menu.addAction(_("Details"), lambda: self.parent.show_lightning_invoice(invoice))
-        if invoice['status'] == PR_UNPAID:
+        else:
+            if len(invoice.outputs) == 1:
+                menu.addAction(_("Copy Address"), lambda: self.parent.do_copy(invoice.get_address(), title='Bitcoin Address'))
+            menu.addAction(_("Details"), lambda: self.parent.show_onchain_invoice(invoice))
+        status = wallet.get_invoice_status(invoice)
+        if status == PR_UNPAID:
             menu.addAction(_("Pay"), lambda: self.parent.do_pay_invoice(invoice))
+        if status == PR_FAILED:
+            menu.addAction(_("Retry"), lambda: self.parent.do_pay_invoice(invoice))
         if self.parent.wallet.lnworker:
             log = self.parent.wallet.lnworker.logs.get(key)
             if log:
                 menu.addAction(_("View log"), lambda: self.show_log(key, log))
-        menu.addAction(_("Delete"), lambda: self.parent.delete_invoice(key))
+        menu.addAction(_("Delete"), lambda: self.parent.delete_invoices([key]))
         menu.exec_(self.viewport().mapToGlobal(position))
 
     def show_log(self, key, log: Sequence[PaymentAttemptLog]):
         d = WindowModalDialog(self, _("Payment log"))
-        d.setMinimumWidth(800)
+        d.setMinimumWidth(600)
         vbox = QVBoxLayout(d)
         log_w = QTreeWidget()
-        log_w.setHeaderLabels([_('Route'), _('Channel ID'), _('Message'), _('Blacklist')])
+        log_w.setHeaderLabels([_('Hops'), _('Channel ID'), _('Message')])
+        log_w.header().setSectionResizeMode(2, QHeaderView.Stretch)
+        log_w.header().setSectionResizeMode(1, QHeaderView.ResizeToContents)
         for payment_attempt_log in log:
-            if not payment_attempt_log.exception:
+            route_str, chan_str, message = payment_attempt_log.formatted_tuple()
-                route = payment_attempt_log.route
+            x = QTreeWidgetItem([route_str, chan_str, message])
-                route_str = '%d'%len(route)
-                if not payment_attempt_log.success:
-                    sender_idx = payment_attempt_log.failure_details.sender_idx
-                    failure_msg = payment_attempt_log.failure_details.failure_msg
-                    blacklist_msg = str(payment_attempt_log.failure_details.is_blacklisted)
-                    short_channel_id = route[sender_idx+1].short_channel_id
-                    data = failure_msg.data
-                    message = repr(failure_msg.code)
-                else:
-                    short_channel_id = route[-1].short_channel_id
-                    message = _('Success')
-                    blacklist_msg = str(False)
-                chan_str = str(short_channel_id)
-            else:
-                route_str = 'None'
-                chan_str = 'N/A'
-                message = str(payment_attempt_log.exception)
-                blacklist_msg = 'N/A'
-            x = QTreeWidgetItem([route_str, chan_str, message, blacklist_msg])
             log_w.addTopLevelItem(x)
         vbox.addWidget(log_w)
         vbox.addLayout(Buttons(CloseButton(d)))

electrum/gui/qt/main_window.py

@@ -45,51 +45,50 @@ from PyQt5.QtWidgets import (QMessageBox, QComboBox, QSystemTrayIcon, QTabWidget
                              QVBoxLayout, QGridLayout, QLineEdit,
                              QHBoxLayout, QPushButton, QScrollArea, QTextEdit,
                              QShortcut, QMainWindow, QCompleter, QInputDialog,
-                             QWidget, QSizePolicy, QStatusBar, QToolTip)
+                             QWidget, QSizePolicy, QStatusBar, QToolTip, QDialog,
+                             QMenu, QAction)
 
 import electrum
 from electrum import (keystore, ecc, constants, util, bitcoin, commands,
                       paymentrequest)
 from electrum.bitcoin import COIN, is_address
-from electrum.plugin import run_hook
+from electrum.plugin import run_hook, BasePlugin
 from electrum.i18n import _
-from electrum.util import (format_time, format_satoshis, format_fee_satoshis,
+from electrum.util import (format_time,
-                           format_satoshis_plain,
                            UserCancelled, profiler,
-                           export_meta, import_meta, bh2u, bfh, InvalidPassword,
+                           bh2u, bfh, InvalidPassword,
-                           decimal_point_to_base_unit_name,
+                           UserFacingException,
-                           UnknownBaseUnit, DECIMAL_POINT_DEFAULT, UserFacingException,
                            get_new_wallet_name, send_exception_to_crash_reporter,
                            InvalidBitcoinURI, maybe_extract_bolt11_invoice, NotEnoughFunds,
                            NoDynamicFeeEstimates, MultipleSpendMaxTxOutputs)
-from electrum.util import PR_TYPE_ONCHAIN, PR_TYPE_LN
+from electrum.invoices import PR_TYPE_ONCHAIN, PR_TYPE_LN, PR_DEFAULT_EXPIRATION_WHEN_CREATING, Invoice
+from electrum.invoices import PR_PAID, PR_FAILED, pr_expiration_values, LNInvoice, OnchainInvoice
 from electrum.transaction import (Transaction, PartialTxInput,
                                   PartialTransaction, PartialTxOutput)
 from electrum.address_synchronizer import AddTransactionException
 from electrum.wallet import (Multisig_Wallet, CannotBumpFee, Abstract_Wallet,
                              sweep_preparations, InternalAddressCorruption)
 from electrum.version import ELECTRUM_VERSION
-from electrum.network import Network, TxBroadcastError, BestEffortRequestFailed
+from electrum.network import Network, TxBroadcastError, BestEffortRequestFailed, UntrustedServerReturnedError
 from electrum.exchange_rate import FxThread
 from electrum.simple_config import SimpleConfig
 from electrum.logging import Logger
-from electrum.util import PR_PAID, PR_FAILED
-from electrum.util import pr_expiration_values
 from electrum.lnutil import ln_dummy_address
+from electrum.lnaddr import lndecode, LnDecodeException
 
 from .exception_window import Exception_Hook
 from .amountedit import AmountEdit, BTCAmountEdit, FreezableLineEdit, FeerateEdit
 from .qrcodewidget import QRCodeWidget, QRDialog
 from .qrtextedit import ShowQRTextEdit, ScanQRTextEdit
 from .transaction_dialog import show_transaction
-#from .fee_slider import FeeSlider
+from .fee_slider import FeeSlider, FeeComboBox
 from .util import (read_QIcon, ColorScheme, text_dialog, icon_path, WaitingDialog,
                    WindowModalDialog, ChoicesLayout, HelpLabel, Buttons,
                    OkButton, InfoButton, WWLabel, TaskThread, CancelButton,
                    CloseButton, HelpButton, MessageBoxMixin, EnterButton,
                    import_meta_gui, export_meta_gui,
                    filename_field, address_field, char_width_in_lineedit, webopen,
-                   TRANSACTION_FILE_EXTENSION_FILTER, MONOSPACE_FONT)
+                   TRANSACTION_FILE_EXTENSION_FILTER_ANY, MONOSPACE_FONT)
 from .util import ButtonsTextEdit
 from .installwizard import WIF_HELP_TEXT
 from .history_list import HistoryList, HistoryModel
@@ -158,6 +157,9 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
     alias_received_signal = pyqtSignal()
     computing_privkeys_signal = pyqtSignal()
     show_privkeys_signal = pyqtSignal()
+    show_error_signal = pyqtSignal(str)
+
+    payment_request: Optional[paymentrequest.PaymentRequest]
 
     def __init__(self, gui_object: 'ElectrumGui', wallet: Abstract_Wallet):
         QMainWindow.__init__(self)
@@ -165,12 +167,12 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.gui_object = gui_object
         self.config = config = gui_object.config  # type: SimpleConfig
         self.gui_thread = gui_object.gui_thread
+        assert wallet, "no wallet"
+        self.wallet = wallet
 
         self.setup_exception_hook()
 
         self.network = gui_object.daemon.network  # type: Network
-        assert wallet, "no wallet"
-        self.wallet = wallet
         self.fx = gui_object.daemon.fx  # type: FxThread
         self.contacts = wallet.contacts
         self.tray = gui_object.tray
@@ -181,6 +183,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.checking_accounts = False
         self.qr_window = None
         self.pluginsdialog = None
+        self.showing_cert_mismatch_error = False
         self.tl_windows = []
         Logger.__init__(self)
 
@@ -189,12 +192,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
 
         self.create_status_bar()
         self.need_update = threading.Event()
-        self.decimal_point = config.get('decimal_point', DECIMAL_POINT_DEFAULT)
-        try:
-            decimal_point_to_base_unit_name(self.decimal_point)
-        except UnknownBaseUnit:
-            self.decimal_point = DECIMAL_POINT_DEFAULT
-        self.num_zeros = int(config.get('num_zeros', 0))
 
         self.completions = QStringListModel()
 
@@ -207,7 +204,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.utxo_tab = self.create_utxo_tab()
         self.console_tab = self.create_console_tab()
         self.contacts_tab = self.create_contacts_tab()
-        self.channels_tab = self.create_channels_tab(wallet)
+        self.channels_tab = self.create_channels_tab()
         tabs.addTab(self.create_history_tab(), read_QIcon("tab_history.png"), _('History'))
         tabs.addTab(self.send_tab, read_QIcon("tab_send.png"), _('Send'))
         tabs.addTab(self.receive_tab, read_QIcon("tab_receive.png"), _('Receive'))
@@ -221,7 +218,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
                 tabs.addTab(tab, icon, description.replace("&", ""))
 
         add_optional_tab(tabs, self.addresses_tab, read_QIcon("tab_addresses.png"), _("&Addresses"), "addresses")
-        if self.wallet.has_lightning():
         add_optional_tab(tabs, self.channels_tab, read_QIcon("lightning.png"), _("Channels"), "channels")
         add_optional_tab(tabs, self.utxo_tab, read_QIcon("tab_coins.png"), _("Co&ins"), "utxo")
         add_optional_tab(tabs, self.contacts_tab, read_QIcon("tab_contacts.png"), _("Con&tacts"), "contacts")
@@ -256,6 +252,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
 
         self.payment_request_ok_signal.connect(self.payment_request_ok)
         self.payment_request_error_signal.connect(self.payment_request_error)
+        self.show_error_signal.connect(self.show_error)
         self.history_list.setFocus(True)
 
         # network callbacks
@@ -266,11 +263,14 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
                          'banner', 'verified', 'fee', 'fee_histogram', 'on_quotes',
                          'on_history', 'channel', 'channels_updated',
                          'invoice_status', 'request_status']
+                         'payment_failed', 'payment_succeeded',
+                         'invoice_status', 'request_status', 'ln_gossip_sync_progress',
+                         'cert_mismatch', 'gossip_db_loaded']
             # To avoid leaking references to "self" that prevent the
             # window from being GC-ed when closed, callbacks should be
             # methods of this class only, and specifically not be
             # partials, lambdas or methods of subobjects.  Hence...
-            self.network.register_callback(self.on_network, interests)
+            util.register_callback(self.on_network, interests)
             # set initial message
             self.console.showMessage(self.network.banner)
 
@@ -295,12 +295,25 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
                     self.update_check_button.setText(_("Update to Electrum {} is available").format(v))
                     self.update_check_button.clicked.connect(lambda: self.show_update_check(v))
                     self.update_check_button.show()
-            self._update_check_thread = UpdateCheckThread(self)
+            self._update_check_thread = UpdateCheckThread()
             self._update_check_thread.checked.connect(on_version_received)
             self._update_check_thread.start()
 
     def setup_exception_hook(self):
-        Exception_Hook(self)
+        Exception_Hook.maybe_setup(config=self.config,
+                                   wallet=self.wallet)
+
+    def run_coroutine_from_thread(self, coro, on_result=None):
+        def task():
+            try:
+                f = asyncio.run_coroutine_threadsafe(coro, self.network.asyncio_loop)
+                r = f.result()
+                if on_result:
+                    on_result(r)
+            except Exception as e:
+                self.logger.exception("exception in coro scheduled via window.wallet")
+                self.show_error_signal.emit(str(e))
+        self.wallet.thread.add(task)
 
     def on_fx_history(self):
         self.history_model.refresh('fx_history')
@@ -409,15 +422,29 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
             self.on_fx_quotes()
         elif event == 'on_history':
             self.on_fx_history()
+        elif event == 'gossip_db_loaded':
+            self.channels_list.gossip_db_loaded.emit(*args)
         elif event == 'channels_updated':
+            wallet = args[0]
+            if wallet == self.wallet:
                 self.channels_list.update_rows.emit(*args)
         elif event == 'channel':
+            wallet = args[0]
+            if wallet == self.wallet:
                 self.channels_list.update_single_row.emit(*args)
                 self.update_status()
         elif event == 'request_status':
             self.on_request_status(*args)
         elif event == 'invoice_status':
             self.on_invoice_status(*args)
+        elif event == 'payment_succeeded':
+            wallet = args[0]
+            if wallet == self.wallet:
+                self.on_payment_succeeded(*args)
+        elif event == 'payment_failed':
+            wallet = args[0]
+            if wallet == self.wallet:
+                self.on_payment_failed(*args)
         elif event == 'status':
             self.update_status()
         elif event == 'banner':
@@ -430,6 +457,10 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
             pass
         elif event == 'fee_histogram':
             self.history_model.on_fee_histogram()
+        elif event == 'ln_gossip_sync_progress':
+            self.update_lightning_icon()
+        elif event == 'cert_mismatch':
+            self.show_cert_mismatch_error()
         else:
             self.logger.info(f"unexpected network event: {event} {args}")
 
@@ -448,14 +479,15 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
     def close_wallet(self):
         if self.wallet:
             self.logger.info(f'close_wallet {self.wallet.storage.path}')
+            self.wallet.thread = None
         run_hook('close_wallet', self.wallet)
 
     @profiler
     def load_wallet(self, wallet):
         wallet.thread = TaskThread(self, self.on_error)
         self.update_recently_visited(wallet.storage.path)
-        if wallet.lnworker and wallet.network:
+        if wallet.has_lightning():
-            wallet.network.trigger_callback('channels_updated', wallet)
+            util.trigger_callback('channels_updated', wallet)
         self.need_update.set()
         # Once GUI has been initialized check if we want to announce something since the callback has been called before the GUI was initialized
         # update menus
@@ -549,20 +581,46 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
             return
         self.gui_object.new_window(filename)
 
+    def select_backup_dir(self, b):
+        name = self.config.get('backup_dir', '')
+        dirname = QFileDialog.getExistingDirectory(self, "Select your wallet backup directory", name)
+        if dirname:
+            self.config.set_key('backup_dir', dirname)
+            self.backup_dir_e.setText(dirname)
 
     def backup_wallet(self):
-        path = self.wallet.storage.path
+        d = WindowModalDialog(self, _("File Backup"))
-        wallet_folder = os.path.dirname(path)
+        vbox = QVBoxLayout(d)
-        filename, __ = QFileDialog.getSaveFileName(self, _('Enter a filename for the copy of your wallet'), wallet_folder)
+        grid = QGridLayout()
-        if not filename:
+        backup_help = ""
+        backup_dir = self.config.get('backup_dir')
+        backup_dir_label = HelpLabel(_('Backup directory') + ':', backup_help)
+        msg = _('Please select a backup directory')
+        if self.wallet.has_lightning() and self.wallet.lnworker.channels:
+            msg += '\n\n' + ' '.join([
+                _("Note that lightning channels will be converted to channel backups."),
+                _("You cannot use channel backups to perform lightning payments."),
+                _("Channel backups can only be used to request your channels to be closed.")
+            ])
+        self.backup_dir_e = QPushButton(backup_dir)
+        self.backup_dir_e.clicked.connect(self.select_backup_dir)
+        grid.addWidget(backup_dir_label, 1, 0)
+        grid.addWidget(self.backup_dir_e, 1, 1)
+        vbox.addLayout(grid)
+        vbox.addWidget(WWLabel(msg))
+        vbox.addLayout(Buttons(CancelButton(d), OkButton(d)))
+        if not d.exec_():
             return
-        new_path = os.path.join(wallet_folder, filename)
-        if new_path != path:
         try:
-                shutil.copy2(path, new_path)
+            new_path = self.wallet.save_backup()
-                self.show_message(_("A copy of your wallet file was created in")+" '%s'" % str(new_path), title=_("Wallet backup created"))
         except BaseException as reason:
             self.show_critical(_("Electrum was unable to copy your wallet file to the specified location.") + "\n" + str(reason), title=_("Unable to create backup"))
+            return
+        if new_path:
+            msg = _("A copy of your wallet file was created in")+" '%s'" % str(new_path)
+            self.show_message(msg, title=_("Wallet backup created"))
+        else:
+            self.show_message(_("You need to configure a backup directory in your preferences"), title=_("Backup not created"))
 
     def update_recently_visited(self, filename):
         recent = self.config.get('recently_open', [])
@@ -604,7 +662,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.recently_visited_menu = file_menu.addMenu(_("&Recently open"))
         file_menu.addAction(_("&Open"), self.open_wallet).setShortcut(QKeySequence.Open)
         file_menu.addAction(_("&New/Restore"), self.new_wallet).setShortcut(QKeySequence.New)
-        file_menu.addAction(_("&Save Copy"), self.backup_wallet).setShortcut(QKeySequence.SaveAs)
+        file_menu.addAction(_("&Save backup"), self.backup_wallet).setShortcut(QKeySequence.SaveAs)
         file_menu.addAction(_("Delete"), self.remove_wallet)
         file_menu.addSeparator()
         file_menu.addAction(_("&Quit"), self.close)
@@ -633,11 +691,14 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         history_menu.addAction(_("&Export"), self.history_list.export_history_dialog)
         contacts_menu = wallet_menu.addMenu(_("Contacts"))
         contacts_menu.addAction(_("&New"), self.new_contact_dialog)
-        contacts_menu.addAction(_("Import"), lambda: self.contact_list.import_contacts())
+        contacts_menu.addAction(_("Import"), lambda: self.import_contacts())
-        contacts_menu.addAction(_("Export"), lambda: self.contact_list.export_contacts())
+        contacts_menu.addAction(_("Export"), lambda: self.export_contacts())
         invoices_menu = wallet_menu.addMenu(_("Invoices"))
-        invoices_menu.addAction(_("Import"), lambda: self.invoice_list.import_invoices())
+        invoices_menu.addAction(_("Import"), lambda: self.import_invoices())
-        invoices_menu.addAction(_("Export"), lambda: self.invoice_list.export_invoices())
+        invoices_menu.addAction(_("Export"), lambda: self.export_invoices())
+        requests_menu = wallet_menu.addMenu(_("Requests"))
+        requests_menu.addAction(_("Import"), lambda: self.import_requests())
+        requests_menu.addAction(_("Export"), lambda: self.export_requests())
 
         wallet_menu.addSeparator()
         wallet_menu.addAction(_("Find"), self.toggle_search).setShortcut(QKeySequence("Ctrl+F"))
@@ -650,15 +711,21 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         view_menu = menubar.addMenu(_("&View"))
         add_toggle_action(view_menu, self.addresses_tab)
         add_toggle_action(view_menu, self.utxo_tab)
-        if self.wallet.has_lightning():
         add_toggle_action(view_menu, self.channels_tab)
         add_toggle_action(view_menu, self.contacts_tab)
         add_toggle_action(view_menu, self.console_tab)
 
-        tools_menu = menubar.addMenu(_("&Tools"))
+        tools_menu = menubar.addMenu(_("&Tools"))  # type: QMenu
+        preferences_action = tools_menu.addAction(_("Preferences"), self.settings_dialog)  # type: QAction
+        if sys.platform == 'darwin':
+            # "Settings"/"Preferences" are all reserved keywords in macOS.
+            # preferences_action will get picked up based on name (and put into a standardized location,
+            # and given a standard reserved hotkey)
+            # Hence, this menu item will be at a "uniform location re macOS processes"
+            preferences_action.setMenuRole(QAction.PreferencesRole)  # make sure OS recognizes it as preferences
+            # Add another preferences item, to also have a "uniform location for Electrum between different OSes"
+            tools_menu.addAction(_("Electrum preferences"), self.settings_dialog)
 
-        # Settings / Preferences are all reserved keywords in macOS using this as work around
-        tools_menu.addAction(_("Electrum preferences") if sys.platform == 'darwin' else _("Preferences"), self.settings_dialog)
         tools_menu.addAction(_("&Network"), self.gui_object.show_network_dialog).setEnabled(bool(self.network))
         tools_menu.addAction(_("&Lightning Network"), self.gui_object.show_lightning_dialog).setEnabled(bool(self.wallet.has_lightning() and self.network))
         tools_menu.addAction(_("Local &Watchtower"), self.gui_object.show_watchtower_dialog).setEnabled(bool(self.network and self.network.local_watchtower))
@@ -693,7 +760,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
     def donate_to_server(self):
         d = self.network.get_donation_address()
         if d:
-            host = self.network.get_parameters().host
+            host = self.network.get_parameters().server.host
             self.pay_to_URI('bitcoin:%s?message=donation for %s'%(d, host))
         else:
             self.show_error(_('No donation address for this server'))
@@ -709,7 +776,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
                            _("Uses icons from the Icons8 icon pack (icons8.com).")))
 
     def show_update_check(self, version=None):
-        self.gui_object._update_check = UpdateCheck(self, version)
+        self.gui_object._update_check = UpdateCheck(latest_version=version)
 
     def show_report_bug(self):
         msg = ' '.join([
@@ -772,13 +839,27 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
             self.config.set_key('io_dir', os.path.dirname(fileName), True)
         return fileName
 
-    def getSaveFileName(self, title, filename, filter = ""):
+    def getSaveFileName(self, title, filename, filter="",
+                        *, default_extension: str = None,
+                        default_filter: str = None) -> Optional[str]:
         directory = self.config.get('io_dir', os.path.expanduser('~'))
-        path = os.path.join( directory, filename )
+        path = os.path.join(directory, filename)
-        fileName, __ = QFileDialog.getSaveFileName(self, title, path, filter)
+
-        if fileName and directory != os.path.dirname(fileName):
+        file_dialog = QFileDialog(self, title, path, filter)
-            self.config.set_key('io_dir', os.path.dirname(fileName), True)
+        file_dialog.setAcceptMode(QFileDialog.AcceptSave)
-        return fileName
+        if default_extension:
+            # note: on MacOS, the selected filter's first extension seems to have priority over this...
+            file_dialog.setDefaultSuffix(default_extension)
+        if default_filter:
+            assert default_filter in filter, f"default_filter={default_filter!r} does not appear in filter={filter!r}"
+            file_dialog.selectNameFilter(default_filter)
+        if file_dialog.exec() != QDialog.Accepted:
+            return None
+
+        selected_path = file_dialog.selectedFiles()[0]
+        if selected_path and directory != os.path.dirname(selected_path):
+            self.config.set_key('io_dir', os.path.dirname(selected_path), True)
+        return selected_path
 
     def timer_actions(self):
         self.request_list.refresh_status()
@@ -795,21 +876,22 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.notify_transactions()
 
     def format_amount(self, x, is_diff=False, whitespaces=False):
-        return format_satoshis(x, self.num_zeros, self.decimal_point, is_diff=is_diff, whitespaces=whitespaces)
+            # x is in sats
+        return self.config.format_amount(x, is_diff=is_diff, whitespaces=whitespaces)
 
     def format_amount_and_units(self, amount):
-        text = self.format_amount(amount) + ' '+ self.base_unit()
+        # amount is in sats
+        text = self.config.format_amount_and_units(amount)
         x = self.fx.format_amount_and_units(amount) if self.fx else None
         if text and x:
             text += ' (%s)'%x
         return text
 
     def format_fee_rate(self, fee_rate):
-        # fee_rate is in sat/kB
+        return self.config.format_fee_rate(fee_rate)
-        return format_fee_satoshis(fee_rate/1000, num_zeros=self.num_zeros) + ' sat/byte'
 
     def get_decimal_point(self):
-        return self.decimal_point
+        return self.config.get_decimal_point()
 
     def base_unit(self):
         return decimal_point_to_base_unit_name(self.decimal_point)
@@ -881,9 +963,9 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
                     text +=  " [%s unconfirmed]"%(self.format_amount(u, is_diff=True).strip())
                 if x:
                     text +=  " [%s unmatured]"%(self.format_amount(x, is_diff=True).strip())
-                if self.wallet.lnworker:
+                if self.wallet.has_lightning():
                     l = self.wallet.lnworker.get_balance()
-                    text += u'    \U0001f5f2 %s'%(self.format_amount_and_units(l).strip())
+                    text += u'    \U000026a1 %s'%(self.format_amount_and_units(l).strip())
 
                 # append fiat balance and price
                 if self.fx.is_enabled():
@@ -924,7 +1006,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.channels_list.update_rows.emit(wallet)
         self.update_completions()
 
-    def create_channels_tab(self, wallet):
+    def create_channels_tab(self):
         self.channels_list = ChannelsList(self)
         t = self.channels_list.get_toolbar()
         return self.create_list_tab(self.channels_list, t)
@@ -944,10 +1026,23 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         d = address_dialog.AddressDialog(self, addr)
         d.exec_()
 
+    def show_channel(self, channel_id):
+        from . import channel_details
+        channel_details.ChannelDetailsDialog(self, channel_id).show()
+
+
+
     def show_transaction(self, tx, *, tx_desc=None):
         '''tx_desc is set only for txs created in the Send tab'''
         show_transaction(tx, parent=self, desc=tx_desc)
 
+    def show_lightning_transaction(self, tx_item):
+        from .lightning_tx_dialog import LightningTxDialog
+        d = LightningTxDialog(self, tx_item)
+        d.show()
+
+
+
     def create_receive_tab(self):
         # A 4-column grid layout.  All the stretch is in the last column.
         # The exchange rate plugin adds a fiat widget in column 2
@@ -977,7 +1072,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         evl = sorted(pr_expiration_values.items())
         evl_keys = [i[0] for i in evl]
         evl_values = [i[1] for i in evl]
-        default_expiry = self.config.get('request_expiry', 3600)
+        default_expiry = self.config.get('request_expiry', PR_DEFAULT_EXPIRATION_WHEN_CREATING)
         try:
             i = evl_keys.index(default_expiry)
         except ValueError:
@@ -994,7 +1089,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
              _('Expired requests have to be deleted manually from your list, in order to free the corresponding LBRY Credits addresses.'),
             _('The LBRY Credits address never expires and will always be part of this electrum wallet.'),
         ])
-        grid.addWidget(HelpLabel(_('Request expires'), msg), 2, 0)
+        grid.addWidget(HelpLabel(_('Expires after'), msg), 2, 0)
         grid.addWidget(self.expires_combo, 2, 1)
         self.expires_label = QLineEdit('')
         self.expires_label.setReadOnly(1)
@@ -1004,21 +1099,25 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
 
         self.clear_invoice_button = QPushButton(_('Clear'))
         self.clear_invoice_button.clicked.connect(self.clear_receive_tab)
-        self.create_invoice_button = QPushButton(_('On-chain'))
+        self.create_invoice_button = QPushButton(_('Request'))
         self.create_invoice_button.setIcon(read_QIcon("bitcoin.png"))
+        self.create_invoice_button.setToolTip('Create on-chain request')
         self.create_invoice_button.clicked.connect(lambda: self.create_invoice(False))
         self.receive_buttons = buttons = QHBoxLayout()
         buttons.addStretch(1)
         buttons.addWidget(self.clear_invoice_button)
         buttons.addWidget(self.create_invoice_button)
         if self.wallet.has_lightning():
+            self.create_invoice_button.setText(_('On-chain'))
             self.create_lightning_invoice_button = QPushButton(_('Lightning'))
+            self.create_lightning_invoice_button.setToolTip('Create lightning request')
             self.create_lightning_invoice_button.setIcon(read_QIcon("lightning.png"))
             self.create_lightning_invoice_button.clicked.connect(lambda: self.create_invoice(True))
             buttons.addWidget(self.create_lightning_invoice_button)
         grid.addLayout(buttons, 4, 3, 1, 2)
 
         self.receive_payreq_e = ButtonsTextEdit()
+        self.receive_payreq_e.setFont(QFont(MONOSPACE_FONT))
         self.receive_payreq_e.addCopyButton(self.app)
         self.receive_payreq_e.setReadOnly(True)
         self.receive_payreq_e.textChanged.connect(self.update_receive_qr)
@@ -1029,21 +1128,12 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.receive_qr.enterEvent = lambda x: self.app.setOverrideCursor(QCursor(Qt.PointingHandCursor))
         self.receive_qr.leaveEvent = lambda x: self.app.setOverrideCursor(QCursor(Qt.ArrowCursor))
 
-        def on_receive_address_changed():
-            addr = str(self.receive_address_e.text())
-            self.receive_address_widgets.setVisible(bool(addr))
-
-        msg = _('LBRY Credits address where the payment should be received. Note that each payment request uses a different LBRY Credits address.')
-        receive_address_label = HelpLabel(_('Receiving address'), msg)
-
         self.receive_address_e = ButtonsTextEdit()
         self.receive_address_e.setFont(QFont(MONOSPACE_FONT))
         self.receive_address_e.addCopyButton(self.app)
         self.receive_address_e.setReadOnly(True)
-        self.receive_address_e.textChanged.connect(on_receive_address_changed)
         self.receive_address_e.textChanged.connect(self.update_receive_address_styling)
-        self.receive_address_e.setMinimumHeight(6 * char_width_in_lineedit())
+
-        self.receive_address_e.setMaximumHeight(10 * char_width_in_lineedit())
         qr_show = lambda: self.show_qrcode(str(self.receive_address_e.text()), _('Receiving address'), parent=self)
         qr_icon = "qrcode_white.png" if ColorScheme.dark_scheme else "qrcode.png"
         self.receive_address_e.addButton(qr_icon, qr_show, _("Show as QR code"))
@@ -1053,34 +1143,24 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         from .request_list import RequestList
         self.request_list = RequestList(self)
 
+        receive_tabs = QTabWidget()
+        receive_tabs.addTab(self.receive_address_e, _('Address'))
+        receive_tabs.addTab(self.receive_payreq_e, _('Request'))
+        receive_tabs.addTab(self.receive_qr, _('QR Code'))
+        receive_tabs.setCurrentIndex(self.config.get('receive_tabs_index', 0))
+        receive_tabs.currentChanged.connect(lambda i: self.config.set_key('receive_tabs_index', i))
+
+
+
         # layout
         vbox_g = QVBoxLayout()
         vbox_g.addLayout(grid)
         vbox_g.addStretch()
 
-        receive_tabbed_widgets = QTabWidget()
-        receive_tabbed_widgets.addTab(self.receive_qr, 'QR Code')
-        receive_tabbed_widgets.addTab(self.receive_payreq_e, 'Text')
-
-        vbox_receive_address = QVBoxLayout()
-        vbox_receive_address.setContentsMargins(0, 0, 0, 0)
-        vbox_receive_address.setSpacing(0)
-        vbox_receive_address.addWidget(receive_address_label)
-        vbox_receive_address.addWidget(self.receive_address_e)
-        self.receive_address_widgets = QWidget()
-        self.receive_address_widgets.setLayout(vbox_receive_address)
-        size_policy = self.receive_address_widgets.sizePolicy()
-        size_policy.setRetainSizeWhenHidden(True)
-        self.receive_address_widgets.setSizePolicy(size_policy)
-
-        vbox_receive = QVBoxLayout()
-        vbox_receive.addWidget(receive_tabbed_widgets)
-        vbox_receive.addWidget(self.receive_address_widgets)
-
         hbox = QHBoxLayout()
         hbox.addLayout(vbox_g)
         hbox.addStretch()
-        hbox.addLayout(vbox_receive)
+        hbox.addWidget(receive_tabs)
 
         w = QWidget()
         w.searchable_list = self.request_list
@@ -1092,11 +1172,10 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         vbox.addWidget(self.request_list)
         vbox.setStretchFactor(self.request_list, 1000)
 
-        on_receive_address_changed()
-
         return w
 
-    def delete_request(self, key):
+    def delete_requests(self, keys):
+        for key in keys:
             self.wallet.delete_request(key)
         self.request_list.update()
         self.clear_receive_tab()
@@ -1109,7 +1188,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
 
     def sign_payment_request(self, addr):
         alias = self.config.get('alias')
-        alias_privkey = None
         if alias and self.alias_info:
             alias_addr, alias_name, validated = self.alias_info
             if alias_addr:
@@ -1131,29 +1209,41 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
     def create_invoice(self, is_lightning):
         amount = self.receive_amount_e.get_amount()
         message = self.receive_message_e.text()
-        expiry = self.config.get('request_expiry', 3600)
+        expiry = self.config.get('request_expiry', PR_DEFAULT_EXPIRATION_WHEN_CREATING)
         if is_lightning:
             key = self.wallet.lnworker.add_request(amount, message, expiry)
         else:
             key = self.create_bitcoin_request(amount, message, expiry)
+            if not key:
+                return
             self.address_list.update()
+        assert key is not None
         self.request_list.update()
         self.request_list.select_key(key)
         # clear request fields
         self.receive_amount_e.setText('')
         self.receive_message_e.setText('')
 
-    def create_bitcoin_request(self, amount, message, expiration):
+        # copy to clipboard
+        r = self.wallet.get_request(key)
+        content = r.invoice if r.is_lightning() else r.get_address()
+        title = _('Invoice') if is_lightning else _('Address')
+        self.do_copy(content, title=title)
+
+    def create_bitcoin_request(self, amount, message, expiration) -> Optional[str]:
         addr = self.wallet.get_unused_address()
         if addr is None:
-            if not self.wallet.is_deterministic():
+            if not self.wallet.is_deterministic():  # imported wallet
                 msg = [
-                    _('No more addresses in your wallet.'),
+                    _('No more addresses in your wallet.'), ' ',
-                    _('You are using a non-deterministic wallet, which cannot create new addresses.'),
+                    _('You are using a non-deterministic wallet, which cannot create new addresses.'), ' ',
-                    _('If you want to create new addresses, use a deterministic wallet instead.')
+                    _('If you want to create new addresses, use a deterministic wallet instead.'), '\n\n',
+                    _('Creating a new payment request will reuse one of your addresses and overwrite an existing request. Continue anyway?'),
                    ]
-                self.show_message(' '.join(msg))
+                if not self.question(''.join(msg)):
                     return
+                addr = self.wallet.get_receiving_address()
+            else:  # deterministic wallet
                 if not self.question(_("Warning: The next address will not be recovered automatically if you restore your wallet from seed; you may need to add it manually.\n\nThis occurs because you have too many unused addresses in your wallet. To avoid this situation, use the existing addresses first.\n\nCreate anyway?")):
                     return
                 addr = self.wallet.create_new_address(False)
@@ -1175,16 +1265,6 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
             tooltip_text = _("{} copied to clipboard").format(title)
         QToolTip.showText(QCursor.pos(), tooltip_text, self)
 
-    def export_payment_request(self, addr):
-        r = self.wallet.receive_requests.get(addr)
-        pr = paymentrequest.serialize_request(r).SerializeToString()
-        name = r['id'] + '.bip70'
-        fileName = self.getSaveFileName(_("Select where to save your payment request"), name, "*.bip70")
-        if fileName:
-            with open(fileName, "wb+") as f:
-                f.write(util.to_bytes(pr))
-            self.show_message(_("Request saved successfully"))
-            self.saved = True
 
     def clear_receive_tab(self):
         self.receive_payreq_e.setText('')
@@ -1193,6 +1273,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.receive_amount_e.setAmount(None)
         self.expires_label.hide()
         self.expires_combo.show()
+        self.request_list.clearSelection()
 
     def toggle_qr_window(self):
         from . import qrwindow
@@ -1416,10 +1497,17 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
 
         return False  # no errors
 
-    def pay_lightning_invoice(self, invoice, amount_sat=None):
+    def pay_lightning_invoice(self, invoice: str, *, amount_msat: Optional[int]):
+        if amount_msat is None:
+            raise Exception("missing amount for LN invoice")
+        amount_sat = Decimal(amount_msat) / 1000
+        # FIXME this is currently lying to user as we truncate to satoshis
+        msg = _("Pay lightning invoice?") + '\n\n' + _("This will send {}?").format(self.format_amount_and_units(amount_sat))
+        if not self.question(msg):
+            return
         attempts = LN_NUM_PAYMENT_ATTEMPTS
         def task():
-            self.wallet.lnworker.pay(invoice, amount_sat, attempts)
+            self.wallet.lnworker.pay(invoice, amount_msat=amount_msat, attempts=attempts)
         self.do_clear()
         self.wallet.thread.add(task)
         self.invoice_list.update()
@@ -1431,43 +1519,49 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
             self.notify(_('Payment received') + '\n' + key)
             self.need_update.set()
 
-    def on_invoice_status(self, key, status):
+    def on_invoice_status(self, key):
-        if key not in self.wallet.invoices:
+        req = self.wallet.get_invoice(key)
+        if req is None:
             return
-        self.invoice_list.update_item(key, status)
+               self.invoice_list.update_item(key, req)
-        if status == PR_PAID:
+
-            self.show_message(_('Payment succeeded'))
+    def on_payment_succeeded(self, wallet, key):
+        description = self.wallet.get_label(key)
+        self.notify(_('Payment succeeded') + '\n\n' + description)
         self.need_update.set()
-        elif status == PR_FAILED:
+
-            self.show_error(_('Payment failed'))
+    def on_payment_failed(self, wallet, key, reason):
-        else:
+        self.show_error(_('Payment failed') + '\n\n' + reason)
-            pass
 
     def read_invoice(self):
         if self.check_send_tab_payto_line_and_show_errors():
             return
         if not self._is_onchain:
-            invoice = self.payto_e.lightning_invoice
+            invoice_str = self.payto_e.lightning_invoice
-            if not invoice:
+            if not invoice_str:
                 return
-            if not self.wallet.lnworker:
+            if not self.wallet.has_lightning():
                 self.show_error(_('Lightning is disabled'))
                 return
-            invoice_dict = self.wallet.lnworker.parse_bech32_invoice(invoice)
+            invoice = LNInvoice.from_bech32(invoice_str)
-            if invoice_dict.get('amount') is None:
+            if invoice.get_amount_msat() is None:
-                amount = self.amount_e.get_amount()
+                amount_sat = self.amount_e.get_amount()
-                if amount:
+                if amount_sat:
-                    invoice_dict['amount'] = amount
+                    invoice.amount_msat = int(amount_sat * 1000)
                 else:
                     self.show_error(_('No amount'))
                     return
-            return invoice_dict
+            return invoice
         else:
             outputs = self.read_outputs()
             if self.check_send_tab_onchain_outputs_and_show_errors(outputs):
                 return
             message = self.message_e.text()
-            return self.wallet.create_invoice(outputs, message, self.payment_request, self.payto_URI)
+            return self.wallet.create_invoice(
+                outputs=outputs,
+                message=message,
+                pr=self.payment_request,
+                URI=self.payto_URI)
 
     def do_save_invoice(self):
         invoice = self.read_invoice()
@@ -1489,15 +1583,16 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
     def pay_multiple_invoices(self, invoices):
         outputs = []
         for invoice in invoices:
-            outputs += invoice['outputs']
+            outputs += invoice.outputs
         self.pay_onchain_dialog(self.get_coins(), outputs)
 
-    def do_pay_invoice(self, invoice):
+    def do_pay_invoice(self, invoice: 'Invoice'):
-        if invoice['type'] == PR_TYPE_LN:
+        if invoice.type == PR_TYPE_LN:
-            self.pay_lightning_invoice(invoice['invoice'], amount_sat=invoice['amount'])
+            assert isinstance(invoice, LNInvoice)
-        elif invoice['type'] == PR_TYPE_ONCHAIN:
+            self.pay_lightning_invoice(invoice.invoice, amount_msat=invoice.get_amount_msat())
-            outputs = invoice['outputs']
+        elif invoice.type == PR_TYPE_ONCHAIN:
-            self.pay_onchain_dialog(self.get_coins(), outputs)
+            assert isinstance(invoice, OnchainInvoice)
+            self.pay_onchain_dialog(self.get_coins(), invoice.outputs)
         else:
             raise Exception('unknown invoice type')
 
@@ -1531,16 +1626,22 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         if output_values.count('!') > 1:
             self.show_error(_("More than one output set to spend max"))
             return
+
+        output_value = '!' if '!' in output_values else sum(output_values)
+        d = ConfirmTxDialog(window=self, make_tx=make_tx, output_value=output_value, is_sweep=is_sweep)
+        if d.not_enough_funds:
+            # Check if we had enough funds excluding fees,
+            # if so, still provide opportunity to set lower fees.
+            if not d.have_enough_funds_assuming_zero_fees():
+                self.show_message(_('Not Enough Funds'))
+                return
+
+        # shortcut to advanced preview (after "enough funds" check!)
         if self.config.get('advanced_preview'):
             self.preview_tx_dialog(make_tx=make_tx,
                                    external_keypairs=external_keypairs)
             return
 
-        output_value = '!' if '!' in output_values else sum(output_values)
-        d = ConfirmTxDialog(window=self, make_tx=make_tx, output_value=output_value, is_sweep=is_sweep)
-        if d.not_enough_funds:
-            self.show_message(_('Not Enough Funds'))
-            return
         cancelled, is_send, password, tx = d.run()
         if cancelled:
             return
@@ -1646,6 +1747,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         # however, the user must not be allowed to broadcast early
         make_tx = self.mktx_for_open_channel(funding_sat)
         d = ConfirmTxDialog(window=self, make_tx=make_tx, output_value=funding_sat, is_sweep=False)
+        d.preview_button.setEnabled(False)
         cancelled, is_send, password, funding_tx = d.run()
         if not is_send:
             return
@@ -1675,7 +1777,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
 
         def on_failure(exc_info):
             type_, e, traceback = exc_info
-            self.show_error(_('Could not open channel: {}').format(e))
+            self.show_error(_('Could not open channel: {}').format(repr(e)))
         WaitingDialog(self, _('Opening channel...'), task, on_success, on_failure)
 
     def query_choice(self, msg, choices):
@@ -1702,7 +1804,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.payto_e.setText(_("please wait..."))
         return True
 
-    def delete_invoice(self, key):
+    def delete_invoices(self, keys):
+        for key in keys:
             self.wallet.delete_invoice(key)
         self.invoice_list.update()
 
@@ -1712,7 +1815,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
             return
         key = pr.get_id()
         invoice = self.wallet.get_invoice(key)
-        if invoice and invoice['status'] == PR_PAID:
+        if invoice and self.wallet.get_invoice_status(invoice) == PR_PAID:
             self.show_message("invoice already paid")
             self.do_clear()
             self.payment_request = None
@@ -1723,7 +1826,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         else:
             self.payto_e.setExpired()
         self.payto_e.setText(pr.get_requestor())
-        self.amount_e.setText(format_satoshis_plain(pr.get_amount(), self.decimal_point))
+        self.amount_e.setAmount(pr.get_amount())
         self.message_e.setText(pr.get_memo())
         # signal to set fee
         self.amount_e.textEdited.emit("")
@@ -1746,7 +1849,7 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
 
     def parse_lightning_invoice(self, invoice):
         """Parse ln invoice, and prepare the send tab for it."""
-        from electrum.lnaddr import lndecode, LnDecodeException
+
         try:
             lnaddr = lndecode(invoice, expected_hrp=constants.net.SEGWIT_HRP)
         except Exception as e:
@@ -1761,8 +1864,8 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.payto_e.setFrozen(True)
         self.payto_e.setText(pubkey)
         self.message_e.setText(description)
-        if lnaddr.amount is not None:
+        if lnaddr.get_amount_sat() is not None:
-            self.amount_e.setAmount(lnaddr.amount * COIN)
+            self.amount_e.setAmount(lnaddr.get_amount_sat())
         #self.amount_e.textEdited.emit("")
         self.set_onchain(False)
 
@@ -1854,8 +1957,13 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         return self.create_list_tab(l)
 
     def remove_address(self, addr):
-        if self.question(_("Do you want to remove {} from your wallet?").format(addr)):
+        if not self.question(_("Do you want to remove {} from your wallet?").format(addr)):
+            return
+        try:
             self.wallet.delete_address(addr)
+        except UserFacingException as e:
+            self.show_error(str(e))
+        else:
             self.need_update.set()  # history, addresses, coins
             self.clear_receive_tab()
 
@@ -1902,16 +2010,48 @@ class ElectrumWindow(QMainWindow, MessageBoxMixin, Logger):
         self.contact_list.update()
         self.update_completions()
 
-    def show_invoice(self, key):
+    def show_onchain_invoice(self, invoice: OnchainInvoice):
-        invoice = self.wallet.get_invoice(key)
+        amount_str = self.format_amount(invoice.amount_sat) + ' ' + self.base_unit()
-        if invoice is None:
+        d = WindowModalDialog(self, _("Onchain Invoice"))
-            self.show_error('Cannot find payment request in wallet.')
+        vbox = QVBoxLayout(d)
-            return
+        grid = QGridLayout()
-        bip70 = invoice.get('bip70')
+        grid.addWidget(QLabel(_("Amount") + ':'), 1, 0)
-        if bip70:
+        grid.addWidget(QLabel(amount_str), 1, 1)
-            pr = paymentrequest.PaymentRequest(bytes.fromhex(bip70))
+        if len(invoice.outputs) == 1:
+            grid.addWidget(QLabel(_("Address") + ':'), 2, 0)
+            grid.addWidget(QLabel(invoice.get_address()), 2, 1)
+        else:
+            outputs_str = '\n'.join(map(lambda x: x.address + ' : ' + self.format_amount(x.value)+ self.base_unit(), invoice.outputs))
+            grid.addWidget(QLabel(_("Outputs") + ':'), 2, 0)
+            grid.addWidget(QLabel(outputs_str), 2, 1)
+        grid.addWidget(QLabel(_("Description") + ':'), 3, 0)
+        grid.addWidget(QLabel(invoice.message), 3, 1)
+        if invoice.exp:
+            grid.addWidget(QLabel(_("Expires") + ':'), 4, 0)
+            grid.addWidget(QLabel(format_time(invoice.exp + invoice.time)), 4, 1)
+        if invoice.bip70:
+            pr = paymentrequest.PaymentRequest(bytes.fromhex(invoice.bip70))
             pr.verify(self.contacts)
-            self.show_bip70_details(pr)
+            grid.addWidget(QLabel(_("Requestor") + ':'), 5, 0)
+            grid.addWidget(QLabel(pr.get_requestor()), 5, 1)
+            grid.addWidget(QLabel(_("Signature") + ':'), 6, 0)
+            grid.addWidget(QLabel(pr.get_verify_status()), 6, 1)
+            def do_export():
+                key = pr.get_id()
+                name = str(key) + '.bip70'
+                fn = self.getSaveFileName(_("Save invoice to file"), name, filter="*.bip70")
+                if not fn:
+                    return
+                with open(fn, 'wb') as f:
+                    data = f.write(pr.raw)
+                self.show_message(_('BIP70 invoice saved as' + ' ' + fn))
+            exportButton = EnterButton(_('Export'), do_export)
+            buttons = Buttons(exportButton, CloseButton(d))
+        else:
+            buttons = Buttons(CloseButton(d))
+        vbox.addLayout(grid)
+        vbox.addLayout(buttons)
+        d.exec_()
 
     def show_bip70_details(self, pr: 'paymentrequest.PaymentRequest'):
         key = pr.get_id()