build: create a standalone build script for libsecp256k1

heavily based on Electron-Cash/Electron-Cash@eda015908e

MANIFEST.in

@@ -12,6 +12,9 @@ graft electrum
 prune electrum/tests
 graft contrib/udev
 
+exclude electrum/*.so
+exclude electrum/*.so.0
+
 global-exclude __pycache__
 global-exclude *.py[co~]
 global-exclude *.py.orig

contrib/build-linux/appimage/build.sh

@@ -10,10 +10,11 @@ BUILDDIR="$CONTRIB_APPIMAGE/build/appimage"
 APPDIR="$BUILDDIR/electrum.AppDir"
 CACHEDIR="$CONTRIB_APPIMAGE/.cache/appimage"
 
+export GCC_STRIP_BINARIES="1"
+
 # pinned versions
 PYTHON_VERSION=3.7.6
 PKG2APPIMAGE_COMMIT="eb8f3acdd9f11ab19b78f5cb15daa772367daf15"
-LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
 SQUASHFSKIT_COMMIT="ae0d656efa2d0df2fcac795b6823b44462f19386"
 
 
@@ -45,7 +46,6 @@ info "building python."
 tar xf "$CACHEDIR/Python-$PYTHON_VERSION.tar.xz" -C "$BUILDDIR"
 (
     cd "$BUILDDIR/Python-$PYTHON_VERSION"
-    export SOURCE_DATE_EPOCH=1530212462
     LC_ALL=C export BUILD_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%b %d %Y")
     LC_ALL=C export BUILD_TIME=$(date -u -d "@$SOURCE_DATE_EPOCH" "+%H:%M:%S")
     # Patch taken from Ubuntu http://archive.ubuntu.com/ubuntu/pool/main/p/python3.7/python3.7_3.7.6-1.debian.tar.xz
@@ -77,26 +77,8 @@ git clone "https://github.com/squashfskit/squashfskit.git" "$BUILDDIR/squashfski
 MKSQUASHFS="$BUILDDIR/squashfskit/squashfs-tools/mksquashfs"
 
 
-info "building libsecp256k1."
+"$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
-(
+cp -f "$PROJECT_ROOT/electrum/libsecp256k1.so.0" "$APPDIR/usr/lib/libsecp256k1.so.0" || fail "Could not copy libsecp to its destination"
-    git clone https://github.com/bitcoin-core/secp256k1 "$CACHEDIR"/secp256k1 \
-        || (cd "$CACHEDIR"/secp256k1 && git reset --hard && git pull)
-    cd "$CACHEDIR"/secp256k1
-    git reset --hard "$LIBSECP_VERSION"
-    git clean -f -x -q
-    export SOURCE_DATE_EPOCH=1530212462
-    echo "LDFLAGS = -no-undefined" >> Makefile.am
-    ./autogen.sh
-    ./configure \
-      --prefix="$APPDIR/usr" \
-      --enable-module-recovery \
-      --enable-experimental \
-      --enable-module-ecdh \
-      --disable-jni \
-      -q
-    make -j4 -s || fail "Could not build libsecp"
-    make -s install > /dev/null || fail "Could not install libsecp"
-)
 
 
 appdir_python() {
@@ -224,7 +206,6 @@ rm -rf "$PYDIR"/site-packages/PyQt5/Qt.so
 
 # these are deleted as they were not deterministic; and are not needed anyway
 find "$APPDIR" -path '*/__pycache__*' -delete
-rm "$APPDIR"/usr/lib/libsecp256k1.a
 # note that jsonschema-*.dist-info is needed by that package as it uses 'pkg_resources.get_distribution'
 # also, see https://gitlab.com/python-devs/importlib_metadata/issues/71
 for f in "$PYDIR"/site-packages/jsonschema-*.dist-info; do mv "$f" "$(echo "$f" | sed s/\.dist-info/\.dist-info2/)"; done

contrib/build-wine/build-electrum-git.sh

@@ -6,7 +6,6 @@ NAME_ROOT=electrum
 export WINEPREFIX=/opt/wine64
 export WINEDEBUG=-all
 export PYTHONDONTWRITEBYTECODE=1
-export PYTHONHASHSEED=22
 
 PYHOME=c:/python3
 PYTHON="wine $PYHOME/python.exe -OO -B"

contrib/build-wine/build-secp256k1.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-# heavily based on https://github.com/ofek/coincurve/blob/417e726f553460f88d7edfa5dc67bfda397c4e4a/.travis/build_windows_wheels.sh
-
-set -e
-
-here="$(dirname "$(readlink -e "$0")")"
-LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
-
-. "$CONTRIB"/build_tools_util.sh
-
-info "building libsecp256k1..."
-
-
-build_dll() {
-    #sudo apt-get install -y mingw-w64
-    export SOURCE_DATE_EPOCH=1530212462
-    echo "LDFLAGS = -no-undefined" >> Makefile.am
-    ./autogen.sh
-    # Note: set both --build and --host when running configure
-    # Otherwise weird voodoo magic happens with Docker and Wine.
-    # https://www.gnu.org/software/autoconf/manual/autoconf-2.69/html_node/Hosts-and-Cross_002dCompilation.html
-    LDFLAGS="-Wl,--no-insert-timestamp" ./configure \
-        --host=$1 \
-        --build=x86_64-pc-linux-gnu \
-        --enable-module-recovery \
-        --enable-experimental \
-        --enable-module-ecdh \
-        --disable-jni
-    make -j4
-    ${1}-strip .libs/libsecp256k1-0.dll
-}
-
-
-cd "$CACHEDIR"
-
-if [ -f "secp256k1/libsecp256k1.dll" ]; then
-    info "libsecp256k1.dll already built, skipping"
-    exit 0
-fi
-
-
-if [ ! -d secp256k1 ]; then
-    git clone https://github.com/bitcoin-core/secp256k1.git
-fi
-
-cd secp256k1
-git reset --hard
-git clean -f -x -q
-git checkout $LIBSECP_VERSION
-
-build_dll i686-w64-mingw32  # 64-bit would be: x86_64-w64-mingw32
-mv .libs/libsecp256k1-0.dll libsecp256k1.dll
-
-find -exec touch -d '2000-11-11T11:11:11+00:00' {} +
-
-info "building libsecp256k1 finished"

contrib/build-wine/build.sh

@@ -2,16 +2,19 @@
 
 set -e
 
-# Lucky number
-export PYTHONHASHSEED=22
-
 here="$(dirname "$(readlink -e "$0")")"
 test -n "$here" -a -d "$here" || exit
 
 export CONTRIB="$here/.."
+export PROJECT_ROOT="$CONTRIB/.."
 export CACHEDIR="$here/.cache"
 export PIP_CACHE_DIR="$CACHEDIR/pip_cache"
 
+export BUILD_TYPE="wine"
+export GCC_TRIPLET_HOST="i686-w64-mingw32"
+export GCC_TRIPLET_BUILD="x86_64-pc-linux-gnu"
+export GCC_STRIP_BINARIES="1"
+
 . "$CONTRIB"/build_tools_util.sh
 
 info "Clearing $here/build and $here/dist..."
@@ -20,7 +23,11 @@ rm "$here"/dist/* -rf
 
 mkdir -p "$CACHEDIR" "$PIP_CACHE_DIR"
 
-$here/build-secp256k1.sh || fail "build-secp256k1 failed"
+if [ -f "$PROJECT_ROOT/electrum/libsecp256k1-0.dll" ]; then
+    info "libsecp256k1 already built, skipping"
+else
+    "$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
+fi
 
 $here/prepare-wine.sh || fail "prepare-wine failed"
 

contrib/build-wine/deterministic.spec

@@ -34,7 +34,7 @@ binaries = []
 # Workaround for "Retro Look":
 binaries += [b for b in collect_dynamic_libs('PyQt5') if 'qwindowsvista' in b[0]]
 
-binaries += [('C:/tmp/libsecp256k1.dll', '.')]
+binaries += [('C:/tmp/libsecp256k1-0.dll', '.')]
 binaries += [('C:/tmp/libusb-1.0.dll', '.')]
 
 datas = [

contrib/build-wine/prepare-wine.sh

@@ -89,7 +89,6 @@ info "Compiling libusb..."
     git remote add origin $LIBUSB_REPO
     git fetch --depth 1 origin $LIBUSB_COMMIT
     git checkout -b pinned FETCH_HEAD
-    export SOURCE_DATE_EPOCH=1530212462
     echo "libusb_1_0_la_LDFLAGS += -Wc,-static" >> libusb/Makefile.am
     ./bootstrap.sh || fail "Could not bootstrap libusb"
     host="i686-w64-mingw32"
@@ -102,8 +101,8 @@ info "Compiling libusb..."
 cp "$CACHEDIR/libusb/libusb/.libs/libusb-1.0.dll" $WINEPREFIX/drive_c/tmp/  || fail "Could not copy libusb to its destination"
 
 
-# copy libsecp dll (already built by build-secp256k1.sh)
+# copy libsecp dll (already built)
-cp "$CACHEDIR/secp256k1/libsecp256k1.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libsecp to its destination"
+cp "$PROJECT_ROOT/electrum/libsecp256k1-0.dll" $WINEPREFIX/drive_c/tmp/ || fail "Could not copy libsecp to its destination"
 
 
 info "Building PyInstaller."

contrib/build_tools_util.sh

@@ -70,3 +70,58 @@ function retry() {
 
   return $result
 }
+
+function gcc_with_triplet()
+{
+    TRIPLET="$1"
+    CMD="$2"
+    shift 2
+    if [ -n "$TRIPLET" ] ; then
+        "$TRIPLET-$CMD" "$@"
+    else
+        "$CMD" "$@"
+    fi
+}
+
+function gcc_host()
+{
+    gcc_with_triplet "$GCC_TRIPLET_HOST" "$@"
+}
+
+function gcc_build()
+{
+    gcc_with_triplet "$GCC_TRIPLET_BUILD" "$@"
+}
+
+function host_strip()
+{
+    if [ "$GCC_STRIP_BINARIES" -ne "0" ] ; then
+        case "$BUILD_TYPE" in
+            linux|wine)
+                gcc_host strip "$@"
+                ;;
+            darwin)
+                # TODO: Strip on macOS?
+                ;;
+        esac
+    fi
+}
+
+
+
+export SOURCE_DATE_EPOCH=1530212462
+export PYTHONHASHSEED=22
+# Set the build type, overridden by wine build
+export BUILD_TYPE="${BUILD_TYPE:-$(uname | tr '[:upper:]' '[:lower:]')}"
+# No additional autoconf flags by default
+export AUTOCONF_FLAGS=""
+# Add host / build flags if the triplets are set
+if [ -n "$GCC_TRIPLET_HOST" ] ; then
+    export AUTOCONF_FLAGS="$AUTOCONF_FLAGS --host=$GCC_TRIPLET_HOST"
+fi
+if [ -n "$GCC_TRIPLET_BUILD" ] ; then
+    export AUTOCONF_FLAGS="$AUTOCONF_FLAGS --build=$GCC_TRIPLET_BUILD"
+fi
+
+export GCC_STRIP_BINARIES="${GCC_STRIP_BINARIES:-0}"
+

contrib/make_libsecp256k1.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
+
+set -e
+
+here=$(dirname $(realpath "$0" 2> /dev/null || grealpath "$0"))
+CONTRIB="$here"
+PROJECT_ROOT="$CONTRIB/.."
+
+. "$here"/build_tools_util.sh || (echo "Could not source build_tools_util.sh" && exit 1)
+
+pkgname="secp256k1"
+info "Building $pkgname..."
+
+(
+    cd $CONTRIB
+    if [ ! -d secp256k1 ]; then
+        git clone https://github.com/bitcoin-core/secp256k1.git
+    fi
+    cd secp256k1
+    git reset --hard
+    git clean -f -x -q
+    git checkout $LIBSECP_VERSION
+
+    if ! [ -x configure ] ; then
+        echo "libsecp256k1_la_LDFLAGS = -no-undefined" >> Makefile.am
+        echo "LDFLAGS = -no-undefined" >> Makefile.am
+        ./autogen.sh || fail "Could not run autogen for $pkgname. Please make sure you have automake and libtool installed, and try again."
+    fi
+    if ! [ -r config.status ] ; then
+        ./configure \
+            $AUTOCONF_FLAGS \
+            --prefix="$here/$pkgname/dist" \
+            --enable-module-recovery \
+            --enable-experimental \
+            --enable-module-ecdh \
+            --disable-jni \
+            --disable-tests \
+            --disable-static \
+            --enable-shared || fail "Could not configure $pkgname. Please make sure you have a C compiler installed and try again."
+    fi
+    make -j4 || fail "Could not build $pkgname"
+    make install || fail "Could not install $pkgname"
+    . "$here/$pkgname/dist/lib/libsecp256k1.la"
+    host_strip "$here/$pkgname/dist/lib/$dlname"
+    cp -fpv "$here/$pkgname/dist/lib/$dlname" "$PROJECT_ROOT/electrum" || fail "Could not copy the $pkgname binary to its destination"
+    info "$dlname has been placed in the inner 'electrum' folder."
+)

contrib/osx/make_osx

@@ -7,6 +7,8 @@ PACKAGE=Electrum
 GIT_REPO=https://github.com/spesmilo/electrum
 LIBSECP_VERSION="b408c6a8b287003d1ade5709e6f7bc3c7f1d5be7"
 
+export GCC_STRIP_BINARIES="1"
+
 . $(dirname "$0")/base.sh
 
 CONTRIB_OSX="$(dirname "$(realpath "$0")")"
@@ -16,7 +18,6 @@ ROOT_FOLDER="$CONTRIB/.."
 src_dir=$(dirname "$0")
 cd $src_dir/../..
 
-export PYTHONHASHSEED=22
 VERSION=`git describe --tags --dirty --always`
 
 which brew > /dev/null 2>&1 || fail "Please install brew from https://brew.sh/ to continue"
@@ -96,17 +97,10 @@ cp $BUILDDIR/libusb/1.0.22/lib/libusb-1.0.dylib contrib/osx
 echo "82c368dfd4da017ceb32b12ca885576f325503428a4966cc09302cbd62702493  contrib/osx/libusb-1.0.dylib" | \
     shasum -a 256 -c || fail "libusb checksum mismatched"
 
-info "Building libsecp256k1"
+info "Preparing for building libsecp256k1"
 brew install autoconf automake libtool
-git clone https://github.com/bitcoin-core/secp256k1 $BUILDDIR/secp256k1
+"$CONTRIB"/make_libsecp256k1.sh || fail "Could not build libsecp"
-pushd $BUILDDIR/secp256k1
+cp "$ROOT_FOLDER"/electrum/libsecp256k1.0.dylib contrib/osx
-git reset --hard $LIBSECP_VERSION
-git clean -f -x -q
-./autogen.sh
-./configure --enable-module-recovery --enable-experimental --enable-module-ecdh --disable-jni
-make -j4
-popd
-cp $BUILDDIR/secp256k1/.libs/libsecp256k1.0.dylib contrib/osx
 
 info "Building CalinsQRReader..."
 d=contrib/osx/CalinsQRReader

electrum/ecc_fast.py

@@ -38,15 +38,25 @@ class LibModuleMissing(Exception): pass
 
 def load_library():
     if sys.platform == 'darwin':
-        library_path = 'libsecp256k1.0.dylib'
+        library_paths = (os.path.join(os.path.dirname(__file__), 'libsecp256k1.0.dylib'),
+                         'libsecp256k1.0.dylib')
     elif sys.platform in ('windows', 'win32'):
-        library_path = 'libsecp256k1.dll'
+        library_paths = (os.path.join(os.path.dirname(__file__), 'libsecp256k1-0.dll'),
+                         'libsecp256k1-0.dll')
     elif 'ANDROID_DATA' in os.environ:
-        library_path = 'libsecp256k1.so'
+        library_paths = ('libsecp256k1.so',)
-    else:
+    else:  # desktop Linux and similar
-        library_path = 'libsecp256k1.so.0'
+        library_paths = (os.path.join(os.path.dirname(__file__), 'libsecp256k1.so.0'),
+                         'libsecp256k1.so.0')
 
-    secp256k1 = ctypes.cdll.LoadLibrary(library_path)
+    secp256k1 = None
+    for libpath in library_paths:
+        try:
+            secp256k1 = ctypes.cdll.LoadLibrary(libpath)
+        except:
+            pass
+        else:
+            break
     if not secp256k1:
         _logger.error('libsecp256k1 library failed to load')
         return None