Merge remote-tracking branch 'upstream/master'

RELEASE-NOTES

@@ -1,3 +1,12 @@
+# Release 3.3.8 - (July 11, 2019)
+
+ * fix some bugs with recent bump fee (RBF) improvements (#5483, #5502)
+ * fix #5491: watch-only wallets could not bump fee in some cases
+ * appimage: URLs could not be opened on some desktop environments (#5425)
+ * faster tx signing for segwit inputs for really large txns (#5494)
+ * A few other minor bugfixes and usability improvements.
+
+
 # Release 3.3.7 - (July 3, 2019)
 
  * The AppImage Linux x86_64 binary and the Windows setup.exe

contrib/build-linux/appimage/Dockerfile

@@ -21,7 +21,7 @@ RUN apt-get update -q && \
         libudev-dev=204-5ubuntu20.31 \
         gettext=0.18.3.1-1ubuntu3.1 \
         libzbar0=0.10+doc-9build1  \
-        faketime=0.9.5-2 \
+        libdbus-1-3=1.6.18-0ubuntu4.5 \
         && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get autoremove -y && \

contrib/build-linux/appimage/build.sh

@@ -132,10 +132,10 @@ info "preparing electrum-locale."
 
 info "installing electrum and its dependencies."
 mkdir -p "$CACHEDIR/pip_cache"
-"$python" -m pip install --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
-"$python" -m pip install --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
-"$python" -m pip install --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
-"$python" -m pip install --cache-dir "$CACHEDIR/pip_cache" "$PROJECT_ROOT"
+"$python" -m pip install --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements.txt"
+"$python" -m pip install --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-binaries.txt"
+"$python" -m pip install --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" -r "$CONTRIB/deterministic-build/requirements-hw.txt"
+"$python" -m pip install --no-warn-script-location --cache-dir "$CACHEDIR/pip_cache" "$PROJECT_ROOT"
 
 
 info "copying zbar"
@@ -157,12 +157,8 @@ info "finalizing AppDir."
 
     cd "$APPDIR"
     # copy system dependencies
-    # note: temporarily move PyQt5 out of the way so
-    # we don't try to bundle its system dependencies.
-    mv "$APPDIR/usr/lib/python3.6/site-packages/PyQt5" "$BUILDDIR"
     copy_deps; copy_deps; copy_deps
     move_lib
-    mv "$BUILDDIR/PyQt5" "$APPDIR/usr/lib/python3.6/site-packages"
 
     # apply global appimage blacklist to exclude stuff
     # move usr/include out of the way to preserve usr/include/python3.6m.
@@ -171,10 +167,12 @@ info "finalizing AppDir."
     mv usr/include.tmp usr/include
 ) || fail "Could not finalize AppDir"
 
-# copy libusb here because it is on the AppImage excludelist and it can cause problems if we use system libusb
-info "Copying libusb"
-cp -f /usr/lib/x86_64-linux-gnu/libusb-1.0.so "$APPDIR/usr/lib/libusb-1.0.so" || fail "Could not copy libusb"
-
+# We copy some libraries here that are on the AppImage excludelist
+info "Copying additional libraries"
+(
+    # On some systems it can cause problems to use the system libusb
+    cp -f /usr/lib/x86_64-linux-gnu/libusb-1.0.so "$APPDIR/usr/lib/libusb-1.0.so" || fail "Could not copy libusb"
+)
 
 info "stripping binaries from debug symbols."
 # "-R .note.gnu.build-id" also strips the build id
@@ -231,8 +229,11 @@ find -exec touch -h -d '2000-11-11T11:11:11+00:00' {} +
 info "creating the AppImage."
 (
     cd "$BUILDDIR"
-    chmod +x "$CACHEDIR/appimagetool"
-    "$CACHEDIR/appimagetool" --appimage-extract
+    cp "$CACHEDIR/appimagetool" "$CACHEDIR/appimagetool_copy"
+    # zero out "appimage" magic bytes, as on some systems they confuse the linker
+    sed -i 's|AI\x02|\x00\x00\x00|' "$CACHEDIR/appimagetool_copy"
+    chmod +x "$CACHEDIR/appimagetool_copy"
+    "$CACHEDIR/appimagetool_copy" --appimage-extract
     # We build a small wrapper for mksquashfs that removes the -mkfs-fixed-time option
     # that mksquashfs from squashfskit does not support. It is not needed for squashfskit.
     cat > ./squashfs-root/usr/lib/appimagekit/mksquashfs << EOF

contrib/build-wine/build-electrum-git.sh

@@ -4,6 +4,7 @@ NAME_ROOT=electrum-mona
 
 # These settings probably don't need any change
 export WINEPREFIX=/opt/wine64
+export WINEDEBUG=-all
 export PYTHONDONTWRITEBYTECODE=1
 export PYTHONHASHSEED=22
 
@@ -42,14 +43,14 @@ popd
 
 
 # Install frozen dependencies
-$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements.txt
+$PYTHON -m pip install --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements.txt
 
-$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements-hw.txt
+$PYTHON -m pip install --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-hw.txt
 
 pushd $WINEPREFIX/drive_c/electrum-mona
 # see https://github.com/pypa/pip/issues/2195 -- pip makes a copy of the entire directory
 info "Pip installing Electrum. This might take a long time if the project folder is large."
-$PYTHON -m pip install .
+$PYTHON -m pip install --no-warn-script-location .
 popd
 
 
@@ -76,4 +77,46 @@ cd dist
 mv electrum-mona-setup.exe $NAME_ROOT-$VERSION-setup.exe
 cd ..
 
+info "Padding binaries to 8-byte boundaries, and fixing COFF image checksum in PE header"
+# note: 8-byte boundary padding is what osslsigncode uses:
+#       https://github.com/mtrojnar/osslsigncode/blob/6c8ec4427a0f27c145973450def818e35d4436f6/osslsigncode.c#L3047
+(
+    cd dist
+    for binary_file in ./*.exe; do
+        info ">> fixing $binary_file..."
+        # code based on https://github.com/erocarrera/pefile/blob/bbf28920a71248ed5c656c81e119779c131d9bd4/pefile.py#L5877
+        python3 <<EOF
+pe_file = "$binary_file"
+with open(pe_file, "rb") as f:
+    binary = bytearray(f.read())
+pe_offset = int.from_bytes(binary[0x3c:0x3c+4], byteorder="little")
+checksum_offset = pe_offset + 88
+checksum = 0
+
+# Pad data to 8-byte boundary.
+remainder = len(binary) % 8
+binary += bytes(8 - remainder)
+
+for i in range(len(binary) // 4):
+    if i == checksum_offset // 4:  # Skip the checksum field
+        continue
+    dword = int.from_bytes(binary[i*4:i*4+4], byteorder="little")
+    checksum = (checksum & 0xffffffff) + dword + (checksum >> 32)
+    if checksum > 2 ** 32:
+        checksum = (checksum & 0xffffffff) + (checksum >> 32)
+
+checksum = (checksum & 0xffff) + (checksum >> 16)
+checksum = (checksum) + (checksum >> 16)
+checksum = checksum & 0xffff
+checksum += len(binary)
+
+# Set the checksum
+binary[checksum_offset : checksum_offset + 4] = int.to_bytes(checksum, byteorder="little", length=4)
+
+with open(pe_file, "wb") as f:
+    f.write(binary)
+EOF
+    done
+)
+
 sha256sum dist/electrum*.exe

contrib/build-wine/prepare-wine.sh

@@ -23,7 +23,7 @@ PYTHON_VERSION=3.6.8
 
 ## These settings probably don't need change
 export WINEPREFIX=/opt/wine64
-#export WINEARCH='win32'
+export WINEDEBUG=-all
 
 PYTHON_FOLDER="python3"
 PYHOME="c:/$PYTHON_FOLDER"
@@ -60,7 +60,7 @@ done
 
 info "Installing dependencies specific to binaries."
 # note that this also installs pinned versions of both pip and setuptools
-$PYTHON -m pip install -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
+$PYTHON -m pip install --no-warn-script-location -r "$CONTRIB"/deterministic-build/requirements-binaries.txt
 
 info "Installing ZBar."
 download_if_not_exist "$CACHEDIR/$ZBAR_FILENAME" "$ZBAR_URL"
@@ -112,6 +112,6 @@ info "Building PyInstaller."
     [[ -e PyInstaller/bootloader/Windows-32bit/runw.exe ]] || fail "Could not find runw.exe in target dir!"
 ) || fail "PyInstaller build failed"
 info "Installing PyInstaller."
-$PYTHON -m pip install ./pyinstaller
+$PYTHON -m pip install --no-warn-script-location ./pyinstaller
 
 info "Wine is configured."

contrib/build-wine/unsign.sh

@@ -24,28 +24,8 @@ for mine in $(ls dist/*.exe); do
     echo "Downloading https://electrum-mona.org/$version/$f"
     wget -q https://electrum-mona.org/$version/$f -O signed/$f
     out="signed/stripped/$f"
-    size=$( wc -c < $mine )
-    # Step 1: Remove PE signature from signed binary
+    # Remove PE signature from signed binary
     osslsigncode remove-signature -in signed/$f -out $out > /dev/null 2>&1
-    # Step 2: Remove checksum and padding from signed binary
-    python3 <<EOF
-pe_file = "$out"
-size= $size
-with open(pe_file, "rb") as f:
-    binary = bytearray(f.read())
-pe_offset = int.from_bytes(binary[0x3c:0x3c+4], byteorder="little")
-checksum_offset = pe_offset + 88
-for b in range(4):
-    binary[checksum_offset + b] = 0
-l = len(binary)
-n = l - size
-if n > 0:
-   if binary[-n:] != bytearray(n):
-       print('expecting failure for', str(pe_file))
-   binary = binary[:size]
-with open(pe_file, "wb") as f:
-    f.write(binary)
-EOF
     chmod +x $out
     if cmp -s $out $mine; then
 	echo "Success: $f"

contrib/make_download

@@ -2,8 +2,15 @@
 import re
 import os
 import sys
+import importlib
 
-from electrum_mona.version import ELECTRUM_VERSION, APK_VERSION
+# load version.py; needlessly complicated alternative to "imp.load_source":
+version_spec = importlib.util.spec_from_file_location('version', 'electrum_mona/version.py')
+version_module = importlib.util.module_from_spec(version_spec)
+version_spec.loader.exec_module(version_module)
+
+ELECTRUM_VERSION = version_module.ELECTRUM_VERSION
+APK_VERSION = version_module.APK_VERSION
 print("version", ELECTRUM_VERSION)
 
 dirname = sys.argv[1]

contrib/upload

@@ -2,16 +2,17 @@
 
 set -e
 
+host=$1
 version=`git describe --tags`
 echo $version
 
 here=$(dirname "$0")
 cd $here/../dist
 
-sftp -oBatchMode=no -b - thomasv@download.electrum.org << !
+sftp -oBatchMode=no -b - thomasv@$host << !
    cd electrum-downloads
    mkdir $version
    cd $version
    mput *
    bye
-!
+!

electrum_mona/gui/kivy/tools/buildozer.spec

@@ -87,6 +87,9 @@ android.ndk_path = /opt/android/android-ndk
 # (str) Android SDK directory (if empty, it will be automatically downloaded.)
 android.sdk_path = /opt/android/android-sdk
 
+# (str) ANT directory (if empty, it will be automatically downloaded.)
+#android.ant_path =
+
 # (str) Android entry point, default is ok for Kivy-based app
 #android.entrypoint = org.renpy.android.PythonActivity
 
@@ -129,6 +132,9 @@ android.manifest.launch_mode = singleTask
 # Don't forget to add the WAKE_LOCK permission if you set this to True
 #android.wakelock = False
 
+# (str) The Android arch to build for, choices: armeabi-v7a, arm64-v8a, x86, x86_64
+android.arch = armeabi-v7a
+
 # (list) Android application meta-data to set (key=value format)
 #android.meta_data =
 
@@ -138,9 +144,27 @@ android.manifest.launch_mode = singleTask
 
 android.whitelist = lib-dynload/_csv.so
 
-# local version that merges branch 866
+
+#
+# Python for android (p4a) specific
+#
+
+# (str) python-for-android git clone directory (if empty, it will be automatically cloned from github)
 p4a.source_dir = /opt/python-for-android
 
+# (str) The directory in which python-for-android should look for your own build recipes (if any)
+#p4a.local_recipes =
+
+# (str) Filename to the hook for p4a
+#p4a.hook =
+
+# (str) Bootstrap to use for android builds
+# p4a.bootstrap = sdl2
+
+# (int) port number to specify an explicit --port= p4a argument (eg for bootstrap flask)
+#p4a.port =
+
+
 #
 # iOS specific
 #

electrum_mona/gui/qt/address_list.py

@@ -23,7 +23,6 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import webbrowser
 from enum import IntEnum
 
 from PyQt5.QtCore import Qt, QPersistentModelIndex, QModelIndex
@@ -36,7 +35,7 @@
 from electrum_mona.bitcoin import is_address
 from electrum_mona.wallet import InternalAddressCorruption
 
-from .util import MyTreeView, MONOSPACE_FONT, ColorScheme
+from .util import MyTreeView, MONOSPACE_FONT, ColorScheme, webopen
 
 
 class AddressList(MyTreeView):
@@ -217,7 +216,7 @@ def create_menu(self, position):
                 menu.addAction(_("Remove from wallet"), lambda: self.parent.remove_address(addr))
             addr_URL = block_explorer_URL(self.config, 'addr', addr)
             if addr_URL:
-                menu.addAction(_("View on block explorer"), lambda: webbrowser.open(addr_URL))
+                menu.addAction(_("View on block explorer"), lambda: webopen(addr_URL))
 
             if not self.wallet.is_frozen_address(addr):
                 menu.addAction(_("Freeze"), lambda: self.parent.set_frozen_state_of_addresses([addr], True))

electrum_mona/gui/qt/contact_list.py

@@ -23,7 +23,6 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import webbrowser
 from enum import IntEnum
 
 from PyQt5.QtGui import QStandardItemModel, QStandardItem
@@ -35,7 +34,7 @@
 from electrum_mona.util import block_explorer_URL
 from electrum_mona.plugin import run_hook
 
-from .util import MyTreeView, import_meta_gui, export_meta_gui
+from .util import MyTreeView, import_meta_gui, export_meta_gui, webopen
 
 
 class ContactList(MyTreeView):
@@ -97,7 +96,7 @@ def create_menu(self, position):
             menu.addAction(_("Delete"), lambda: self.parent.delete_contacts(selected_keys))
             URLs = [block_explorer_URL(self.config, 'addr', key) for key in filter(is_address, selected_keys)]
             if URLs:
-                menu.addAction(_("View on block explorer"), lambda: [webbrowser.open(u) for u in URLs])
+                menu.addAction(_("View on block explorer"), lambda: [webopen(u) for u in URLs])
 
         run_hook('create_contact_menu', menu, selected_keys)
         menu.exec_(self.viewport().mapToGlobal(position))

electrum_mona/gui/qt/history_list.py

@@ -24,7 +24,6 @@
 # SOFTWARE.
 
 import os
-import webbrowser
 import datetime
 from datetime import date
 from typing import TYPE_CHECKING, Tuple, Dict
@@ -47,7 +46,7 @@
 
 from .util import (read_QIcon, MONOSPACE_FONT, Buttons, CancelButton, OkButton,
                    filename_field, MyTreeView, AcceptFileDragDrop, WindowModalDialog,
-                   CloseButton)
+                   CloseButton, webopen)
 
 if TYPE_CHECKING:
     from electrum_mona.wallet import Abstract_Wallet
@@ -609,7 +608,7 @@ def create_menu(self, position: QPoint):
         if pr_key:
             menu.addAction(read_QIcon("seal"), _("View invoice"), lambda: self.parent.show_invoice(pr_key))
         if tx_URL:
-            menu.addAction(_("View on block explorer"), lambda: webbrowser.open(tx_URL))
+            menu.addAction(_("View on block explorer"), lambda: webopen(tx_URL))
         menu.exec_(self.viewport().mapToGlobal(position))
 
     def remove_local_tx(self, delete_tx):