rewrite release.py script

build/changelog.py

@@ -1,8 +1,5 @@
-import argparse
 import datetime
 import re
-import sys
-
 
 CHANGELOG_START_RE = re.compile(r'^\#\# \[Unreleased\]')
 CHANGELOG_END_RE = re.compile(r'^\#\# \[.*\] - \d{4}-\d{2}-\d{2}')
@@ -14,118 +11,110 @@ EMPTY_RE = re.compile(r'^\w*\*\w*$')
 ENTRY_RE = re.compile(r'\* (.*)')
 VALID_SECTIONS = ['Added', 'Changed', 'Deprecated', 'Removed', 'Fixed', 'Security']
 
-
 # allocate some entries to cut-down on merge conflicts
 TEMPLATE = """### Added
   *
   *
-  *
 
 ### Changed
   *
   *
-  *
 
 ### Fixed
   *
   *
-  *
 
 """
 
 
-def main():
+class Changelog(object):
-    print "i am broken"
+    def __init__(self, path):
-    return 1
+        self.path = path
-    parser = argparse.ArgumentParser()
+        self.start = []
-    parser.add_argument('changelog')
+        self.unreleased = []
-    parser.add_argument('version')
+        self.rest = []
-    args = parser.parse_args()
+        self._parse()
-    bump(changelog, version)
 
+    def _parse(self):
+        with open(self.path) as fp:
+            lines = fp.readlines()
 
-def bump(changelog, version):
+        unreleased_start_found = False
-    with open(changelog) as fp:
+        unreleased_end_found = False
-        lines = fp.readlines()
 
-    start = []
+        for line in lines:
-    unreleased = []
+            if not unreleased_start_found:
-    rest = []
+                self.start.append(line)
-    unreleased_start_found = False
+                if CHANGELOG_START_RE.search(line):
-    unreleased_end_found = False
+                    unreleased_start_found = True
-    for line in lines:
+                continue
-        if not unreleased_start_found:
+            if unreleased_end_found:
-            start.append(line)
+                self.rest.append(line)
-            if CHANGELOG_START_RE.search(line):
+                continue
-                unreleased_start_found = True
+            if CHANGELOG_END_RE.search(line):
-            continue
+                self.rest.append(line)
-        if unreleased_end_found:
+                unreleased_end_found = True
-            rest.append(line)
+                continue
-            continue
+            if CHANGELOG_ERROR_RE.search(line):
-        if CHANGELOG_END_RE.search(line):
+                raise Exception(
-            rest.append(line)
+                    'Failed to parse {}: {}'.format(self.path, 'unexpected section header found'))
-            unreleased_end_found = True
+            self.unreleased.append(line)
-            continue
-        if CHANGELOG_ERROR_RE.search(line):
-            raise Exception(
-                'Failed to parse {}: {}'.format(changelog, 'unexpected section header found'))
-        unreleased.append(line)
 
-    today = datetime.datetime.today()
+        self.unreleased = self._normalize_section(self.unreleased)
-    header = '## [{}] - {}\n'.format(version, today.strftime('%Y-%m-%d'))
-    released = normalize(unreleased)
-    if not released:
-        # If we don't have anything in the Unreleased section, then leave the
-        # changelog as it is and return None
-        return
 
-    changelog_data = (
+    @staticmethod
-        ''.join(start) +
+    def _normalize_section(lines):
-        TEMPLATE +
+        """Parse a changelog entry and output a normalized form"""
-        header +
+        sections = {}
-        '\n'.join(released) + '\n\n'
+        current_section_name = None
-        + ''.join(rest)
+        current_section_contents = []
-    )
+        for line in lines:
-    with open(changelog, 'w') as fp:
+            line = line.strip()
-        fp.write(changelog_data)
+            if not line or EMPTY_RE.match(line):
-    return '\n'.join(released) + '\n\n'
+                continue
+            match = SECTION_RE.match(line)
+            if match:
+                if current_section_contents:
+                    sections[current_section_name] = current_section_contents
+                current_section_contents = []
+                current_section_name = match.group(1)
+                if current_section_name not in VALID_SECTIONS:
+                    raise ValueError("Section '{}' is not valid".format(current_section_name))
+                continue
+            match = ENTRY_RE.match(line)
+            if match:
+                current_section_contents.append(match.group(1))
+                continue
+            raise Exception('Something is wrong with line: {}'.format(line))
+        if current_section_contents:
+            sections[current_section_name] = current_section_contents
 
+        output = []
+        for section in VALID_SECTIONS:
+            if section not in sections:
+                continue
+            output.append('### {}'.format(section))
+            for entry in sections[section]:
+                output.append(' * {}'.format(entry))
+        return output
 
-def normalize(lines):
+    def get_unreleased(self):
-    """Parse a changelog entry and output a normalized form"""
+        return '\n'.join(self.unreleased) if self.unreleased else None
-    sections = {}
-    current_section_name = None
-    current_section_contents = []
-    for line in lines:
-        line = line.strip()
-        if not line or EMPTY_RE.match(line):
-            continue
-        match = SECTION_RE.match(line)
-        if match:
-            if current_section_contents:
-                sections[current_section_name] = current_section_contents
-            current_section_contents = []
-            current_section_name = match.group(1)
-            if current_section_name not in VALID_SECTIONS:
-                raise ValueError("Section '{}' is not valid".format(current_section_name))
-            continue
-        match = ENTRY_RE.match(line)
-        if match:
-            current_section_contents.append(match.group(1))
-            continue
-        raise Exception('Something is wrong with line: {}'.format(line))
-    if current_section_contents:
-        sections[current_section_name] = current_section_contents
 
-    output = []
+    def bump(self, version):
-    for section in VALID_SECTIONS:
+        if not self.unreleased:
-        if section not in sections:
+            return
-            continue
-        output.append('### {}'.format(section))
-        for entry in sections[section]:
-            output.append(' * {}'.format(entry))
-    return output
 
+        today = datetime.datetime.today()
+        header = '## [{}] - {}\n'.format(version, today.strftime('%Y-%m-%d'))
 
-if __name__ == '__main__':
+        changelog_data = (
-    sys.exit(main())
+            ''.join(self.start) +
+            TEMPLATE +
+            header +
+            '\n'.join(self.unreleased) + '\n\n'
+            + ''.join(self.rest)
+        )
+
+        with open(self.path, 'w') as fp:
+            fp.write(changelog_data)

build/release.py

@@ -1,13 +1,11 @@
-"""Trigger a release.
+"""Bump version and create Github release
 
-This script is to be run locally (not on a build server).
+This script should be run locally, not on a build server.
 """
 import argparse
 import contextlib
-import logging
 import os
 import re
-import string
 import subprocess
 import sys
 
@@ -16,122 +14,131 @@ import github
 
 import changelog
 
-# TODO: ask bumpversion for these
+ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-LBRY_PARTS = ('major', 'minor', 'patch', 'release', 'candidate')
-LBRYUM_PARTS = ('major', 'minor', 'patch')
 
 
 def main():
-    parser = argparse.ArgumentParser()
+    bumpversion_parts = get_bumpversion_parts()
-    parser.add_argument(
-        "lbry_part", help="part of lbry version to bump",
-        choices=LBRY_PARTS
-    )
-    parser.add_argument(
-        "--skip-lbryum", help="skip bumping lbryum, even if there are changes",
-        action="store_true",
-    )
-    parser.add_argument(
-        "--lbryum-part", help="part of lbryum version to bump",
-        choices=LBRYUM_PARTS
-    )
-    parser.add_argument(
-        "--last-release",
-        help=("manually set the last release version. The default is to query and parse the"
-              " value from the release page.")
-    )
-    parser.add_argument(
-        "--skip-sanity-checks", action="store_true")
-    parser.add_argument(
-        "--require-changelog", action="store_true",
-        help=("Set this flag to raise an exception if a submodules has changes without a"
-              " corresponding changelog entry. The default is to log a warning")
-    )
-    parser.add_argument(
-        "--skip-push", action="store_true",
-        help="Set to not push changes to remote repo"
-    )
 
+    parser = argparse.ArgumentParser()
+    parser.add_argument("part", choices=bumpversion_parts, help="part of version to bump")
+    parser.add_argument("--skip-sanity-checks", action="store_true")
+    parser.add_argument("--skip-push", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--confirm", action="store_true")
     args = parser.parse_args()
 
-    base = git.Repo(os.getcwd())
+    if args.dry_run:
+        print "DRY RUN. Nothing will be committed/pushed."
+
+    repo = Repo('lbry-app', args.part, ROOT)
     branch = 'master'
 
+    print 'Current version: {}'.format(repo.current_version)
+    print 'New version: {}'.format(repo.new_version)
+
+    if not args.confirm and not confirm():
+        print "Aborting"
+        return 1
+
     if not args.skip_sanity_checks:
-        run_sanity_checks(base, branch)
+        run_sanity_checks(repo, branch)
+    repo.assert_new_tag_is_absent()
 
-    base_repo = Repo('lbry-app', args.lbry_part, os.getcwd())
+    is_rc = re.search('\drc\d+$', repo.new_version) is not None
-    base_repo.assert_new_tag_is_absent()
+    # only have a release message for real releases, not for RCs
+    release_msg = '' if is_rc else repo.get_unreleased_changelog()
 
-    last_release = args.last_release or base_repo.get_last_tag()
+    if args.dry_run:
-    logging.info('Last release: %s', last_release)
+        print "rc: " + ("yes" if is_rc else "no")
+        print "release message: \n" + (release_msg or "  NO MESSAGE FOR RCs")
+        return
 
     gh_token = get_gh_token()
     auth = github.Github(gh_token)
     github_repo = auth.get_repo('lbryio/lbry-app')
 
-    names = ['lbryum', 'lbry']
+    if not is_rc:
-    repos = {name: Repo(name, get_part(args, name)) for name in names}
+        repo.bump_changelog()
+    repo.bumpversion()
 
-    changelogs = {}
+    new_tag = repo.get_new_tag()
+    github_repo.create_git_release(new_tag, new_tag, release_msg, draft=True, prerelease=is_rc)
 
-    for repo in repos.values():
+    if args.skip_push:
-        logging.info('Processing repo: %s', repo.name)
+        print (
-        repo.checkout(branch)
+            'Skipping push; you will have to reset and delete tags if '
-        last_submodule_hash = base_repo.get_submodule_hash(last_release, repo.name)
+            'you want to run this script again.'
-        if repo.has_changes_from_revision(last_submodule_hash):
+        )
-            if repo.name == 'lbryum':
-                if args.skip_lbryum:
-                    continue
-                if not repo.part:
-                    repo.part = get_lbryum_part()
-            entry = repo.get_changelog_entry()
-            if entry:
-                changelogs[repo.name] = entry.strip()
-                repo.add_changelog()
-            else:
-                msg = 'Changelog entry is missing for {}'.format(repo.name)
-                if args.require_changelog:
-                    raise Exception(msg)
-                else:
-                    logging.warning(msg)
-        else:
-            logging.warning('Submodule %s has no changes.', repo.name)
-            if repo.name == 'lbryum':
-                # The other repos have their version track each other so need to bump
-                # them even if there aren't any changes, but lbryum should only be
-                # bumped if it has changes
-                continue
-        # bumpversion will fail if there is already the tag we want in the repo
-        repo.assert_new_tag_is_absent()
-        repo.bumpversion()
-
-    release_msg = get_release_msg(changelogs, names)
-
-    for name in names:
-        base.git.add(name)
-
-    base_repo.bumpversion()
-    current_tag = base.git.describe()
-
-    is_rc = re.match('\drc\d+$', current_tag) is not None
-
-    github_repo.create_git_release(current_tag, current_tag, release_msg, draft=True,
-                                   prerelease=is_rc)
-    no_change_msg = ('No change since the last release. This release is simply a placeholder'
-                     ' so that LBRY and LBRY App track the same version')
-    lbrynet_daemon_release_msg = changelogs.get('lbry', no_change_msg)
-    auth.get_repo('lbryio/lbry').create_git_release(
-        current_tag, current_tag, lbrynet_daemon_release_msg, draft=True)
-
-    if not args.skip_push:
-        for repo in repos.values():
-            repo.git.push(follow_tags=True)
-        base.git.push(follow_tags=True, recurse_submodules='check')
     else:
-        logging.info('Skipping push; you will have to reset and delete tags if '
+        repo.git_repo.git.push(follow_tags=True, recurse_submodules='check')
-                     'you want to run this script again. Take a look at reset.sh; '
+
-                     'it probably does what you want.')
+
+class Repo(object):
+    def __init__(self, name, part, directory):
+        self.name = name
+        self.part = part
+        if not self.part:
+            raise Exception('Part required')
+        self.directory = directory
+        self.git_repo = git.Repo(self.directory)
+        self._bumped = False
+
+        self.current_version = self._get_current_version()
+        self.new_version = self._get_new_version()
+        self._changelog = changelog.Changelog(os.path.join(self.directory, 'CHANGELOG.md'))
+
+    def get_new_tag(self):
+        return 'v' + self.new_version
+
+    def get_unreleased_changelog(self):
+        return self._changelog.get_unreleased()
+
+    def bump_changelog(self):
+        self._changelog.bump(self.new_version)
+        with pushd(self.directory):
+            self.git_repo.git.add(os.path.basename(self._changelog.path))
+
+    def _get_current_version(self):
+        with pushd(self.directory):
+            output = subprocess.check_output(
+                ['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part])
+            return re.search('^current_version=(.*)$', output, re.M).group(1)
+
+    def _get_new_version(self):
+        with pushd(self.directory):
+            output = subprocess.check_output(
+                ['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part])
+            return re.search('^new_version=(.*)$', output, re.M).group(1)
+
+    def bumpversion(self):
+        if self._bumped:
+            raise Exception('Cowardly refusing to bump a repo twice')
+        with pushd(self.directory):
+            subprocess.check_call(['bumpversion', '--allow-dirty', self.part])
+            self._bumped = True
+
+    def assert_new_tag_is_absent(self):
+        new_tag = self.get_new_tag()
+        tags = self.git_repo.git.tag()
+        if new_tag in tags.split('\n'):
+            raise Exception('Tag {} is already present in repo {}.'.format(new_tag, self.name))
+
+    def is_behind(self, branch):
+        self.git_repo.remotes.origin.fetch()
+        rev_list = '{branch}...origin/{branch}'.format(branch=branch)
+        commits_behind = self.git_repo.git.rev_list(rev_list, right_only=True, count=True)
+        commits_behind = int(commits_behind)
+        return commits_behind > 0
+
+
+def get_bumpversion_parts():
+    with pushd(ROOT):
+        output = subprocess.check_output([
+            'bumpversion', '--dry-run', '--list', '--allow-dirty', 'fake-part',
+        ])
+    parse_line = re.search('^parse=(.*)$', output, re.M).group(1)
+    return tuple(re.findall('<([^>]+)>', parse_line))
 
 
 def get_gh_token():
@@ -148,131 +155,36 @@ in the future"""
         return raw_input('token: ').strip()
 
 
-def get_lbryum_part():
+def confirm():
-    print """The lbryum repo has changes but you didn't specify how to bump the
+    return raw_input('Is this what you want? [y/N] ').strip().lower() == 'y'
-version. Please enter one of: {}""".format(', '.join(LBRYUM_PARTS))
-    while True:
-        part = raw_input('part: ').strip()
-        if part in LBRYUM_PARTS:
-            return part
-        print 'Invalid part. Enter one of: {}'.format(', '.join(LBRYUM_PARTS))
 
 
-def get_release_msg(changelogs, names):
+def run_sanity_checks(repo, branch):
-    lines = []
+    if repo.git_repo.is_dirty():
-    for name in names:
-        entry = changelogs.get(name)
-        if not entry:
-            continue
-        lines.append('## {}\n'.format(name))
-        lines.append('{}\n'.format(entry))
-    return '\n'.join(lines)
-
-
-def run_sanity_checks(base, branch):
-    if base.is_dirty():
         print 'Cowardly refusing to release a dirty repo'
         sys.exit(1)
-    if base.active_branch.name != branch:
+    if repo.git_repo.active_branch.name != branch:
         print 'Cowardly refusing to release when not on the {} branch'.format(branch)
         sys.exit(1)
-    if is_behind(base, branch):
+    if repo.is_behind(branch):
         print 'Cowardly refusing to release when behind origin'
         sys.exit(1)
-    check_bumpversion()
+    if not is_custom_bumpversion_version():
-
+        print (
-
+            'Install LBRY\'s fork of bumpversion: '
-def is_behind(base, branch):
+            'pip install -U git+https://github.com/lbryio/bumpversion.git'
-    base.remotes.origin.fetch()
+        )
-    rev_list = '{branch}...origin/{branch}'.format(branch=branch)
-    commits_behind = base.git.rev_list(rev_list, right_only=True, count=True)
-    commits_behind = int(commits_behind)
-    return commits_behind > 0
-
-
-def check_bumpversion():
-    def require_new_version():
-        print 'Install bumpversion: pip install -U git+https://github.com/lbryio/bumpversion.git'
         sys.exit(1)
 
+
+def is_custom_bumpversion_version():
     try:
-        output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT)
+        output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT).strip()
-        output = output.strip()
+        if output == 'bumpversion 0.5.4-lbry':
-        if output != 'bumpversion 0.5.4-lbry':
+            return True
-            require_new_version()
+    except (subprocess.CalledProcessError, OSError):
-    except (subprocess.CalledProcessError, OSError) as err:
+        pass
-        require_new_version()
+    return False
-
-
-def get_part(args, name):
-    return getattr(args, name + '_part') or args.lbry_part
-
-
-class Repo(object):
-    def __init__(self, name, part, directory=None):
-        self.name = name
-        self.part = part
-        self.directory = directory or os.path.join(os.getcwd(), name)
-        self.git_repo = git.Repo(self.directory)
-        self.saved_commit = None
-        self._bumped = False
-
-    def get_last_tag(self):
-        return string.split(self.git_repo.git.describe(tags=True), '-')[0]
-
-    def get_submodule_hash(self, revision, submodule_path):
-        line = getattr(self.git_repo.git, 'ls-tree')(revision, submodule_path)
-        return string.split(line)[2] if line else None
-
-    def has_changes_from_revision(self, revision):
-        commit = str(self.git_repo.commit())
-        logging.info('%s =? %s', commit, revision)
-        return commit != revision
-
-    def save_commit(self):
-        self.saved_commit = self.git_repo.commit()
-        logging.info('Saved ', self.git_repo.commit(), self.saved_commit)
-
-    def checkout(self, branch):
-        self.git_repo.git.checkout(branch)
-        self.git_repo.git.pull(rebase=True)
-
-    def get_changelog_entry(self):
-        filename = os.path.join(self.directory, 'CHANGELOG.md')
-        return changelog.bump(filename, self.new_version())
-
-    def add_changelog(self):
-        with pushd(self.directory):
-            self.git_repo.git.add('CHANGELOG.md')
-
-    def new_version(self):
-        if self._bumped:
-            raise Exception('Cannot calculate a new version on an already bumped repo')
-        if not self.part:
-            raise Exception('Cannot calculate a new version without a part')
-        with pushd(self.directory):
-            output = subprocess.check_output(
-                ['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part])
-            return re.search('^new_version=(.*)$', output, re.M).group(1)
-
-    def bumpversion(self):
-        if self._bumped:
-            raise Exception('Cowardly refusing to bump a repo twice')
-        if not self.part:
-            raise Exception('Cannot bump version for {}: no part specified'.format(repo.name))
-        with pushd(self.directory):
-            subprocess.check_call(['bumpversion', '--allow-dirty', self.part])
-            self._bumped = True
-
-    def assert_new_tag_is_absent(self):
-        new_tag = 'v' + self.new_version()
-        tags = self.git_repo.git.tag()
-        if new_tag in tags.split('\n'):
-            raise Exception('Tag {} is already present in repo {}.'.format(new_tag, self.name))
-
-    @property
-    def git(self):
-        return self.git_repo.git
 
 
 @contextlib.contextmanager
@@ -284,10 +196,4 @@ def pushd(new_dir):
 
 
 if __name__ == '__main__':
-    logging.basicConfig(
-        format="%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s",
-        level='INFO'
-    )
     sys.exit(main())
-else:
-    log = logging.getLogger('__name__')