From 38fdf56583049a931880c1a592b19ca3f1813592 Mon Sep 17 00:00:00 2001 From: Alex Grintsvayg Date: Wed, 12 Apr 2017 18:46:32 -0400 Subject: [PATCH] rewrite release.py script --- build/changelog.py | 173 +++++++++++----------- build/release.py | 348 +++++++++++++++++---------------------------- 2 files changed, 208 insertions(+), 313 deletions(-) diff --git a/build/changelog.py b/build/changelog.py index 724324251..eb03b682f 100644 --- a/build/changelog.py +++ b/build/changelog.py @@ -1,8 +1,5 @@ -import argparse import datetime import re -import sys - CHANGELOG_START_RE = re.compile(r'^\#\# \[Unreleased\]') CHANGELOG_END_RE = re.compile(r'^\#\# \[.*\] - \d{4}-\d{2}-\d{2}') @@ -14,118 +11,110 @@ EMPTY_RE = re.compile(r'^\w*\*\w*$') ENTRY_RE = re.compile(r'\* (.*)') VALID_SECTIONS = ['Added', 'Changed', 'Deprecated', 'Removed', 'Fixed', 'Security'] - # allocate some entries to cut-down on merge conflicts TEMPLATE = """### Added * * - * ### Changed * * - * ### Fixed * * - * """ -def main(): - print "i am broken" - return 1 - parser = argparse.ArgumentParser() - parser.add_argument('changelog') - parser.add_argument('version') - args = parser.parse_args() - bump(changelog, version) +class Changelog(object): + def __init__(self, path): + self.path = path + self.start = [] + self.unreleased = [] + self.rest = [] + self._parse() + def _parse(self): + with open(self.path) as fp: + lines = fp.readlines() -def bump(changelog, version): - with open(changelog) as fp: - lines = fp.readlines() + unreleased_start_found = False + unreleased_end_found = False - start = [] - unreleased = [] - rest = [] - unreleased_start_found = False - unreleased_end_found = False - for line in lines: - if not unreleased_start_found: - start.append(line) - if CHANGELOG_START_RE.search(line): - unreleased_start_found = True - continue - if unreleased_end_found: - rest.append(line) - continue - if CHANGELOG_END_RE.search(line): - rest.append(line) - unreleased_end_found = True - continue - if CHANGELOG_ERROR_RE.search(line): - raise Exception( - 'Failed to parse {}: {}'.format(changelog, 'unexpected section header found')) - unreleased.append(line) + for line in lines: + if not unreleased_start_found: + self.start.append(line) + if CHANGELOG_START_RE.search(line): + unreleased_start_found = True + continue + if unreleased_end_found: + self.rest.append(line) + continue + if CHANGELOG_END_RE.search(line): + self.rest.append(line) + unreleased_end_found = True + continue + if CHANGELOG_ERROR_RE.search(line): + raise Exception( + 'Failed to parse {}: {}'.format(self.path, 'unexpected section header found')) + self.unreleased.append(line) - today = datetime.datetime.today() - header = '## [{}] - {}\n'.format(version, today.strftime('%Y-%m-%d')) - released = normalize(unreleased) - if not released: - # If we don't have anything in the Unreleased section, then leave the - # changelog as it is and return None - return + self.unreleased = self._normalize_section(self.unreleased) - changelog_data = ( - ''.join(start) + - TEMPLATE + - header + - '\n'.join(released) + '\n\n' - + ''.join(rest) - ) - with open(changelog, 'w') as fp: - fp.write(changelog_data) - return '\n'.join(released) + '\n\n' + @staticmethod + def _normalize_section(lines): + """Parse a changelog entry and output a normalized form""" + sections = {} + current_section_name = None + current_section_contents = [] + for line in lines: + line = line.strip() + if not line or EMPTY_RE.match(line): + continue + match = SECTION_RE.match(line) + if match: + if current_section_contents: + sections[current_section_name] = current_section_contents + current_section_contents = [] + current_section_name = match.group(1) + if current_section_name not in VALID_SECTIONS: + raise ValueError("Section '{}' is not valid".format(current_section_name)) + continue + match = ENTRY_RE.match(line) + if match: + current_section_contents.append(match.group(1)) + continue + raise Exception('Something is wrong with line: {}'.format(line)) + if current_section_contents: + sections[current_section_name] = current_section_contents + output = [] + for section in VALID_SECTIONS: + if section not in sections: + continue + output.append('### {}'.format(section)) + for entry in sections[section]: + output.append(' * {}'.format(entry)) + return output -def normalize(lines): - """Parse a changelog entry and output a normalized form""" - sections = {} - current_section_name = None - current_section_contents = [] - for line in lines: - line = line.strip() - if not line or EMPTY_RE.match(line): - continue - match = SECTION_RE.match(line) - if match: - if current_section_contents: - sections[current_section_name] = current_section_contents - current_section_contents = [] - current_section_name = match.group(1) - if current_section_name not in VALID_SECTIONS: - raise ValueError("Section '{}' is not valid".format(current_section_name)) - continue - match = ENTRY_RE.match(line) - if match: - current_section_contents.append(match.group(1)) - continue - raise Exception('Something is wrong with line: {}'.format(line)) - if current_section_contents: - sections[current_section_name] = current_section_contents + def get_unreleased(self): + return '\n'.join(self.unreleased) if self.unreleased else None - output = [] - for section in VALID_SECTIONS: - if section not in sections: - continue - output.append('### {}'.format(section)) - for entry in sections[section]: - output.append(' * {}'.format(entry)) - return output + def bump(self, version): + if not self.unreleased: + return + today = datetime.datetime.today() + header = '## [{}] - {}\n'.format(version, today.strftime('%Y-%m-%d')) -if __name__ == '__main__': - sys.exit(main()) + changelog_data = ( + ''.join(self.start) + + TEMPLATE + + header + + '\n'.join(self.unreleased) + '\n\n' + + ''.join(self.rest) + ) + + with open(self.path, 'w') as fp: + fp.write(changelog_data) diff --git a/build/release.py b/build/release.py index 4100cf8cf..619ad7df9 100644 --- a/build/release.py +++ b/build/release.py @@ -1,13 +1,11 @@ -"""Trigger a release. +"""Bump version and create Github release -This script is to be run locally (not on a build server). +This script should be run locally, not on a build server. """ import argparse import contextlib -import logging import os import re -import string import subprocess import sys @@ -16,122 +14,131 @@ import github import changelog -# TODO: ask bumpversion for these -LBRY_PARTS = ('major', 'minor', 'patch', 'release', 'candidate') -LBRYUM_PARTS = ('major', 'minor', 'patch') +ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - "lbry_part", help="part of lbry version to bump", - choices=LBRY_PARTS - ) - parser.add_argument( - "--skip-lbryum", help="skip bumping lbryum, even if there are changes", - action="store_true", - ) - parser.add_argument( - "--lbryum-part", help="part of lbryum version to bump", - choices=LBRYUM_PARTS - ) - parser.add_argument( - "--last-release", - help=("manually set the last release version. The default is to query and parse the" - " value from the release page.") - ) - parser.add_argument( - "--skip-sanity-checks", action="store_true") - parser.add_argument( - "--require-changelog", action="store_true", - help=("Set this flag to raise an exception if a submodules has changes without a" - " corresponding changelog entry. The default is to log a warning") - ) - parser.add_argument( - "--skip-push", action="store_true", - help="Set to not push changes to remote repo" - ) + bumpversion_parts = get_bumpversion_parts() + parser = argparse.ArgumentParser() + parser.add_argument("part", choices=bumpversion_parts, help="part of version to bump") + parser.add_argument("--skip-sanity-checks", action="store_true") + parser.add_argument("--skip-push", action="store_true") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--confirm", action="store_true") args = parser.parse_args() - base = git.Repo(os.getcwd()) + if args.dry_run: + print "DRY RUN. Nothing will be committed/pushed." + + repo = Repo('lbry-app', args.part, ROOT) branch = 'master' + print 'Current version: {}'.format(repo.current_version) + print 'New version: {}'.format(repo.new_version) + + if not args.confirm and not confirm(): + print "Aborting" + return 1 + if not args.skip_sanity_checks: - run_sanity_checks(base, branch) + run_sanity_checks(repo, branch) + repo.assert_new_tag_is_absent() - base_repo = Repo('lbry-app', args.lbry_part, os.getcwd()) - base_repo.assert_new_tag_is_absent() + is_rc = re.search('\drc\d+$', repo.new_version) is not None + # only have a release message for real releases, not for RCs + release_msg = '' if is_rc else repo.get_unreleased_changelog() - last_release = args.last_release or base_repo.get_last_tag() - logging.info('Last release: %s', last_release) + if args.dry_run: + print "rc: " + ("yes" if is_rc else "no") + print "release message: \n" + (release_msg or " NO MESSAGE FOR RCs") + return gh_token = get_gh_token() auth = github.Github(gh_token) github_repo = auth.get_repo('lbryio/lbry-app') - names = ['lbryum', 'lbry'] - repos = {name: Repo(name, get_part(args, name)) for name in names} + if not is_rc: + repo.bump_changelog() + repo.bumpversion() - changelogs = {} + new_tag = repo.get_new_tag() + github_repo.create_git_release(new_tag, new_tag, release_msg, draft=True, prerelease=is_rc) - for repo in repos.values(): - logging.info('Processing repo: %s', repo.name) - repo.checkout(branch) - last_submodule_hash = base_repo.get_submodule_hash(last_release, repo.name) - if repo.has_changes_from_revision(last_submodule_hash): - if repo.name == 'lbryum': - if args.skip_lbryum: - continue - if not repo.part: - repo.part = get_lbryum_part() - entry = repo.get_changelog_entry() - if entry: - changelogs[repo.name] = entry.strip() - repo.add_changelog() - else: - msg = 'Changelog entry is missing for {}'.format(repo.name) - if args.require_changelog: - raise Exception(msg) - else: - logging.warning(msg) - else: - logging.warning('Submodule %s has no changes.', repo.name) - if repo.name == 'lbryum': - # The other repos have their version track each other so need to bump - # them even if there aren't any changes, but lbryum should only be - # bumped if it has changes - continue - # bumpversion will fail if there is already the tag we want in the repo - repo.assert_new_tag_is_absent() - repo.bumpversion() - - release_msg = get_release_msg(changelogs, names) - - for name in names: - base.git.add(name) - - base_repo.bumpversion() - current_tag = base.git.describe() - - is_rc = re.match('\drc\d+$', current_tag) is not None - - github_repo.create_git_release(current_tag, current_tag, release_msg, draft=True, - prerelease=is_rc) - no_change_msg = ('No change since the last release. This release is simply a placeholder' - ' so that LBRY and LBRY App track the same version') - lbrynet_daemon_release_msg = changelogs.get('lbry', no_change_msg) - auth.get_repo('lbryio/lbry').create_git_release( - current_tag, current_tag, lbrynet_daemon_release_msg, draft=True) - - if not args.skip_push: - for repo in repos.values(): - repo.git.push(follow_tags=True) - base.git.push(follow_tags=True, recurse_submodules='check') + if args.skip_push: + print ( + 'Skipping push; you will have to reset and delete tags if ' + 'you want to run this script again.' + ) else: - logging.info('Skipping push; you will have to reset and delete tags if ' - 'you want to run this script again. Take a look at reset.sh; ' - 'it probably does what you want.') + repo.git_repo.git.push(follow_tags=True, recurse_submodules='check') + + +class Repo(object): + def __init__(self, name, part, directory): + self.name = name + self.part = part + if not self.part: + raise Exception('Part required') + self.directory = directory + self.git_repo = git.Repo(self.directory) + self._bumped = False + + self.current_version = self._get_current_version() + self.new_version = self._get_new_version() + self._changelog = changelog.Changelog(os.path.join(self.directory, 'CHANGELOG.md')) + + def get_new_tag(self): + return 'v' + self.new_version + + def get_unreleased_changelog(self): + return self._changelog.get_unreleased() + + def bump_changelog(self): + self._changelog.bump(self.new_version) + with pushd(self.directory): + self.git_repo.git.add(os.path.basename(self._changelog.path)) + + def _get_current_version(self): + with pushd(self.directory): + output = subprocess.check_output( + ['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part]) + return re.search('^current_version=(.*)$', output, re.M).group(1) + + def _get_new_version(self): + with pushd(self.directory): + output = subprocess.check_output( + ['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part]) + return re.search('^new_version=(.*)$', output, re.M).group(1) + + def bumpversion(self): + if self._bumped: + raise Exception('Cowardly refusing to bump a repo twice') + with pushd(self.directory): + subprocess.check_call(['bumpversion', '--allow-dirty', self.part]) + self._bumped = True + + def assert_new_tag_is_absent(self): + new_tag = self.get_new_tag() + tags = self.git_repo.git.tag() + if new_tag in tags.split('\n'): + raise Exception('Tag {} is already present in repo {}.'.format(new_tag, self.name)) + + def is_behind(self, branch): + self.git_repo.remotes.origin.fetch() + rev_list = '{branch}...origin/{branch}'.format(branch=branch) + commits_behind = self.git_repo.git.rev_list(rev_list, right_only=True, count=True) + commits_behind = int(commits_behind) + return commits_behind > 0 + + +def get_bumpversion_parts(): + with pushd(ROOT): + output = subprocess.check_output([ + 'bumpversion', '--dry-run', '--list', '--allow-dirty', 'fake-part', + ]) + parse_line = re.search('^parse=(.*)$', output, re.M).group(1) + return tuple(re.findall('<([^>]+)>', parse_line)) def get_gh_token(): @@ -148,131 +155,36 @@ in the future""" return raw_input('token: ').strip() -def get_lbryum_part(): - print """The lbryum repo has changes but you didn't specify how to bump the -version. Please enter one of: {}""".format(', '.join(LBRYUM_PARTS)) - while True: - part = raw_input('part: ').strip() - if part in LBRYUM_PARTS: - return part - print 'Invalid part. Enter one of: {}'.format(', '.join(LBRYUM_PARTS)) +def confirm(): + return raw_input('Is this what you want? [y/N] ').strip().lower() == 'y' -def get_release_msg(changelogs, names): - lines = [] - for name in names: - entry = changelogs.get(name) - if not entry: - continue - lines.append('## {}\n'.format(name)) - lines.append('{}\n'.format(entry)) - return '\n'.join(lines) - - -def run_sanity_checks(base, branch): - if base.is_dirty(): +def run_sanity_checks(repo, branch): + if repo.git_repo.is_dirty(): print 'Cowardly refusing to release a dirty repo' sys.exit(1) - if base.active_branch.name != branch: + if repo.git_repo.active_branch.name != branch: print 'Cowardly refusing to release when not on the {} branch'.format(branch) sys.exit(1) - if is_behind(base, branch): + if repo.is_behind(branch): print 'Cowardly refusing to release when behind origin' sys.exit(1) - check_bumpversion() - - -def is_behind(base, branch): - base.remotes.origin.fetch() - rev_list = '{branch}...origin/{branch}'.format(branch=branch) - commits_behind = base.git.rev_list(rev_list, right_only=True, count=True) - commits_behind = int(commits_behind) - return commits_behind > 0 - - -def check_bumpversion(): - def require_new_version(): - print 'Install bumpversion: pip install -U git+https://github.com/lbryio/bumpversion.git' + if not is_custom_bumpversion_version(): + print ( + 'Install LBRY\'s fork of bumpversion: ' + 'pip install -U git+https://github.com/lbryio/bumpversion.git' + ) sys.exit(1) + +def is_custom_bumpversion_version(): try: - output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT) - output = output.strip() - if output != 'bumpversion 0.5.4-lbry': - require_new_version() - except (subprocess.CalledProcessError, OSError) as err: - require_new_version() - - -def get_part(args, name): - return getattr(args, name + '_part') or args.lbry_part - - -class Repo(object): - def __init__(self, name, part, directory=None): - self.name = name - self.part = part - self.directory = directory or os.path.join(os.getcwd(), name) - self.git_repo = git.Repo(self.directory) - self.saved_commit = None - self._bumped = False - - def get_last_tag(self): - return string.split(self.git_repo.git.describe(tags=True), '-')[0] - - def get_submodule_hash(self, revision, submodule_path): - line = getattr(self.git_repo.git, 'ls-tree')(revision, submodule_path) - return string.split(line)[2] if line else None - - def has_changes_from_revision(self, revision): - commit = str(self.git_repo.commit()) - logging.info('%s =? %s', commit, revision) - return commit != revision - - def save_commit(self): - self.saved_commit = self.git_repo.commit() - logging.info('Saved ', self.git_repo.commit(), self.saved_commit) - - def checkout(self, branch): - self.git_repo.git.checkout(branch) - self.git_repo.git.pull(rebase=True) - - def get_changelog_entry(self): - filename = os.path.join(self.directory, 'CHANGELOG.md') - return changelog.bump(filename, self.new_version()) - - def add_changelog(self): - with pushd(self.directory): - self.git_repo.git.add('CHANGELOG.md') - - def new_version(self): - if self._bumped: - raise Exception('Cannot calculate a new version on an already bumped repo') - if not self.part: - raise Exception('Cannot calculate a new version without a part') - with pushd(self.directory): - output = subprocess.check_output( - ['bumpversion', '--dry-run', '--list', '--allow-dirty', self.part]) - return re.search('^new_version=(.*)$', output, re.M).group(1) - - def bumpversion(self): - if self._bumped: - raise Exception('Cowardly refusing to bump a repo twice') - if not self.part: - raise Exception('Cannot bump version for {}: no part specified'.format(repo.name)) - with pushd(self.directory): - subprocess.check_call(['bumpversion', '--allow-dirty', self.part]) - self._bumped = True - - def assert_new_tag_is_absent(self): - new_tag = 'v' + self.new_version() - tags = self.git_repo.git.tag() - if new_tag in tags.split('\n'): - raise Exception('Tag {} is already present in repo {}.'.format(new_tag, self.name)) - - @property - def git(self): - return self.git_repo.git + output = subprocess.check_output(['bumpversion', '-v'], stderr=subprocess.STDOUT).strip() + if output == 'bumpversion 0.5.4-lbry': + return True + except (subprocess.CalledProcessError, OSError): + pass + return False @contextlib.contextmanager @@ -284,10 +196,4 @@ def pushd(new_dir): if __name__ == '__main__': - logging.basicConfig( - format="%(asctime)s %(levelname)-8s %(name)s:%(lineno)d: %(message)s", - level='INFO' - ) sys.exit(main()) -else: - log = logging.getLogger('__name__')