packaging/release-rsync

   1 #!/usr/bin/python3 -B
   2
   3 # This script expects the directory ~/samba-rsync-ftp to exist and to be a
   4 # copy of the /home/ftp/pub/rsync dir on samba.org.  When the script is done,
   5 # the git repository in the current directory will be updated, and the local
   6 # ~/samba-rsync-ftp dir will be ready to be rsynced to samba.org.
   7
   8 import os, sys, re, argparse, glob, shutil, signal
   9 from datetime import datetime
  10 from getpass import getpass
  11
  12 sys.path = ['packaging'] + sys.path
  13
  14 from pkglib import *
  15
  16 dest = os.environ['HOME'] + '/samba-rsync-ftp'
  17 ORIGINAL_PATH = os.environ['PATH']
  18 MAKE_GEN_CMDS = [
  19         'make -f prepare-source.mak conf'.split(),
  20         './config.status'.split(),
  21         'make gen'.split(),
  22         ]
  23
  24 def main():
  25     now = datetime.now()
  26     cl_today = now.strftime('* %a %b %d %Y')
  27     year = now.strftime('%Y')
  28     ztoday = now.strftime('%d %b %Y')
  29     today = ztoday.lstrip('0')
  30
  31     curdir = os.getcwd()
  32
  33     signal.signal(signal.SIGINT, signal_handler)
  34
  35     extra_files = get_extra_files()
  36
  37     dash_line = '=' * 74
  38
  39     print(f"""\
  40 {dash_line}
  41 == This will release a new version of rsync onto an unsuspecting world. ==
  42 {dash_line}
  43 """)
  44
  45     if not os.path.isdir(dest):
  46         die(dest, "dest does not exist")
  47     if not os.path.isdir('.git'):
  48         die("There is no .git dir in the current directory.")
  49     if os.path.lexists('a'):
  50         die('"a" must not exist in the current directory.')
  51     if os.path.lexists('b'):
  52         die('"b" must not exist in the current directory.')
  53     if os.path.lexists('patches.gen'):
  54         die('"patches.gen" must not exist in the current directory.')
  55
  56     check_git_state(args.master_branch, True, 'patches')
  57
  58     confversion = get_configure_version()
  59
  60     # All version values are strings!
  61     lastversion, last_protocol_version = get_OLDNEWS_version_info()
  62     protocol_version, subprotocol_version = get_protocol_versions()
  63
  64     version = confversion
  65     m = re.search(r'pre(\d+)', version)
  66     if m:
  67         version = re.sub(r'pre\d+', 'pre' + str(int(m[1]) + 1), version)
  68     else:
  69         version = version.replace('dev', 'pre1')
  70
  71     ans = input(f"Please enter the version number of this release: [{version}] ")
  72     if ans == '.':
  73         version = re.sub(r'pre\d+', '', version)
  74     elif ans != '':
  75         version = ans
  76     if not re.match(r'^[\d.]+(pre\d+)?$', version):
  77         die(f'Invalid version: "{version}"')
  78
  79     v_ver = 'v' + version
  80     rsync_ver = 'rsync-' + version
  81     rsync_lastver = 'rsync-' + lastversion
  82
  83     if os.path.lexists(rsync_ver):
  84         die(f'"{rsync_ver}" must not exist in the current directory.')
  85     if os.path.lexists(rsync_lastver):
  86         die(f'"{rsync_lastver}" must not exist in the current directory.')
  87
  88     out = cmd_txt_chk(['git', 'tag', '-l', v_ver])
  89     if out != '':
  90         print(f"Tag {v_ver} already exists.")
  91         ans = input("\nDelete tag or quit? [Q/del] ")
  92         if not re.match(r'^del', ans, flags=re.I):
  93             die("Aborted")
  94         cmd_chk(['git', 'tag', '-d', v_ver])
  95
  96     version = re.sub(r'[-.]*pre[-.]*', 'pre', version)
  97     if 'pre' in version and not confversion.endswith('dev'):
  98         lastversion = confversion
  99
 100     ans = input(f"Enter the previous version to produce a patch against: [{lastversion}] ")
 101     if ans != '':
 102         lastversion = ans
 103     lastversion = re.sub(r'[-.]*pre[-.]*', 'pre', lastversion)
 104
 105     m = re.search(r'(pre\d+)', version)
 106     pre = m[1] if m else ''
 107
 108     release = '0.1' if pre else '1'
 109     ans = input(f"Please enter the RPM release number of this release: [{release}] ")
 110     if ans != '':
 111         release = ans
 112     if pre:
 113         release += '.' + pre
 114
 115     finalversion = re.sub(r'pre\d+', '', version)
 116     if protocol_version == last_protocol_version:
 117         proto_changed = 'unchanged'
 118         proto_change_date = ' ' * 11
 119     else:
 120         proto_changed = 'changed'
 121         if finalversion in pdate:
 122             proto_change_date = pdate[finalversion]
 123         else:
 124             while True:
 125                 ans = input("On what date did the protocol change to {protocol_version} get checked in? (dd Mmm yyyy) ")
 126                 if re.match(r'^\d\d \w\w\w \d\d\d\d$', ans):
 127                     break
 128             proto_change_date = ans
 129
 130     if 'pre' in lastversion:
 131         if not pre:
 132             die("You should not diff a release version against a pre-release version.")
 133         srcdir = srcdiffdir = lastsrcdir = 'src-previews'
 134         skipping = ' ** SKIPPING **'
 135     elif pre:
 136         srcdir = srcdiffdir = 'src-previews'
 137         lastsrcdir = 'src'
 138         skipping = ' ** SKIPPING **'
 139     else:
 140         srcdir = lastsrcdir = 'src'
 141         srcdiffdir = 'src-diffs'
 142         skipping = ''
 143
 144     print(f"""
 145 {dash_line}
 146 version is "{version}"
 147 lastversion is "{lastversion}"
 148 dest is "{dest}"
 149 curdir is "{curdir}"
 150 srcdir is "{srcdir}"
 151 srcdiffdir is "{srcdiffdir}"
 152 lastsrcdir is "{lastsrcdir}"
 153 release is "{release}"
 154
 155 About to:
 156     - tweak SUBPROTOCOL_VERSION in rsync.h, if needed
 157     - tweak the version in configure.ac and the spec files
 158     - tweak NEWS.md and OLDNEWS.md to ensure header values are correct
 159     - generate configure.sh, config.h.in, and proto.h
 160     - touch the latest-year.h file and generate the manpages
 161     - page through the differences
 162 """)
 163     ans = input("<Press Enter to continue> ")
 164
 165     specvars = {
 166         'Version:': finalversion,
 167         'Release:': release,
 168         '%define fullversion': f'%{{version}}{pre}',
 169         'Released': version + '.',
 170         '%define srcdir': srcdir,
 171         }
 172
 173     tweak_files = 'configure.ac rsync.h NEWS.md OLDNEWS.md'.split()
 174     tweak_files += glob.glob('packaging/*.spec')
 175     tweak_files += glob.glob('packaging/*/*.spec')
 176
 177     for fn in tweak_files:
 178         with open(fn, 'r', encoding='utf-8') as fh:
 179             old_txt = txt = fh.read()
 180         if 'configure' in fn:
 181             x_re = re.compile(r'^(AC_INIT\(\[rsync\],\s*\[)\d.+?(\])', re.M)
 182             txt = replace_or_die(x_re, r'\g<1>%s\2' % version, txt, f"Unable to update AC_INIT with version in {fn}")
 183         elif '.spec' in fn:
 184             for var, val in specvars.items():
 185                 x_re = re.compile(r'^%s .*' % re.escape(var), re.M)
 186                 txt = replace_or_die(x_re, var + ' ' + val, txt, f"Unable to update {var} in {fn}")
 187             x_re = re.compile(r'^\* \w\w\w \w\w\w \d\d \d\d\d\d (.*)', re.M)
 188             txt = replace_or_die(x_re, r'%s \1' % cl_today, txt, f"Unable to update ChangeLog header in {fn}")
 189         elif fn == 'rsync.h':
 190             x_re = re.compile('(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)')
 191             repl = lambda m: m[1] + ' ' + '0' if not pre or proto_changed != 'changed' else 1 if m[2] == '0' else m[2]
 192             txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}")
 193         elif fn == 'NEWS.md':
 194             x_re = re.compile(
 195                     r'^(# NEWS for rsync %s )(\(UNRELEASED\))\s*(\n\nProtocol: )(\d+) (\([^)]+\))\n' % re.escape(finalversion),
 196                     re.I)
 197             repl = lambda m: m[1] + (m[2] if pre else f"({today})") + m[3] + f"{protocol_version} ({proto_changed})\n"
 198             msg = (f"The first 3 lines of {fn} are not in the right format.  They must be:\n"
 199                     + f"# NEWS for rsync {finalversion} (UNRELEASED)\n\n"
 200                     + f"Protocol: {protocol_version} ({proto_changed})")
 201             txt = replace_or_die(x_re, repl, txt, msg)
 202         elif fn == 'OLDNEWS.md':
 203             efv = re.escape(finalversion)
 204             x_re = re.compile(r'^(\| )(\S{2} \S{3} \d{4})(\s+\|\s+%s\s+\| ).{11}(\s+\| )\S{2}(\s+\|+)$' % efv, re.M)
 205             repl = lambda m: m[1] + (m[2] if pre else ztoday) + m[3] + proto_change_date + m[4] + protocol_version + m[5]
 206             txt = replace_or_die(x_re, repl, txt, f'Unable to find "| ?? ??? {year} | {finalversion} | ... |" line in {fn}')
 207         else:
 208             die(f"Unrecognized file in tweak_files: {fn}")
 209
 210         if txt != old_txt:
 211             print(f"Updating {fn}")
 212             with open(fn, 'w', encoding='utf-8') as fh:
 213                 fh.write(txt)
 214
 215     cmd_chk(['packaging/year-tweak'])
 216     cmd_chk([touch, 'latest-year.h']) # This file's date is put in the generated manpages.
 217     cmd_chk(['make', 'reconfigure'])
 218     cmd_chk(['make', 'man'])
 219
 220     print(dash_line)
 221     cmd_run("git diff --color | less -p '^diff .*'")
 222
 223     srctar_name = f"{rsync_ver}.tar.gz"
 224     pattar_name = f"rsync-patches-{version}.tar.gz"
 225     diff_name = f"{rsync_lastver}-{version}.diffs.gz"
 226     srctar_file = f"{dest}/{srcdir}/{srctar_name}"
 227     pattar_file = f"{dest}/{srcdir}/{pattar_name}"
 228     diff_file = f"{dest}/{srcdiffdir}/{diff_name}"
 229     news_file = f"{dest}/{srcdir}/{rsync_ver}-NEWS.md"
 230     lasttar_file = f"{dest}/{lastsrcdir}/{rsync_lastver}.tar.gz"
 231
 232     print(f"""\
 233 {dash_line}
 234
 235 About to:
 236     - commit all version changes
 237     - merge the {args.master_branch} branch into the patch/{args.master_branch}/* branches
 238     - update the files in the "patches" dir and OPTIONALLY
 239       (if you type 'y') to launch a shell for each patch
 240 """)
 241     ans = input("<Press Enter OR 'y' to continue> ")
 242
 243     s = cmd_run(['git', 'commit', '-a', '-m', f'Preparing for release of {version}'])
 244     if s.returncode:
 245         die('Aborting')
 246
 247     print(f'Creating any missing patch branches.')
 248     s = cmd_run(f'packaging/branch-from-patch --branch={args.master_branch} --add-missing')
 249     if s.returncode:
 250         die('Aborting')
 251
 252     print('Updating files in "patches" dir ...')
 253     s = cmd_run(f'packaging/patch-update --branch={args.master_branch}')
 254     if s.returncode:
 255         die('Aborting')
 256
 257     if re.match(r'^y', ans, re.I):
 258         print(f'\nVisiting all "patch/{args.master_branch}/*" branches ...')
 259         cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --shell")
 260
 261     if os.path.isdir('patches/.git'):
 262         s = cmd_run(f"cd patches && git commit -a -m 'The patches for {version}.'")
 263         if s.returncode:
 264             die('Aborting')
 265
 266     print(f"""\
 267 {dash_line}
 268
 269 About to:
 270     - create signed tag for this release: {v_ver}
 271     - create release diffs, "{diff_name}"
 272     - create release tar, "{srctar_name}"
 273     - generate {rsync_ver}/patches/* files
 274     - create patches tar, "{pattar_name}"
 275     - update top-level README.md, *NEWS.md, TODO, and ChangeLog
 276     - update top-level rsync*.html manpages
 277     - gpg-sign the release files
 278     - update hard-linked top-level release files{skipping}
 279 """)
 280     ans = input("<Press Enter to continue> ")
 281
 282     # TODO: is there a better way to ensure that our passphrase is in the agent?
 283     cmd_run("touch TeMp; gpg --sign TeMp; rm TeMp*")
 284
 285     out = cmd_txt(f"git tag -s -m 'Version {version}.' {v_ver}", capture='combined')
 286     print(out, end='')
 287     if 'bad passphrase' in out or 'failed' in out:
 288         die('Aborting')
 289
 290     if os.path.isdir('patches/.git'):
 291         out = cmd_txt(f"cd patches && git tag -s -m 'Version {version}.' {v_ver}", capture='combined')
 292         print(out, end='')
 293         if 'bad passphrase' in out or 'failed' in out:
 294             die('Aborting')
 295
 296     os.environ['PATH'] = ORIGINAL_PATH
 297
 298     # Extract the generated files from the old tar.
 299     tweaked_extra_files = [ f"{rsync_lastver}/{x}" for x in extra_files ]
 300     cmd_run(['tar', 'xzf', lasttar_file, *tweaked_extra_files])
 301     os.rename(rsync_lastver, 'a')
 302
 303     print(f"Creating {diff_file} ...")
 304     for cmd in MAKE_GEN_CMDS:
 305         cmd_chk(cmd)
 306     cmd_chk(['rsync', '-a', *extra_files, 'b/'])
 307
 308     sed_script = r's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:' # CAUTION: must not contain any single quotes!
 309     cmd_chk(f"(git diff v{lastversion} {v_ver} -- ':!.github'; diff -upN a b | sed -r '{sed_script}') | gzip -9 >{diff_file}")
 310     shutil.rmtree('a')
 311     os.rename('b', rsync_ver)
 312
 313     print(f"Creating {srctar_file} ...")
 314     cmd_chk(f"git archive --format=tar --prefix={rsync_ver}/ {v_ver} | tar xf -")
 315     cmd_chk(f"support/git-set-file-times --quiet --prefix={rsync_ver}/")
 316     cmd_chk(['fakeroot', 'tar', 'czf', srctar_file, '--exclude=.github', rsync_ver])
 317     shutil.rmtree(rsync_ver)
 318
 319     print(f'Updating files in "{rsync_ver}/patches" dir ...')
 320     os.mkdir(rsync_ver, 0o755)
 321     os.mkdir(f"{rsync_ver}/patches", 0o755)
 322     cmd_chk(f"packaging/patch-update --skip-check --branch={args.master_branch} --gen={rsync_ver}/patches".split())
 323
 324     print(f"Creating {pattar_file} ...")
 325     cmd_chk(['fakeroot', 'tar', 'chzf', pattar_file, rsync_ver + '/patches'])
 326     shutil.rmtree(rsync_ver)
 327
 328     print(f"Updating the other files in {dest} ...")
 329     cmd_chk('rsync -a README.md NEWS.md OLDNEWS.md TODO'.split() + [dest])
 330     if os.path.lexists(news_file):
 331         os.unlink(news_file)
 332     os.link(f"{dest}/NEWS.md", news_file)
 333     cmd_chk(f"git log --name-status | gzip -9 >{dest}/ChangeLog.gz")
 334
 335     for md_fn in glob.glob('*.[1-9].md'):
 336         html_fn = md_fn.replace('.md', '.html')
 337         cmd_chk(['rsync', '-a', html_fn, os.path.join(dest, html_fn)])
 338
 339     for fn in (srctar_file, pattar_file, diff_file):
 340         asc_fn = fn + '.asc'
 341         if os.path.lexists(asc_fn):
 342             os.unlink(asc_fn)
 343         res = cmd_run(['gpg', '--batch', '-ba', fn])
 344         if res.returncode != 0 and res.returncode != 2:
 345             die("gpg signing failed")
 346
 347     if not pre:
 348         for find in f'{dest}/rsync-*.gz {dest}/rsync-*.asc {dest}/rsync-*-NEWS.md {dest}/src-previews/rsync-*diffs.gz*'.split():
 349             for fn in glob.glob(find):
 350                 os.unlink(fn)
 351         top_link = [
 352                 srctar_file, f"{srctar_file}.asc",
 353                 pattar_file, f"{pattar_file}.asc",
 354                 diff_file, f"{diff_file}.asc",
 355                 news_file,
 356                 ]
 357         for fn in top_link:
 358             os.link(fn, re.sub(r'/src(-\w+)?/', '/', fn))
 359
 360     print(f"""\
 361 {dash_line}
 362
 363 Local changes are done.  When you're satisfied, push the git repository
 364 and rsync the release files.  Remember to announce the release on *BOTH*
 365 rsync-announce@lists.samba.org and rsync@lists.samba.org (and the web)!
 366 """)
 367
 368
 369 def replace_or_die(regex, repl, txt, die_msg):
 370     m = regex.search(txt)
 371     if not m:
 372         die(die_msg)
 373     return regex.sub(repl, txt, 1)
 374
 375
 376 def signal_handler(sig, frame):
 377     die("\nAborting due to SIGINT.")
 378
 379
 380 if __name__ == '__main__':
 381     parser = argparse.ArgumentParser(description="Prepare a new release of rsync in the git repo & ftp dir.", add_help=False)
 382     parser.add_argument('--branch', '-b', dest='master_branch', default='master', help="The branch to release. Default: master.")
 383     parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.")
 384     args = parser.parse_args()
 385     main()
 386
 387 # vim: sw=4 et