From ec3c9f2f5a6af532b1b5b80ed9baf477500dc8d5 Mon Sep 17 00:00:00 2001 From: Wayne Davison Date: Sun, 28 Jun 2020 13:22:15 -0700 Subject: [PATCH] Improve alternate build-dir support We now put the configure.sh, config.h.in, and aclocal.m4 files in the alternate build dir along with the other generated files. This requires that we create symlinks for configure.ac & m4 in the build dir, which is handled on the first run of configure or prepare-source. I also changed the patch-branch handling away from the .gen-stash dir to an automatic build/$PATCH subdir idiom that will keep each branch's configuration separated. These automatic build dirs are only used when there is a .git dir, a build/master dir, and no top-dir Makefile. You'll also want to have package/make early on your path for optimal ease of use. --- .gitignore | 2 +- Makefile.in | 11 ++++++----- configure | 28 +++++++++++++--------------- packaging/make | 18 ++++++++++++++++++ packaging/patch-update | 41 ++--------------------------------------- packaging/pkglib.py | 15 ++++++++++----- packaging/release-rsync | 19 ++++++++++--------- prepare-source | 31 ++++++++++++++++++++++++++++--- prepare-source.mak | 3 +++ testsuite/itemize.test | 4 ++-- 10 files changed, 93 insertions(+), 79 deletions(-) create mode 100755 packaging/make diff --git a/.gitignore b/.gitignore index 62cf9a54..6080cdac 100644 --- a/.gitignore +++ b/.gitignore @@ -49,5 +49,5 @@ aclocal.m4 /testsuite/xattrs-hlink.test /patches /SaVeDiR -/.gen-stash +/build .deps diff --git a/Makefile.in b/Makefile.in index 6b2d5cec..725ee420 100644 --- a/Makefile.in +++ b/Makefile.in @@ -169,16 +169,17 @@ aclocal.m4: $(srcdir)/m4/*.m4 aclocal -I $(srcdir)/m4 configure.sh config.h.in: configure.ac aclocal.m4 - @test -f "$(srcdir)/configure.sh" && cp -p "$(srcdir)/configure.sh" configure.sh.old || touch configure.sh.old - @test -f "$(srcdir)/config.h.in" && cp -p "$(srcdir)/config.h.in" config.h.in.old || touch config.h.in.old - (cd "$(srcdir)" && autoconf -o configure.sh && autoheader && touch config.h.in) - @if diff "$(srcdir)/configure.sh" configure.sh.old >/dev/null 2>&1; then \ + @if test -f configure.sh; then cp -p configure.sh configure.sh.old; else touch configure.sh.old; fi + @if test -f config.h.in; then cp -p config.h.in config.h.in.old; else touch config.h.in.old; fi + autoconf -o configure.sh + autoheader && touch config.h.in + @if diff configure.sh configure.sh.old >/dev/null 2>&1; then \ echo "configure.sh is unchanged."; \ rm configure.sh.old; \ else \ echo "configure.sh has CHANGED."; \ fi - @if diff "$(srcdir)/config.h.in" config.h.in.old >/dev/null 2>&1; then \ + @if diff config.h.in config.h.in.old >/dev/null 2>&1; then \ echo "config.h.in is unchanged."; \ rm config.h.in.old; \ else \ diff --git a/configure b/configure index 32cc3b5d..75b9ed7c 100755 --- a/configure +++ b/configure @@ -4,24 +4,22 @@ # then transfer control to the configure.sh script to do the real work. dir=`dirname $0` -realconfigure="$dir/configure.sh" -if test ! -f "$realconfigure"; then - if test -f "$HOME/build_farm/build_test.fns"; then - # Test the included popt - set -- --with-included-popt "${@}" - # Allow the build farm to grab latest files via rsync. - actions='build fetch' - else - actions='build' - fi - if "$dir/prepare-source" $actions; then - : - else +# Support automatic switching to build/$BRANCH subdirs. It's also good +# to put packaging/make somewhere early in your $PATH if you use this! +if test "$dir" = '.' -a ! -f Makefile -a -d build/master -a -d .git; then + builddir=build/`git rev-parse --abbrev-ref HEAD | tr / %` + test -d "$builddir" || mkdir "$builddir" + cd "$builddir" || exit 1 + dir=../.. +fi + +if test ! -f configure.sh; then + if ! "$dir/prepare-source" build; then echo 'Failed to build configure.sh and/or config.h.in -- giving up.' >&2 - rm -f "$realconfigure" + rm -f configure.sh exit 1 fi fi -exec "$realconfigure" "${@}" +exec ./configure.sh --srcdir="$dir" "${@}" diff --git a/packaging/make b/packaging/make new file mode 100755 index 00000000..91583009 --- /dev/null +++ b/packaging/make @@ -0,0 +1,18 @@ +#!/bin/sh + +# You can put this script early on your PATH to support a top-dir "make" +# changing into a per-branch build dir before running the make. Just start +# with a pristine top dir (e.g. run "make distclean") and then create the dir +# "build/master", at which point you can stay in the top dir of the source and +# run "./configure" and "make" commands and have the appropriate build dir be +# automatically used (the non-master subdirs will be auto-created). + +if [ ! -f Makefile -a -d build/master -a -d .git -a x"$1" != x-* ]; then + builddir=build/`git rev-parse --abbrev-ref HEAD | tr / %` + [ -d "$builddir" ] || mkdir "$builddir" + # Let make do a noisy chdir so that a calling editor knows where the files are. + set -- -C "$builddir" ${1+"$@"} +fi + +export PATH="/usr/lib/ccache:$PATH" +exec /usr/bin/make ${1+"$@"} diff --git a/packaging/patch-update b/packaging/patch-update index c5bbe1c3..7b265288 100755 --- a/packaging/patch-update +++ b/packaging/patch-update @@ -13,9 +13,8 @@ sys.path = ['packaging'] + sys.path from pkglib import * MAKE_GEN_CMDS = [ - 'make -f prepare-source.mak conf'.split(), - './config.status'.split(), - 'make gen'.split(), + './prepare-source'.split(), + 'make restatus gen'.split(), ] TMP_DIR = "patches.gen" @@ -31,12 +30,6 @@ def main(): starting_branch, args.base_branch = check_git_state(args.base_branch, not args.skip_check, args.patches_dir) - if not os.path.isdir('.gen-stash'): - os.mkdir('.gen-stash', 0o700) - gen_stash(starting_branch) - if starting_branch == 'master': - cmd_chk('md5sum configure.ac >.gen-stash/configure.ac.sum') - master_commit = latest_git_hash(args.base_branch) if args.gen: @@ -101,8 +94,6 @@ def main(): time.sleep(1) cmd_chk(['git', 'checkout', starting_branch]) - gen_unstash(starting_branch) - def update_patch(patch): global last_touch @@ -132,7 +123,6 @@ def update_patch(patch): s = cmd_run(['git', 'merge', based_on]) ok = s.returncode == 0 if not ok or args.shell: - gen_unstash(branch) m = re.search(r'([^/]+)$', parent) parent_dir = m[1] if not ok: @@ -149,7 +139,6 @@ def update_patch(patch): if is_clean: break print(status_txt, end='') - gen_stash(branch) with open(f"{args.patches_dir}/{patch}.diff", 'w', encoding='utf-8') as fh: fh.write(description[patch]) @@ -203,32 +192,6 @@ def update_patch(patch): return 1 -# The autoconf cache dir can totally mess up if 2 different *.ac files have the same mtime! -def gen_stash_prep(branch): - if os.path.isdir('autom4te.cache'): - shutil.rmtree('autom4te.cache') - return '.gen-stash/' + branch.replace('/', '%') + '.tar.gz' - - -def gen_stash(branch): - files = 'configure.sh config.h.in Makefile'.split() - for fn in files: - if not os.path.exists(fn): - return - tar_fn = gen_stash_prep(branch) - cmd_run(['./prepare-source']) - cmd_chk(['tar', 'czf', tar_fn, *files]) - - -def gen_unstash(branch): - tar_fn = gen_stash_prep(branch) - if os.path.exists(tar_fn): - cmd_chk(['tar', 'xf', tar_fn]) - out, rc = cmd_txt_status('md5sum --status -c .gen-stash/configure.ac.sum'.split()) - if rc: - print('*' * 10, "configure.ac differs from master version", '*' * 10) - - if __name__ == '__main__': parser = argparse.ArgumentParser(description="Turn a git branch back into a diff files in the patches dir.", add_help=False) parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.") diff --git a/packaging/pkglib.py b/packaging/pkglib.py index 628669b0..c83a1cdc 100644 --- a/packaging/pkglib.py +++ b/packaging/pkglib.py @@ -181,25 +181,30 @@ def mandate_gensend_hook(): # Snag the GENFILES values out of the Makefile.in file and return them as a list. -def get_gen_files(): +def get_gen_files(want_dir_plus_list=False): cont_re = re.compile(r'\\\n') - extras = [ ] + gen_files = [ ] + + builddir = os.path.join('build', cmd_txt('git rev-parse --abbrev-ref HEAD').replace('/', '%')) with open('Makefile.in', 'r', encoding='utf-8') as fh: for line in fh: - if not extras: + if not gen_files: chk = re.sub(r'^GENFILES=', '', line) if line == chk: continue line = chk m = re.search(r'\\$', line) line = re.sub(r'^\s+|\s*\\\n?$|\s+$', '', line) - extras += line.split() + gen_files += line.split() if not m: break - return extras + if want_dir_plus_list: + return (builddir, gen_files) + + return [ os.path.join(builddir, fn) for fn in gen_files ] def get_configure_version(): diff --git a/packaging/release-rsync b/packaging/release-rsync index 541678f0..33a9ef2d 100755 --- a/packaging/release-rsync +++ b/packaging/release-rsync @@ -30,7 +30,8 @@ def main(): signal.signal(signal.SIGINT, signal_handler) - gen_files = get_gen_files() + builddir, gen_files = get_gen_files(True) + gen_pathnames = [ os.path.join(builddir, fn) for fn in gen_files ] dash_line = '=' * 74 @@ -215,10 +216,10 @@ About to: srctar_name = f"{rsync_ver}.tar.gz" pattar_name = f"rsync-patches-{version}.tar.gz" diff_name = f"{rsync_lastver}-{version}.diffs.gz" - srctar_file = f"{dest}/{srcdir}/{srctar_name}" - pattar_file = f"{dest}/{srcdir}/{pattar_name}" - diff_file = f"{dest}/{srcdiffdir}/{diff_name}" - lasttar_file = f"{dest}/{lastsrcdir}/{rsync_lastver}.tar.gz" + srctar_file = os.path.join(dest, srcdir, srctar_name) + pattar_file = os.path.join(dest, srcdir, pattar_name) + diff_file = os.path.join(dest, srcdiffdir, diff_name) + lasttar_file = os.path.join(dest, lastsrcdir, rsync_lastver + '.tar.gz') print(f"""\ {dash_line} @@ -237,7 +238,7 @@ About to: die('Aborting') cmd_chk('make gen') - cmd_chk(['rsync', '-a', *gen_files, 'SaVeDiR/']) + cmd_chk(['rsync', '-a', *gen_pathnames, 'SaVeDiR/']) print(f'Creating any missing patch branches.') s = cmd_run(f'packaging/branch-from-patch --branch={args.master_branch} --add-missing') @@ -294,12 +295,12 @@ About to: os.environ['PATH'] = ORIGINAL_PATH # Extract the generated files from the old tar. - tweaked_gen_files = [ f"{rsync_lastver}/{x}" for x in gen_files ] + tweaked_gen_files = [ os.path.join(rsync_lastver, fn) for fn in gen_files ] cmd_run(['tar', 'xzf', lasttar_file, *tweaked_gen_files]) os.rename(rsync_lastver, 'a') print(f"Creating {diff_file} ...") - cmd_chk(['rsync', '-a', *gen_files, 'b/']) + cmd_chk(['rsync', '-a', *gen_pathnames, 'b/']) sed_script = r's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:' # CAUTION: must not contain any single quotes! cmd_chk(f"(git diff v{lastversion} {v_ver} -- ':!.github'; diff -upN a b | sed -r '{sed_script}') | gzip -9 >{diff_file}") @@ -328,7 +329,7 @@ About to: print(f"Updating the other files in {dest} ...") md_files = 'README.md NEWS.md'.split() - html_files = [ fn for fn in gen_files if fn.endswith('.html') ] + html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ] cmd_chk(['rsync', '-a', *md_files, *html_files, dest]) cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ]) diff --git a/prepare-source b/prepare-source index a41a66a3..8cc7911a 100755 --- a/prepare-source +++ b/prepare-source @@ -16,6 +16,27 @@ if test x"$dir" = x; then dir=. fi +# Support automatic switching to build/$BRANCH subdirs. It's also good +# to put packaging/make somewhere early in your $PATH if you use this! +if test "$dir" = . -a ! -f Makefile -a -d build/master -a -d .git; then + builddir=build/`git rev-parse --abbrev-ref HEAD | tr / %` + test -d "$builddir" || mkdir "$builddir" + cd "$builddir" || exit 1 + dir=../.. +fi + +if test "$dir" != '.'; then + for lnk in configure.ac m4; do + if test ! -h $lnk; then + rm -f $lnk # Just in case + ln -s "$dir/$lnk" $lnk + fi + done + for fn in configure.sh config.h.in aclocal.m4; do + test ! -f $fn -a -f "$dir/$fn" && cp -p "$dir/$fn" $fn + done +fi + if test $# = 0; then set -- build fi @@ -23,13 +44,16 @@ fi for action in "${@}"; do case "$action" in build|make) - (cd $dir && make -f prepare-source.mak) + make -f "$dir/prepare-source.mak" ;; fetch|fetchgen) - $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[ca]*' $dir if test "$action" = fetchgen; then - $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[^ca]*' . + match='*' + else + match='[ca]*' fi + $dir/rsync-ssl -iipc --no-motd "rsync://download.samba.org/rsyncftp/generated-files/$match" ./ + test $? != 0 && continue sleep 1 # The following files need to be newer than aclocal.m4 touch configure.sh config.h.in ;; @@ -39,6 +63,7 @@ for action in "${@}"; do *) echo "Unknown action: $action" exit 1 + ;; esac if test $? = 0; then exit diff --git a/prepare-source.mak b/prepare-source.mak index bc1938de..08b916de 100644 --- a/prepare-source.mak +++ b/prepare-source.mak @@ -1,3 +1,6 @@ +SHELL=/bin/sh + +.PHONY: conf conf: configure.sh config.h.in aclocal.m4: m4/*.m4 diff --git a/testsuite/itemize.test b/testsuite/itemize.test index 553d0cef..bbdfc942 100644 --- a/testsuite/itemize.test +++ b/testsuite/itemize.test @@ -18,7 +18,7 @@ outfile="$scratchdir/rsync.out" makepath "$fromdir/foo" makepath "$fromdir/bar/baz" cp -p "$srcdir/configure.ac" "$fromdir/foo/config1" -cp -p "$srcdir/config.h.in" "$fromdir/foo/config2" +cp -p "$srcdir/config.sub" "$fromdir/foo/config2" cp -p "$srcdir/rsync.h" "$fromdir/bar/baz/rsync" chmod 600 "$fromdir"/foo/config? "$fromdir/bar/baz/rsync" umask 0 @@ -84,7 +84,7 @@ rm "$todir/foo/sym" umask 0 ln -s ../bar/baz "$todir/foo/sym" umask 022 -cp -p "$srcdir/config.h.in" "$fromdir/foo/config2" +cp -p "$srcdir/config.sub" "$fromdir/foo/config2" chmod 600 "$fromdir/foo/config2" chmod 777 "$todir/bar/baz/rsync" -- 2.47.2