From: Alex Rousskov Date: Tue, 5 Jul 2022 22:58:13 +0000 (+0000) Subject: Maintenance: Improve stability of generated automake Makefiles (#1081) X-Git-Tag: SQUID_6_0_1~159 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=52d824b56a26de3cd08a51187fb26b95800c0a37;p=thirdparty%2Fsquid.git Maintenance: Improve stability of generated automake Makefiles (#1081) Our source-maintenance.sh script generates five automake Makefiles using "git ls-files " output as the source of filename entries. These entries were piped through "sort -u". That problematic step is now gone: * "sort" may not produce stable results across different locales. * "sort" does not use the order enforced by our format-makefile-am.pl. That Perl script itself did not fix the order because it only ordered entries in *_SOURCES groups that these generated files do not use. * "sort -u" could silently drop duplicate Makefile entries (that should not exist) instead of letting that Perl script warn us about them. The first two bullets made automake Makefile generation unstable (e.g., 502bcc7 reversed the ASCII order of en.lang and en_AU.lang entries, but current runs in unofficial environments could reorder those entries). format-makefile-am.pl now recognizes the generated group name patterns, properly formatting entries in those five generated automake Makefiles. That fix alone created persistent misleading "File ... changed: by ... format-makefile-am.pl" NOTICEs because the Perl script was (always) changing the _generated_ file. We now pipeline the generation and formatting steps, applying NOTICE-generation check to the final result. (Positive) side effect: The generated automake Makefiles now include a notice that they are generated. --- diff --git a/doc/manuals/language.am b/doc/manuals/language.am index 3fd97ca555..1270c4a28d 100644 --- a/doc/manuals/language.am +++ b/doc/manuals/language.am @@ -6,16 +6,18 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## +## This file is generated by scripts/source-maintenance.sh. + TRANSLATE_LANGUAGES = \ - ar.lang \ - cs.lang \ - de.lang \ - en_AU.lang \ - en.lang \ - es.lang \ - fr.lang \ - it.lang \ - oc.lang \ - pt.lang \ - ro.lang \ - ru.lang + ar.lang \ + cs.lang \ + de.lang \ + en.lang \ + en_AU.lang \ + es.lang \ + fr.lang \ + it.lang \ + oc.lang \ + pt.lang \ + ro.lang \ + ru.lang diff --git a/errors/language.am b/errors/language.am index c3a610697e..e91130b828 100644 --- a/errors/language.am +++ b/errors/language.am @@ -6,50 +6,52 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## +## This file is generated by scripts/source-maintenance.sh. + TRANSLATE_LANGUAGES = \ - af.lang \ - ar.lang \ - az.lang \ - bg.lang \ - ca.lang \ - cs.lang \ - da.lang \ - de.lang \ - el.lang \ - en.lang \ - es-mx.lang \ - es.lang \ - et.lang \ - fa.lang \ - fi.lang \ - fr.lang \ - he.lang \ - hu.lang \ - hy.lang \ - id.lang \ - it.lang \ - ja.lang \ - ka.lang \ - ko.lang \ - lt.lang \ - lv.lang \ - ms.lang \ - nl.lang \ - oc.lang \ - pl.lang \ - pt-br.lang \ - pt.lang \ - ro.lang \ - ru.lang \ - sk.lang \ - sl.lang \ - sr-cyrl.lang \ - sr-latn.lang \ - sv.lang \ - th.lang \ - tr.lang \ - uk.lang \ - uz.lang \ - vi.lang \ - zh-hans.lang \ - zh-hant.lang + af.lang \ + ar.lang \ + az.lang \ + bg.lang \ + ca.lang \ + cs.lang \ + da.lang \ + de.lang \ + el.lang \ + en.lang \ + es-mx.lang \ + es.lang \ + et.lang \ + fa.lang \ + fi.lang \ + fr.lang \ + he.lang \ + hu.lang \ + hy.lang \ + id.lang \ + it.lang \ + ja.lang \ + ka.lang \ + ko.lang \ + lt.lang \ + lv.lang \ + ms.lang \ + nl.lang \ + oc.lang \ + pl.lang \ + pt-br.lang \ + pt.lang \ + ro.lang \ + ru.lang \ + sk.lang \ + sl.lang \ + sr-cyrl.lang \ + sr-latn.lang \ + sv.lang \ + th.lang \ + tr.lang \ + uk.lang \ + uz.lang \ + vi.lang \ + zh-hans.lang \ + zh-hant.lang diff --git a/errors/template.am b/errors/template.am index 3147a981bb..404482ee1c 100644 --- a/errors/template.am +++ b/errors/template.am @@ -6,46 +6,48 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## +## This file is generated by scripts/source-maintenance.sh. + ERROR_TEMPLATES = \ - templates/ERR_ACCESS_DENIED \ - templates/ERR_ACL_TIME_QUOTA_EXCEEDED \ - templates/ERR_AGENT_CONFIGURE \ - templates/ERR_AGENT_WPAD \ - templates/ERR_CACHE_ACCESS_DENIED \ - templates/ERR_CACHE_MGR_ACCESS_DENIED \ - templates/ERR_CANNOT_FORWARD \ - templates/ERR_CONFLICT_HOST \ - templates/ERR_CONNECT_FAIL \ - templates/ERR_DIR_LISTING \ - templates/ERR_DNS_FAIL \ - templates/ERR_ESI \ - templates/ERR_FORWARDING_DENIED \ - templates/ERR_FTP_DISABLED \ - templates/ERR_FTP_FAILURE \ - templates/ERR_FTP_FORBIDDEN \ - templates/ERR_FTP_NOT_FOUND \ - templates/ERR_FTP_PUT_CREATED \ - templates/ERR_FTP_PUT_ERROR \ - templates/ERR_FTP_PUT_MODIFIED \ - templates/ERR_FTP_UNAVAILABLE \ - templates/ERR_GATEWAY_FAILURE \ - templates/ERR_ICAP_FAILURE \ - templates/ERR_INVALID_REQ \ - templates/ERR_INVALID_RESP \ - templates/ERR_INVALID_URL \ - templates/ERR_LIFETIME_EXP \ - templates/ERR_NO_RELAY \ - templates/ERR_ONLY_IF_CACHED_MISS \ - templates/ERR_PRECONDITION_FAILED \ - templates/ERR_PROTOCOL_UNKNOWN \ - templates/ERR_READ_ERROR \ - templates/ERR_READ_TIMEOUT \ - templates/ERR_SECURE_CONNECT_FAIL \ - templates/ERR_SHUTTING_DOWN \ - templates/ERR_SOCKET_FAILURE \ - templates/ERR_TOO_BIG \ - templates/ERR_UNSUP_HTTPVERSION \ - templates/ERR_UNSUP_REQ \ - templates/ERR_URN_RESOLVE \ - templates/ERR_WRITE_ERROR \ - templates/ERR_ZERO_SIZE_OBJECT + templates/ERR_ACCESS_DENIED \ + templates/ERR_ACL_TIME_QUOTA_EXCEEDED \ + templates/ERR_AGENT_CONFIGURE \ + templates/ERR_AGENT_WPAD \ + templates/ERR_CACHE_ACCESS_DENIED \ + templates/ERR_CACHE_MGR_ACCESS_DENIED \ + templates/ERR_CANNOT_FORWARD \ + templates/ERR_CONFLICT_HOST \ + templates/ERR_CONNECT_FAIL \ + templates/ERR_DIR_LISTING \ + templates/ERR_DNS_FAIL \ + templates/ERR_ESI \ + templates/ERR_FORWARDING_DENIED \ + templates/ERR_FTP_DISABLED \ + templates/ERR_FTP_FAILURE \ + templates/ERR_FTP_FORBIDDEN \ + templates/ERR_FTP_NOT_FOUND \ + templates/ERR_FTP_PUT_CREATED \ + templates/ERR_FTP_PUT_ERROR \ + templates/ERR_FTP_PUT_MODIFIED \ + templates/ERR_FTP_UNAVAILABLE \ + templates/ERR_GATEWAY_FAILURE \ + templates/ERR_ICAP_FAILURE \ + templates/ERR_INVALID_REQ \ + templates/ERR_INVALID_RESP \ + templates/ERR_INVALID_URL \ + templates/ERR_LIFETIME_EXP \ + templates/ERR_NO_RELAY \ + templates/ERR_ONLY_IF_CACHED_MISS \ + templates/ERR_PRECONDITION_FAILED \ + templates/ERR_PROTOCOL_UNKNOWN \ + templates/ERR_READ_ERROR \ + templates/ERR_READ_TIMEOUT \ + templates/ERR_SECURE_CONNECT_FAIL \ + templates/ERR_SHUTTING_DOWN \ + templates/ERR_SOCKET_FAILURE \ + templates/ERR_TOO_BIG \ + templates/ERR_UNSUP_HTTPVERSION \ + templates/ERR_UNSUP_REQ \ + templates/ERR_URN_RESOLVE \ + templates/ERR_WRITE_ERROR \ + templates/ERR_ZERO_SIZE_OBJECT diff --git a/icons/icon.am b/icons/icon.am index c98b07306f..c9dd05e6b7 100644 --- a/icons/icon.am +++ b/icons/icon.am @@ -6,54 +6,56 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## +## This file is generated by scripts/source-maintenance.sh. + ICONS = \ - silk/application.png \ - silk/arrow_up.png \ - silk/bomb.png \ - silk/box.png \ - silk/bricks.png \ - silk/bullet_red.png \ - silk/cd.png \ - silk/chart_line.png \ - silk/compress.png \ - silk/computer_link.png \ - silk/css.png \ - silk/cup.png \ - silk/database.png \ - silk/database_table.png \ - silk/drive_disk.png \ - silk/film_key.png \ - silk/film.png \ - silk/folder.png \ - silk/folder_table.png \ - silk/image.png \ - silk/information.png \ - silk/layers.png \ - silk/layout.png \ - silk/link.png \ - silk/music.png \ - silk/package_go.png \ - silk/package.png \ - silk/page_code.png \ - silk/page_excel.png \ - silk/page_green.png \ - silk/page_white_acrobat.png \ - silk/page_white_cplusplus.png \ - silk/page_white_c.png \ - silk/page_white_flash.png \ - silk/page_white_magnify.png \ - silk/page_white_picture.png \ - silk/page_white.png \ - silk/page_white_powerpoint.png \ - silk/page_white_stack.png \ - silk/page_white_text.png \ - silk/page_white_word.png \ - silk/page_white_zip.png \ - silk/page_world.png \ - silk/photo.png \ - silk/picture.png \ - silk/plugin_add.png \ - silk/plugin.png \ - silk/script_gear.png \ - silk/script_palette.png \ - silk/script.png + silk/application.png \ + silk/arrow_up.png \ + silk/bomb.png \ + silk/box.png \ + silk/bricks.png \ + silk/bullet_red.png \ + silk/cd.png \ + silk/chart_line.png \ + silk/compress.png \ + silk/computer_link.png \ + silk/css.png \ + silk/cup.png \ + silk/database.png \ + silk/database_table.png \ + silk/drive_disk.png \ + silk/film.png \ + silk/film_key.png \ + silk/folder.png \ + silk/folder_table.png \ + silk/image.png \ + silk/information.png \ + silk/layers.png \ + silk/layout.png \ + silk/link.png \ + silk/music.png \ + silk/package.png \ + silk/package_go.png \ + silk/page_code.png \ + silk/page_excel.png \ + silk/page_green.png \ + silk/page_white.png \ + silk/page_white_acrobat.png \ + silk/page_white_c.png \ + silk/page_white_cplusplus.png \ + silk/page_white_flash.png \ + silk/page_white_magnify.png \ + silk/page_white_picture.png \ + silk/page_white_powerpoint.png \ + silk/page_white_stack.png \ + silk/page_white_text.png \ + silk/page_white_word.png \ + silk/page_white_zip.png \ + silk/page_world.png \ + silk/photo.png \ + silk/picture.png \ + silk/plugin.png \ + silk/plugin_add.png \ + silk/script.png \ + silk/script_gear.png \ + silk/script_palette.png diff --git a/scripts/format-makefile-am.pl b/scripts/format-makefile-am.pl index 875cdb590b..e41f200f7b 100755 --- a/scripts/format-makefile-am.pl +++ b/scripts/format-makefile-am.pl @@ -19,7 +19,9 @@ while (<>) { } # accumulate files and prep for sorting my %files = (); - if (/^(\S+_SOURCES)\s*(\+?=)\s*(.*[^\\])$/ ) { + # TODO: Handle or rename /\S+SOURCE=/ and /\S*[^_]SOURCES=/ + my $groupNameRx = qr/\S+_SOURCES|ICONS|\S+_TEMPLATES|\S+_LANGUAGES|STUB_SOURCE/; + if (/^($groupNameRx)\s*(\+?=)\s*(.*[^\\])$/ ) { my @parts = split(/\s+/, $3); if ($#parts == 0) { # one file only specified on same line as SOURCES print "$1 $2 $3\n"; @@ -32,7 +34,7 @@ while (<>) { &print_files(\%files); next; } - if (/^(\S+_SOURCES)\s*(\+?=)\s*(.*?)\s*\\$/) { + if (/^($groupNameRx)\s*(\+?=)\s*(.*?)\s*\\$/) { $current_source_section=$1; print "$1 $2 \\\n"; if (defined $3) { diff --git a/scripts/source-maintenance.sh b/scripts/source-maintenance.sh index 03ad33dff7..3872d7a0d4 100755 --- a/scripts/source-maintenance.sh +++ b/scripts/source-maintenance.sh @@ -108,6 +108,9 @@ if ! git diff --quiet; then exit 1 fi +made="generated" # a hack: prevents $GeneratedByMe searches matching this file +GeneratedByMe="This file is $made by scripts/source-maintenance.sh." + # On squid-cache.org we have to use the python scripted md5sum HOST=`hostname` if test "$HOST" = "squid-cache.org" ; then @@ -441,8 +444,11 @@ for FILENAME in `git ls-files`; do ;; *.am) - applyPluginsTo ${FILENAME} scripts/format-makefile-am.pl || return - ;; + # generated automake files are formatted during their generation + if ! grep -q -F "$GeneratedByMe" ${FILENAME}; then + applyPluginsTo ${FILENAME} scripts/format-makefile-am.pl || return + fi + ;; ChangeLog|CREDITS|CONTRIBUTORS|COPYING|*.png|*.po|*.pot|rfcs/|*.txt|test-suite/squidconf/empty|.bzrignore) # we do not enforce copyright blurbs in: @@ -479,31 +485,52 @@ done run_ processDebugMessages || return } -printAmFile () +printRawAmFile () { sed -e 's%\ \*%##%; s%/\*%##%; s%##/%##%' < scripts/boilerplate.h + + echo "## $GeneratedByMe" + echo + echo -n "$1 =" - git ls-files $2$3 | sed -e s%$2%%g | sort -u | while read f; do + # Only some files are formed from *.po filenames, but all such files + # should list *.lang filenames instead. + git ls-files $2$3 | sed -e s%$2%%g -e 's%\.po%\.lang%g' | while read f; do echo " \\" echo -n " ${f}" done echo "" } +generateAmFile () +{ + amFile="$1" + shift + + # format immediately/here instead of in srcFormat to avoid misleading + # "NOTICE: File ... changed by scripts/format-makefile-am.pl" in srcFormat + printRawAmFile "$@" | scripts/format-makefile-am.pl > $amFile.new + + # Distinguishing generation-only changes from formatting-only changes is + # difficult, so we only check/report cumulative changes. Most interesting + # changes are triggered by printRawAmFile() finding new entries. + updateIfChanged $amFile $amFile.new 'by generateAmFile()' +} + # Build icons install include from current icons available -printAmFile ICONS "icons/" "silk/*" > icons/icon.am +generateAmFile icons/icon.am ICONS "icons/" "silk/*" # Build templates install include from current templates available -printAmFile ERROR_TEMPLATES "errors/" "templates/ERR_*" > errors/template.am +generateAmFile errors/template.am ERROR_TEMPLATES "errors/" "templates/ERR_*" # Build errors translation install include from current .PO available -printAmFile TRANSLATE_LANGUAGES "errors/" "*.po" | sed 's%\.po%\.lang%g' > errors/language.am +generateAmFile errors/language.am TRANSLATE_LANGUAGES "errors/" "*.po" # Build manuals translation install include from current .PO available -printAmFile TRANSLATE_LANGUAGES "doc/manuals/" "*.po" | sed 's%\.po%\.lang%g' > doc/manuals/language.am +generateAmFile doc/manuals/language.am TRANSLATE_LANGUAGES "doc/manuals/" "*.po" # Build STUB framework include from current stub_* available -printAmFile STUB_SOURCE "src/" "tests/stub_*.cc" > src/tests/Stub.am +generateAmFile src/tests/Stub.am STUB_SOURCE "src/" "tests/stub_*.cc" # Build the GPERF generated content make -C src/http gperf-files diff --git a/src/tests/Stub.am b/src/tests/Stub.am index df8894116a..a264704e33 100644 --- a/src/tests/Stub.am +++ b/src/tests/Stub.am @@ -6,89 +6,91 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## +## This file is generated by scripts/source-maintenance.sh. + STUB_SOURCE = \ - tests/stub_access_log.cc \ - tests/stub_acl.cc \ - tests/stub_cache_cf.cc \ - tests/stub_CacheDigest.cc \ - tests/stub_cache_manager.cc \ - tests/stub_CachePeer.cc \ - tests/stub_carp.cc \ - tests/stub_cbdata.cc \ - tests/stub_client_db.cc \ - tests/stub_client_side.cc \ - tests/stub_client_side_request.cc \ - tests/stub_CollapsedForwarding.cc \ - tests/stub_comm.cc \ - tests/stub_CommIO.cc \ - tests/stub_debug.cc \ - tests/stub_DelayId.cc \ - tests/stub_errorpage.cc \ - tests/stub_ETag.cc \ - tests/stub_event.cc \ - tests/stub_EventLoop.cc \ - tests/stub_external_acl.cc \ - tests/stub_fatal.cc \ - tests/stub_fd.cc \ - tests/stub_fqdncache.cc \ - tests/stub_gopher.cc \ - tests/stub_helper.cc \ - tests/stub_HelperChildConfig.cc \ - tests/stub_http.cc \ - tests/stub_HttpControlMsg.cc \ - tests/stub_HttpHeader.cc \ - tests/stub_HttpReply.cc \ - tests/stub_HttpRequest.cc \ - tests/stub_HttpUpgradeProtocolAccess.cc \ - tests/stub_icp.cc \ - tests/stub_internal.cc \ - tests/stub_ipcache.cc \ - tests/stub_ipc.cc \ - tests/stub_ipc_Forwarder.cc \ - tests/stub_IpcIoFile.cc \ - tests/stub_ipc_TypedMsgHdr.cc \ - tests/stub_libanyp.cc \ - tests/stub_libauth_acls.cc \ - tests/stub_libauth.cc \ - tests/stub_libcomm.cc \ - tests/stub_libdiskio.cc \ - tests/stub_liberror.cc \ - tests/stub_libeui.cc \ - tests/stub_libformat.cc \ - tests/stub_libhttp.cc \ - tests/stub_libicmp.cc \ - tests/stub_libip.cc \ - tests/stub_liblog.cc \ - tests/stub_libmem.cc \ - tests/stub_libmgr.cc \ - tests/stub_libsecurity.cc \ - tests/stub_libsslsquid.cc \ - tests/stub_libstore.cc \ - tests/stub_libtime.cc \ - tests/stub_main_cc.cc \ - tests/stub_MemBuf.cc \ - tests/stub_mem_node.cc \ - tests/stub_MemObject.cc \ - tests/stub_MemStore.cc \ - tests/stub_mime.cc \ - tests/stub_neighbors.cc \ - tests/stub_pconn.cc \ - tests/stub_Port.cc \ - tests/stub_redirect.cc \ - tests/stub_SBuf.cc \ - tests/stub_stat.cc \ - tests/stub_StatHist.cc \ - tests/stub_stmem.cc \ - tests/stub_store.cc \ - tests/stub_store_client.cc \ - tests/stub_store_digest.cc \ - tests/stub_StoreMeta.cc \ - tests/stub_store_rebuild.cc \ - tests/stub_store_stats.cc \ - tests/stub_store_swapout.cc \ - tests/stub_tools.cc \ - tests/stub_tunnel.cc \ - tests/stub_UdsOp.cc \ - tests/stub_wccp2.cc \ - tests/stub_whois.cc \ - tests/stub_wordlist.cc + tests/stub_CacheDigest.cc \ + tests/stub_CachePeer.cc \ + tests/stub_CollapsedForwarding.cc \ + tests/stub_CommIO.cc \ + tests/stub_DelayId.cc \ + tests/stub_ETag.cc \ + tests/stub_EventLoop.cc \ + tests/stub_HelperChildConfig.cc \ + tests/stub_HttpControlMsg.cc \ + tests/stub_HttpHeader.cc \ + tests/stub_HttpReply.cc \ + tests/stub_HttpRequest.cc \ + tests/stub_HttpUpgradeProtocolAccess.cc \ + tests/stub_IpcIoFile.cc \ + tests/stub_MemBuf.cc \ + tests/stub_MemObject.cc \ + tests/stub_MemStore.cc \ + tests/stub_Port.cc \ + tests/stub_SBuf.cc \ + tests/stub_StatHist.cc \ + tests/stub_StoreMeta.cc \ + tests/stub_UdsOp.cc \ + tests/stub_access_log.cc \ + tests/stub_acl.cc \ + tests/stub_cache_cf.cc \ + tests/stub_cache_manager.cc \ + tests/stub_carp.cc \ + tests/stub_cbdata.cc \ + tests/stub_client_db.cc \ + tests/stub_client_side.cc \ + tests/stub_client_side_request.cc \ + tests/stub_comm.cc \ + tests/stub_debug.cc \ + tests/stub_errorpage.cc \ + tests/stub_event.cc \ + tests/stub_external_acl.cc \ + tests/stub_fatal.cc \ + tests/stub_fd.cc \ + tests/stub_fqdncache.cc \ + tests/stub_gopher.cc \ + tests/stub_helper.cc \ + tests/stub_http.cc \ + tests/stub_icp.cc \ + tests/stub_internal.cc \ + tests/stub_ipc.cc \ + tests/stub_ipc_Forwarder.cc \ + tests/stub_ipc_TypedMsgHdr.cc \ + tests/stub_ipcache.cc \ + tests/stub_libanyp.cc \ + tests/stub_libauth.cc \ + tests/stub_libauth_acls.cc \ + tests/stub_libcomm.cc \ + tests/stub_libdiskio.cc \ + tests/stub_liberror.cc \ + tests/stub_libeui.cc \ + tests/stub_libformat.cc \ + tests/stub_libhttp.cc \ + tests/stub_libicmp.cc \ + tests/stub_libip.cc \ + tests/stub_liblog.cc \ + tests/stub_libmem.cc \ + tests/stub_libmgr.cc \ + tests/stub_libsecurity.cc \ + tests/stub_libsslsquid.cc \ + tests/stub_libstore.cc \ + tests/stub_libtime.cc \ + tests/stub_main_cc.cc \ + tests/stub_mem_node.cc \ + tests/stub_mime.cc \ + tests/stub_neighbors.cc \ + tests/stub_pconn.cc \ + tests/stub_redirect.cc \ + tests/stub_stat.cc \ + tests/stub_stmem.cc \ + tests/stub_store.cc \ + tests/stub_store_client.cc \ + tests/stub_store_digest.cc \ + tests/stub_store_rebuild.cc \ + tests/stub_store_stats.cc \ + tests/stub_store_swapout.cc \ + tests/stub_tools.cc \ + tests/stub_tunnel.cc \ + tests/stub_wccp2.cc \ + tests/stub_whois.cc \ + tests/stub_wordlist.cc