diff --git a/appveyor.yml b/appveyor.yml index 3a77e286be..05a752da69 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,7 +28,16 @@ platform: x64 # https://www.appveyor.com/docs/build-cache/ environment: APPVEYOR_SAVE_CACHE_ON_ERROR: true - APPVEYOR_CACHE_ENTRY_ZIP_ARGS: -t7z -m0=lzma -mx=6 + # Store cache entries without (re-)compression: the data is mostly already + # compressed (ccache objects, pacman *.zst packages), and single-threaded + # LZMA at -mx=6 took 1-2 minutes per run for the ccache entry alone. + # The cache limit for PR builds is 1GB total across all entries, so + # ccache max_size is capped below (see install steps) to leave a margin + # for the other entries (pacman packages ~100MB, nut-win-deps, nut-ci): + APPVEYOR_CACHE_ENTRY_ZIP_ARGS: -t7z -mx=0 + # Keep config.h stable across PRs, so ccache can reuse -g3 objects + # (see ci_build.sh for details): + CI_AUTOGEN_STABLE_VERSION: true CCACHE_DIR: /home/appveyor/.ccache DO_CLEAN_NUTCI_CACHE: no DO_USE_NUTCI_CACHE: yes @@ -57,6 +66,19 @@ notifications: # or look for the chapter in nearby lines in later (current) revisions. # Note: not using `time` in scripts currently - they did upset # AppVeyor console log scanner with a /^sys.*/ match (apparently) +# AppVeyor kills the job at 60 minutes, without saving the build cache: +# a cold-ccache build then never gets warm, and an interrupted cache save +# can even lose the old entry. So we set a deadline at 51 minutes after +# the job start, and our build/test/bundle steps fail (via `timeout`) if +# they would run past it. With APPVEYOR_SAVE_CACHE_ON_ERROR this leaves +# a few minutes for the 7z step, artifact upload and saving the cache. +# Each bash step below computes its time left like this (no CMD special +# characters, so it passes through CMD intact): +# R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) +init: + - ps: | + Set-AppveyorBuildVariable -Name CI_DEADLINE_EPOCH -Value ([DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + 51 * 60) + install: - cmd: | REM Do not give pacman cause for complaints: @@ -103,7 +125,7 @@ install: set CHERE_INVOKING=yes REM Start a 64 bit Mingw environment: set MSYSTEM=MINGW64 - C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; pwd ; ccache -sv || echo "SKIP: Could not query ccache stats" ; ccache -o sloppiness=file_macro || true ; ccache -o compression=true || true ' + C:\msys64\usr\bin\bash -lc 'PATH="/mingw64/bin:$PATH" ; export PATH ; pwd ; ccache -sv || echo "SKIP: Could not query ccache stats" ; ccache -o sloppiness=file_macro || true ; ccache -o compression=true || true ; ccache -o max_size=750M || true ' # NOTE: The script aims to configure for minimal build results, e.g. @@ -120,7 +142,7 @@ before_build: set CHERE_INVOKING=yes REM Start a 64 bit Mingw environment: set MSYSTEM=MINGW64 - C:\msys64\usr\bin\bash -lc 'date -u; export MSYS2_PATH ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" bash ./scripts/Windows/build-mingw-prereqs.sh' + C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; export MSYS2_PATH ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" bash ./scripts/Windows/build-mingw-prereqs.sh' build_script: @@ -140,7 +162,7 @@ build_script: REM to find "nearby" program or configuration files (see common.c REM for current implementation). Hard-coded fallback strings may REM end up getting used in those cases. - C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/bin:$PATH" CI_SKIP_CHECK=true CANBUILD_WITH_LIBMODBUS_USB=yes WITH_LIBNUTPRIVATE=true NUT_SSL_VARIANTS=nss ./ci_build.sh --with-docs=no' + C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/bin:$PATH" CI_SKIP_CHECK=true CANBUILD_WITH_LIBMODBUS_USB=yes WITH_LIBNUTPRIVATE=true NUT_SSL_VARIANTS=nss timeout -k 30 "$R" ./ci_build.sh --with-docs=no' after_build: @@ -159,9 +181,9 @@ test_script: REM Start a 64 bit Mingw environment: set MSYSTEM=MINGW64 REM Start Mingw-based integration and unit checks: - C:\msys64\usr\bin\bash -lc 'date -u; NUT_STATEPATH="C:\\Users\\appveyor\\AppData\\Local\\Temp\\nut-test"; mkdir -p "${NUT_STATEPATH}"; export NUT_STATEPATH; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" make -s check || bash -lc "for F in tests/*.log tests/*.trs ; do echo \"===---=== $F :\"; cat \"$F\"; done; exit 1;" ' + C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; NUT_STATEPATH="C:\\Users\\appveyor\\AppData\\Local\\Temp\\nut-test"; mkdir -p "${NUT_STATEPATH}"; export NUT_STATEPATH; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s check || bash -lc "for F in tests/*.log tests/*.trs ; do echo \"===---=== $F :\"; cat \"$F\"; done; exit 1;" ' REM Start a Mingw-based documentation spellcheck (note that "make check" above could cover some documents in path of artifact delivery, but maybe not all that we have): - C:\msys64\usr\bin\bash -lc 'date -u; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" make -s -j 8 spellcheck' + C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 8 spellcheck' after_test: @@ -172,7 +194,7 @@ after_test: set MSYSTEM=MINGW64 REM Oh the joys of shell scripting with strings passed through CMD: REM Note: currently Python installation path with MSYS is buggy [#1584] - C:\msys64\usr\bin\bash -lc 'date -u; set -e ; if ! rm -rf ".inst" ; then echo "WARNING: Failed to clean away .inst" ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" make -s -j 8 install-win-bundle DESTDIR="`pwd`/.inst/NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%"' + C:\msys64\usr\bin\bash -lc 'date -u; R=$(( ${CI_DEADLINE_EPOCH:-EPOCHSECONDS+3600} - EPOCHSECONDS )) ; echo "CI deadline: $R seconds left" ; if [ "$R" -le 0 ] ; then exit 124 ; fi ; set -e ; if ! rm -rf ".inst" ; then echo "WARNING: Failed to clean away .inst" ; fi ; PATH="/mingw64/lib/ccache/bin:/mingw64/bin:$PATH" timeout -k 30 "$R" make -s -j 8 install-win-bundle DESTDIR="`pwd`/.inst/NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%"' C:\msys64\usr\bin\bash -lc 'date -u; set -e ; rm -rf ./.inst/NUT-for-Windows-x86_64-SNAPSHOT || true ; ln -fs "NUT-for-Windows-x86_64-SNAPSHOT-%APPVEYOR_BUILD_VERSION%" ./.inst/NUT-for-Windows-x86_64-SNAPSHOT ; ( cd .inst/NUT-for-Windows-x86_64-SNAPSHOT ; find . -ls ; )' C:\msys64\usr\bin\bash -lc 'date -u' cd .inst diff --git a/ci_build.sh b/ci_build.sh index c781c42540..72efc0e490 100755 --- a/ci_build.sh +++ b/ci_build.sh @@ -1135,11 +1135,32 @@ do_autogen_get_CONFIGURE_SCRIPT() { # older system) we have to remove it when we already have the script. # This matches the use-case of distro-building from release tarballs that # include all needed pre-generated files to rely less on OS facilities. - if [ "$CI_OS_NAME" = "windows" ] ; then - $CI_TIME ./autogen.sh || true - else - $CI_TIME ./autogen.sh ### 2>/dev/null - fi || exit + ( + # Optionally drop the PR-specific last component of the version + # (e.g. 2.8.5.1466.13 => 2.8.5.1466) which AC_INIT puts into + # config.h as PACKAGE_VERSION. The ccache direct mode hashes all + # included headers, so a version change misses for nearly every + # file and defeats ccache across pull requests. Only autogen.sh gets + # this setting, so `make` still generates nut_version.h (and the + # UPS_VERSION reported by programs) from actual Git metadata. + if [ x"${CI_AUTOGEN_STABLE_VERSION-}" = xtrue ] && [ -z "${NUT_VERSION_FORCED-}" ] ; then + _NUT_VERSION_STABLE="`NUT_VERSION_QUERY=VER50 ./tools/gitlog2version.sh 2>/dev/null`" || _NUT_VERSION_STABLE="" + case "${_NUT_VERSION_STABLE}" in + *.*.*.*.*) _NUT_VERSION_STABLE="${_NUT_VERSION_STABLE%.*}" ;; + esac + if [ -n "${_NUT_VERSION_STABLE}" ] ; then + echo "INFO: CI_AUTOGEN_STABLE_VERSION: using NUT_VERSION_FORCED='${_NUT_VERSION_STABLE}' for autogen.sh" >&2 + NUT_VERSION_FORCED="${_NUT_VERSION_STABLE}" + export NUT_VERSION_FORCED + fi + fi + + if [ "$CI_OS_NAME" = "windows" ] ; then + $CI_TIME ./autogen.sh || true + else + $CI_TIME ./autogen.sh ### 2>/dev/null + fi + ) || exit fi # Retain the full path to configure script file diff --git a/scripts/Windows/dllldd.sh b/scripts/Windows/dllldd.sh index 2ef3dcfe2f..88b056bef8 100755 --- a/scripts/Windows/dllldd.sh +++ b/scripts/Windows/dllldd.sh @@ -81,8 +81,23 @@ discover_COMPILER_PATHS() { } discover_COMPILER_PATHS +# Only scan initialized data sections for DLL names (as older binutils did +# by default), not the whole file: with debug info (-g3) most of our binary +# size is debug sections, so a full scan produced ~200 times more text (all +# piped through filters below), with the same DLL names found. +# Fall back to a whole-file scan if this `strings` lacks the option. +STRINGS_DATA_ONLY="" +if (strings --help 2>&1 | ${GREP} -- '--data' >/dev/null) ; then + STRINGS_DATA_ONLY="-d" +fi + filter_away_system_DLLs() { - ${EGREP} -v -i '^(/.*/)?(msvcrt|userenv|bcrypt|bcryptprimitives|dnsapi|dwrite|iphlpapi|kernelbase|mswsock|shlwapi|winmm|rpcrt4|usp10|ntdll|api-ms-win-[^ ]*|(advapi|crypt|kernel|user|wsock|ws2_|gdi|ole|shell)(32|64))'"${DLLEXT_REGEX_EOL}" + ${EGREP} -v -i '^(/.*/)?(msvcrt|userenv|bcrypt|bcryptprimitives|combase|dnsapi|dwrite|hid|iphlpapi|kernelbase|mswsock|setupapi|shfolder|shlwapi|winmm|wship6|xmllite|rpcrt4|usp10|ntdll|api-ms-win-[^ ]*|(advapi|crypt|kernel|user|wsock|ws2_|gdi|ole|shell)(32|64))'"${DLLEXT_REGEX_EOL}" +} + +first_file_named() { + # Print the first line of stdin (a file path) whose base name is "$1" + awk -v f="$1" '{ n = $0; sub(/^.*\//, "", n); if (n == f) { print; exit } }' } filter_away_NUT_DLLs() { @@ -132,24 +147,37 @@ dllldd_with_tools() ( SEEN=0 NOTSEEN_OD="" if [ -n "${ARCH-}${MINGW_PREFIX-}${MSYSTEM_PREFIX-}" ] ; then + # List candidate locations once, rather than calling `find` for + # each DLL name: spawning processes and walking big directories + # (e.g. /mingw64/bin) is slow on MSYS2, and this adds up quickly + # in recursive searches. Listing order is kept as it was in the + # per-name searches, so the first hit is the same file as before. + DESTDIR_FILES="" + if [ -n "$DESTDIR" -a -d "${DESTDIR}" ] ; then + DESTDIR_FILES="`find \"$DESTDIR\" -type f \! -size 0 2>/dev/null`" + fi + SEARCH_INPUT_FILES="" + if [ -n "$SEARCH_INPUT_PATH" ] ; then + SEARCH_INPUT_FILES="`for D in ${SEARCH_INPUT_PATH_MULTILINE} ; do find \"$D\" -type f \! -size 0 2>/dev/null ; done`" + fi + for OD in objdump "$ARCH-objdump" ; do (command -v "$OD" >/dev/null 2>/dev/null) || continue - ODOUT="`$OD -x \"$@\" 2>/dev/null | ${EGREP} -i '(DLL Name:|^'"${REGEX_WS}"'*NEEDED)' | awk '{print $NF}' | sort | uniq | filter_away_system_DLLs`" \ + # Private headers (-p) hold the PE import table ("DLL Name:") + # and ELF dynamic section ("NEEDED"); -x would also dump the + # (large, for debug builds) symbol tables and relocations. + ODOUT="`$OD -p \"$@\" 2>/dev/null | ${EGREP} -i '(DLL Name:|^'"${REGEX_WS}"'*NEEDED)' | awk '{print $NF}' | sort | uniq | filter_away_system_DLLs`" \ && [ -n "$ODOUT" ] || continue for F in $ODOUT ; do - if [ -n "$DESTDIR" -a -d "${DESTDIR}" ] ; then - OUT="`find \"$DESTDIR\" -type f -name \"$F\" \! -size 0 2>/dev/null | head -1`" \ + if [ -n "$DESTDIR_FILES" ] ; then + OUT="`echo \"$DESTDIR_FILES\" | first_file_named \"$F\"`" \ && [ -n "$OUT" ] && { echo "$OUT" ; SEEN="`expr $SEEN + 1`" ; continue ; } fi - if [ -n "$SEARCH_INPUT_PATH" ] ; then - SEEN_INPUT=false - for D in ${SEARCH_INPUT_PATH_MULTILINE} ; do - OUT="`find \"$D\" -type f -name \"$F\" \! -size 0 2>/dev/null | head -1`" \ - && [ -n "$OUT" ] && { echo "$OUT" ; SEEN_INPUT=true ; break ; } - done - $SEEN_INPUT && { SEEN="`expr $SEEN + 1`" ; continue ; } + if [ -n "$SEARCH_INPUT_FILES" ] ; then + OUT="`echo \"$SEARCH_INPUT_FILES\" | first_file_named \"$F\"`" \ + && [ -n "$OUT" ] && { echo "$OUT" ; SEEN="`expr $SEEN + 1`" ; continue ; } fi if [ -n "$ARCH" -a -d "/usr/${ARCH}" ] ; then OUT="`ls -1 \"/usr/${ARCH}/bin/$F\" \"/usr/${ARCH}/lib/$F\" 2>/dev/null || true`" \ @@ -201,10 +229,10 @@ dllldd_with_tools() ( ) dllldd_with_strings() ( - strings "$@" | tr ' ' '\n' | tr '&' '\n' \ + strings ${STRINGS_DATA_ONLY} "$@" | tr ' ' '\n' | tr '&' '\n' \ | ${EGREP} -i '..*'"${DLLEXT_REGEX_EOL}" | sort | uniq \ | filter_away_system_DLLs \ - | ${EGREP} -vi '^(lib)*%s'"${DLLEXT_REGEX_EOL}" \ + | ${EGREP} -vi '^(.*[/\\])?(lib)*%s'"${DLLEXT_REGEX_EOL}" \ | while read DLL ; do ( # Skip out if we already reported this file # (The for/case loop below is surprisingly expensive on MSYS2) diff --git a/tests/NIT/nit.sh b/tests/NIT/nit.sh index 738586886a..c1095f6c3d 100755 --- a/tests/NIT/nit.sh +++ b/tests/NIT/nit.sh @@ -1235,7 +1235,7 @@ if [ x"${DO_USE_NIT_TESTCERT_CACHE-}" = xyes ] ; then log_info "Found cached NIT certificates in ${CI_CACHE_NIT_HASHDIR}" mkdir -p "${TESTCERT_PATH_BASE}" # See comments above about no TESTCERT_PATH_SEP for shell globs. - cp -prf "${CI_CACHE_NIT_HASHDIR}"/* "${TESTCERT_PATH_BASE}${TESTCERT_PATH_SEP}" || \\ + cp -prf "${CI_CACHE_NIT_HASHDIR}"/* "${TESTCERT_PATH_BASE}${TESTCERT_PATH_SEP}" || \ cp -prfL "${CI_CACHE_NIT_HASHDIR}"/* "${TESTCERT_PATH_BASE}${TESTCERT_PATH_SEP}" # If there is a setup script there, source it to get variables @@ -1252,7 +1252,9 @@ if [ x"${DO_USE_NIT_TESTCERT_CACHE-}" = xyes ] ; then log_warn "FAILED check_NIT_certs with cached data, will generate anew. Removing:" find "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" -ls || true rm -rf "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" || true - mkdir -p "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" + # Do not re-create the (now empty) cache dir: it is populated + # below only if absent, after we generate the certificates + mkdir -p "${TESTCERT_PATH_BASE}" fi else log_info "Did not find a CI_CACHE_NIT_HASHDIR, will populate a new one after generating certificates as '${CI_CACHE_NIT_HASHDIR}'" @@ -1362,7 +1364,8 @@ case "${WITH_SSL_CLIENT}${WITH_SSL_SERVER}" in log_warn "FAILED check_NIT_certs with cached data, will generate anew" rm -rf "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" || true - mkdir -p "${TESTCERT_PATH_BASE}" "${CI_CACHE_NIT_HASHDIR}" + # Do not re-create the cache dir, see above + mkdir -p "${TESTCERT_PATH_BASE}" fi # if [ -n "${BASH_VERSION-}" ]; then