diff --git a/.Rprofile b/.Rprofile index f26ce91be..ad7f3fb4a 100644 --- a/.Rprofile +++ b/.Rprofile @@ -24,7 +24,7 @@ local({ # when content changes). Patching here intercepts every caller — devtools, # pkgbuild, RStudio auto-build — not just our manual wrapper script. local({ - .normalise_lf <- function(pkgdir) { + .normalize_lf <- function(pkgdir) { for (f in file.path(pkgdir, c("src/RcppExports.cpp", "R/RcppExports.R"))) { if (!file.exists(f)) next lines <- readLines(f, warn = FALSE) @@ -42,7 +42,7 @@ local({ if (isTRUE(attr(orig, ".lf_patched"))) return() patched <- function(pkgdir = ".", verbose = getOption("verbose")) { result <- orig(pkgdir = pkgdir, verbose = verbose) - .normalise_lf(pkgdir) + .normalize_lf(pkgdir) invisible(result) } attr(patched, ".lf_patched") <- TRUE diff --git a/.github/workflows/R-CMD-check.yml b/.github/workflows/R-CMD-check.yml index 219d521fd..42fe99ac3 100644 --- a/.github/workflows/R-CMD-check.yml +++ b/.github/workflows/R-CMD-check.yml @@ -113,26 +113,18 @@ jobs: # shinytest2 pulls chromote, whose declared system requirement is # `chromium`. On Linux that makes pak add the ppa:xtradeb/apps PPA, # re-run `apt-get update`, and install Chromium and its GTK/X11 - # dependency tree -- ~70 s on EVERY run, including cache hits, since - # the sysreqs pass is not cached. Nothing here needs it: shinytest2 - # is used only by inst/Parsimony/tests, which is .Rbuildignore'd and - # so absent from the tarball R CMD check sees, and those tests run in - # the dedicated `shiny` job on Windows. + # dependency tree -- ~70 s on every run, including cache hits, since + # the sysreqs pass is not cached. Nothing here needs it. + # # Coreset isn't on CRAN, so without this pak fetches and compiles it - # from source on every cache miss -- ~67 s of the ~174 s dependency - # step (measured on this exact leg, run 30697330564). This URL is a - # prebuilt binary for this exact platform/R combo (aarch64, - # release), published by ms609/packages' build-maxmin workflow. - # The URL has no version/commit in it -- it always points at - # whatever Coreset build is current -- so it never needs updating - # here when Coreset changes. + # from source on every cache miss -- ~67 s. The URL is a prebuilt + # binary for this exact platform/R combo (aarch64, release), + # published by ms609/packages' build-maxmin workflow. + # + # Remove ms609/TreeTools when >=2.4.1 reaches CRAN (exptd 2026-09-16) extra-packages: | shinytest2=?ignore url::https://ms609.github.io/packages/bin/linux/aarch64-release/Coreset_latest.tar.gz - # cache-version bumped to 2: the v1 caches were built while a - # project-level .Rprofile shadowed ~/.Rprofile, so every package - # in them was compiled from source. Retire them once. - cache-version: 2 - name: Check package uses: r-lib/actions/check-r-package@v2 @@ -252,10 +244,6 @@ jobs: # fails on Coreset by name; the branch only fires when pandoc is off # PATH, which is why only the non-Linux legs hit it. install-pandoc: true - # cache-version bumped to 2: the v1 caches were built while a - # project-level .Rprofile shadowed ~/.Rprofile, so every package - # in them was compiled from source. Retire them once. - cache-version: 2 - name: Set up R dependencies uses: r-lib/actions/setup-r-dependencies@v2 @@ -278,10 +266,6 @@ jobs: shinytest2=?ignore ${{ matrix.config.r == '4.1' && 'url::https://ms609.github.io/packages/bin/linux/x86_64-4.1/Coreset_latest.tar.gz' || 'url::https://ms609.github.io/packages/bin/linux/aarch64-devel/Coreset_latest.tar.gz' }} ${{ matrix.config.r == '4.1' && 'url::https://cran.r-project.org/src/contrib/Archive/highs/highs_1.12.0-3.tar.gz' || '' }} - # cache-version bumped to 2: the v1 caches were built while a - # project-level .Rprofile shadowed ~/.Rprofile, so every package - # in them was compiled from source. Retire them once. - cache-version: 2 - name: Check package uses: r-lib/actions/check-r-package@v2 @@ -354,10 +338,6 @@ jobs: # (auto-detect's `pak::pkg_deps(".")` solve cannot see extra-packages). # `false` here: this job runs shinytest2, not vignettes. install-pandoc: false - # cache-version bumped to 2: the v1 caches were built while a - # project-level .Rprofile shadowed ~/.Rprofile, so every package - # in them was compiled from source. Retire them once. - cache-version: 2 - name: Run EasyTrees shinytest2 suite run: | @@ -448,10 +428,6 @@ jobs: # pre-emptive, since whether a macOS runner ships pandoc on PATH is # exactly the condition that decides whether the bad branch fires. install-pandoc: true - # cache-version bumped to 2: the v1 caches were built while a - # project-level .Rprofile shadowed ~/.Rprofile, so every package - # in them was compiled from source. Retire them once. - cache-version: 2 - name: Check package uses: r-lib/actions/check-r-package@v2 diff --git a/AGENTS.md b/AGENTS.md index 93c7038dd..a00b9d604 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -224,7 +224,7 @@ whose comment names a worktree is being worked there — often by the human deve Closing **without** a fix (not-a-bug, superseded design, negative result) needs more: close as *not planned* with `deferred`/`wontfix` **and** a comment carrying the reasoning and **what would make it live again**. A stated reopening condition is what let a later round -recognise T-377 firing rather than re-hunt it. Long reasoning goes in `dev/benchmarks/*.md`, +recognize T-377 firing rather than re-hunt it. Long reasoning goes in `dev/benchmarks/*.md`, linked. **Blocked on GHA, Hamilton or review?** Comment what you await, its reference, and the @@ -261,7 +261,7 @@ Run these before committing whenever the trigger applies: |---------|---------| | Function signature or roxygen block changed | `Rscript -e "devtools::check_man()"` | | Documentation prose changed | `Rscript -e "spelling::spell_check_package()"` | -| C++ signature changed | `Rscript .claude/tools/compile-attrs.R` (normalises LF + verifies `ts_rcpp.cpp` / `TreeSearch-init.c` arg counts) | +| C++ signature changed | `Rscript .claude/tools/compile-attrs.R` (normalizes LF + verifies `ts_rcpp.cpp` / `TreeSearch-init.c` arg counts) | | Search behaviour changed (heuristics, scoring, stopping, pool) | Update `vignettes/search-algorithm.Rmd` | Full details: `.AGENTS/memory/r-package-conventions.md`. diff --git a/DESCRIPTION b/DESCRIPTION index feabaf90b..7f822d08c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -51,8 +51,9 @@ Imports: Rdpack (>= 0.7), stats, TreeDist (>= 2.6.3), - TreeTools (>= 2.3.0.9002), + TreeTools (>= 2.4.0.9002), utils, +Remotes: ms609/TreeTools, Suggests: cluster, Coreset, @@ -60,6 +61,7 @@ Suggests: highs, knitr, phangorn (>= 2.2.1), + pkgload, PlotTools, promises, protoclust, @@ -90,11 +92,12 @@ Config/Needs/website: pkgdown, remotes, shinylive, -Config/roxygen2/version: 8.0.0 +Config/roxygen2/version: 8.1.0 Config/testthat/edition: 3 Collate: 'AdditionTree.R' 'Bootstrap.R' + 'Bremer.R' 'CharacterHierarchy.R' 'ClusterStrings.R' 'Concordance.R' @@ -122,6 +125,7 @@ Collate: 'recode_hierarchy.R' 'SPR.R' 'ScoreSpectrum.R' + 'SuboptimalTrees.R' 'SuccessiveApproximations.R' 'TBR.R' 'TaxonInfluence.R' diff --git a/NAMESPACE b/NAMESPACE index a7039a09b..4d38d87c5 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -25,6 +25,7 @@ export(.NonDuplicateRoot) export(.UniqueExceptHits) export(AdditionTree) export(BootstrapTree) +export(Bremer) export(Carter1) export(CharacterHierarchy) export(CharacterLength) @@ -111,6 +112,7 @@ export(SingleCharData) export(SingleCharMorphy) export(StepInformation) export(StopUnlessBifurcating) +export(SuboptimalTrees) export(Suboptimality) export(SuccessiveApproximations) export(SuccessiveWeights) @@ -135,99 +137,119 @@ export(is.morphyPtr) export(mc_fitch_scores) importFrom(Rcpp,compileAttributes) importFrom(Rdpack,reprompt) -importFrom(TreeDist,ClusteringEntropy) -importFrom(TreeDist,ClusteringInfo) -importFrom(TreeDist,ClusteringInfoDistance) -importFrom(TreeDist,Entropy) -importFrom(TreeDist,MutualClusteringInfo) -importFrom(TreeDist,SharedPhylogeneticInfo) -importFrom(TreeDist,entropy_int) -importFrom(TreeTools,AddUnconstrained) -importFrom(TreeTools,CharacterInformation) -importFrom(TreeTools,CladeSizes) -importFrom(TreeTools,CladisticInfo) -importFrom(TreeTools,CompatibleSplits) -importFrom(TreeTools,Consensus) -importFrom(TreeTools,DescendantEdges) -importFrom(TreeTools,DescendantTips) -importFrom(TreeTools,DoubleFactorial) -importFrom(TreeTools,EdgeAncestry) -importFrom(TreeTools,KeepTip) -importFrom(TreeTools,LnUnrooted) -importFrom(TreeTools,Log2DoubleFactorial) -importFrom(TreeTools,Log2Unrooted) -importFrom(TreeTools,Log2UnrootedMult) -importFrom(TreeTools,LogDoubleFactorial) -importFrom(TreeTools,MakeTreeBinary) -importFrom(TreeTools,MatchStrings) -importFrom(TreeTools,MatrixToPhyDat) -importFrom(TreeTools,NRooted) -importFrom(TreeTools,NSplits) -importFrom(TreeTools,NTip) -importFrom(TreeTools,NUnrooted) -importFrom(TreeTools,NUnrootedMult) -importFrom(TreeTools,NexusTokens) -importFrom(TreeTools,PaintTree) -importFrom(TreeTools,PectinateTree) -importFrom(TreeTools,PhyDatToMatrix) -importFrom(TreeTools,PolarizeSplits) -importFrom(TreeTools,Postorder) -importFrom(TreeTools,PostorderOrder) -importFrom(TreeTools,Preorder) -importFrom(TreeTools,RandomTree) -importFrom(TreeTools,Renumber) -importFrom(TreeTools,RenumberEdges) -importFrom(TreeTools,RenumberTips) -importFrom(TreeTools,RenumberTree) -importFrom(TreeTools,RootNode) -importFrom(TreeTools,RootTree) -importFrom(TreeTools,SampleOne) -importFrom(TreeTools,SplitConflicts) -importFrom(TreeTools,SplitFrequency) -importFrom(TreeTools,StringToPhyDat) -importFrom(TreeTools,Subsplit) -importFrom(TreeTools,SupportColour) -importFrom(TreeTools,TipLabels) -importFrom(TreeTools,TreeIsRooted) -importFrom(TreeTools,as.Splits) -importFrom(TreeTools,as.multiPhylo) +importFrom(TreeDist, + ClusteringEntropy, + ClusteringInfo, + ClusteringInfoDistance, + Entropy, + MutualClusteringInfo, + SharedPhylogeneticInfo, + entropy_int +) +importFrom(TreeTools, + AddUnconstrained, + CharacterInformation, + CladeSizes, + CladisticInfo, + CompatibleSplits, + Consensus, + DescendantEdges, + DescendantTips, + DoubleFactorial, + EdgeAncestry, + KeepTip, + LnUnrooted, + Log2DoubleFactorial, + Log2Unrooted, + Log2UnrootedMult, + LogDoubleFactorial, + MakeTreeBinary, + MatchStrings, + MatrixToPhyDat, + NRooted, + NSplits, + NTip, + NUnrooted, + NUnrootedMult, + NexusTokens, + PaintTree, + PectinateTree, + PhyDatToMatrix, + PolarizeSplits, + Postorder, + PostorderOrder, + Preorder, + RandomTree, + Renumber, + RenumberEdges, + RenumberTips, + RenumberTree, + RootNode, + RootTree, + SampleOne, + SplitConflicts, + SplitFrequency, + StringToPhyDat, + Subsplit, + SupportColour, + TipLabels, + TreeIsRooted, + as.Splits, + as.multiPhylo +) importFrom(abind,abind) -importFrom(ape,consensus) -importFrom(ape,keep.tip) -importFrom(ape,nodelabels) -importFrom(ape,plot.phylo) -importFrom(ape,read.nexus) -importFrom(ape,root) -importFrom(ape,write.nexus) -importFrom(cli,cli_alert) -importFrom(cli,cli_alert_info) -importFrom(cli,cli_alert_success) -importFrom(cli,cli_alert_warning) -importFrom(cli,cli_h1) -importFrom(cli,cli_progress_bar) -importFrom(cli,cli_progress_update) -importFrom(colorspace,hex) -importFrom(colorspace,max_chroma) -importFrom(colorspace,polarLUV) -importFrom(fastmatch,"%fin%") -importFrom(fastmatch,fmatch) -importFrom(grDevices,col2rgb) -importFrom(grDevices,convertColor) -importFrom(grDevices,rgb) -importFrom(graphics,abline) -importFrom(graphics,image) -importFrom(graphics,mtext) -importFrom(graphics,par) -importFrom(stats,as.dist) -importFrom(stats,cophenetic) -importFrom(stats,cutree) -importFrom(stats,dnorm) -importFrom(stats,median) -importFrom(stats,runif) -importFrom(stats,sd) -importFrom(stats,setNames) -importFrom(stats,weighted.mean) -importFrom(utils,adist) -importFrom(utils,combn) -importFrom(utils,head) +importFrom(ape, + consensus, + keep.tip, + nodelabels, + plot.phylo, + read.nexus, + root, + write.nexus +) +importFrom(cli, + cli_alert, + cli_alert_info, + cli_alert_success, + cli_alert_warning, + cli_h1, + cli_progress_bar, + cli_progress_update +) +importFrom(colorspace, + hex, + max_chroma, + polarLUV +) +importFrom(fastmatch, + "%fin%", + fmatch +) +importFrom(grDevices, + col2rgb, + convertColor, + rgb +) +importFrom(graphics, + abline, + image, + mtext, + par +) +importFrom(stats, + as.dist, + cophenetic, + cutree, + dnorm, + median, + runif, + sd, + setNames, + weighted.mean +) +importFrom(utils, + adist, + combn, + head +) useDynLib(TreeSearch, .registration = TRUE) diff --git a/NEWS.md b/NEWS.md index f69a23ba1..1a7a2bb91 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,14 @@ # To integrate into 2.0.0 notes +- New `Bremer()` calculates Bremer (decay) support for each clade of a reference + tree. + +- New `SuboptimalTrees()` returns every tree the search retained within a given + number of steps of the optimum, each annotated with its parsimony score, for + landscape analysis. `MaximizeParsimony(collapse = FALSE)` now surfaces the + per-tree scores of the retained pool via a `scores` attribute (and a `score` + attribute on each tree), so `Suboptimality()` works on the result directly. + - `QuartetConcordance()` gains `unit`, selecting the currency in which quartets are counted. `unit = "nrqs"` (the new default) counts only non-redundant quartet statements: of the quartets resolved by a split of sizes @@ -157,7 +166,7 @@ `attr(result, "score")` did not match `TreeLength()` of the very tree returned (measured: 178 reported against 183 returned on a 36-taxon matrix), and re-rooting a returned tree changed its length again. Both boundaries now - canonicalise on the first taxon of `dataset`, so one topology has one length + canonicalize on the first taxon of `dataset`, so one topology has one length and the two agree by construction. **X-transformation scores may therefore differ slightly from previous @@ -166,7 +175,7 @@ upper bound on the rooting-free minimum, exceeding it by at most the total number of secondary characters across hierarchy blocks (attained exactly by 87--98% of rootings in simulation). This changes reporting only -- what the - search optimises is untouched. + search optimizes is untouched. `MaximizeParsimony()` now also warns when the trees it returns do not share a length at that common rooting, which can happen because pool membership is @@ -206,7 +215,7 @@ scores 7 and 5. Unlike the x-transformation change above, this one alters what the search - optimises: the criterion is now a function of the unrooted tree, so + optimizes: the criterion is now a function of the unrooted tree, so `MaximizeParsimony()`'s reported score matches `TreeLength()` of the trees it returns without any re-scoring, and every tree in a returned set shares that length. A secondary character is treated as unconstrained only at taxa whose @@ -268,7 +277,7 @@ The rungs are the former presets — `sprint`, `default`, `thorough`, and `large` (which was only ever `thorough` with `maxReplicates = 500`) — so the - ladder generalises an axis the package already had. Beyond `large`, each + ladder generalizes an axis the package already had. Beyond `large`, each further notch doubles BOTH the replicate budget (1000, 2000, 4000 ...) and the hit target, so one notch always means roughly twice the work whichever bound a dataset is under. There is no policy ceiling: extra replicates cost @@ -317,7 +326,7 @@ returned or searched from. The topology and score are unchanged -- the rooting of an addition tree is an arbitrary artefact of the order taxa were added in, as `?AdditionTree` notes -- but it is the rooting in which the rest of the - constraint machinery can recognise every split the tree displays. Without it, + constraint machinery can recognize every split the tree displays. Without it, a constrained search took several times longer to reach the same score. - Fix `TreeLength()` and `LengthAdded()` errors when scoring, under profile @@ -577,7 +586,7 @@ - New functions `LeastSquaresTree()` and `LeastSquaresFit()` search for, and fit branch lengths to, the tree that best matches a target distance matrix - under a least-squares criterion, reusing the optimised C++ rearrangement + under a least-squares criterion, reusing the optimized C++ rearrangement kernel (NNI + SPR). Ordinary (`method = "ols"`) and non-negative (`method = "nnls"`) least squares are supported, with optional Fitch-Margoliash (`weight = "fm"`) or custom weighting. @@ -780,7 +789,7 @@ faster; inapplicable character handling (Brazeau _et al._ 2019) is built in. same and neighbouring score plateaus, up to `poolMaxSize`. - `LeastSquaresTree()` and `LeastSquaresFit()` search for, and fit branch lengths to, the tree that best matches a target distance matrix - under a least-squares criterion, reusing the optimised C++ rearrangement + under a least-squares criterion, reusing the optimized C++ rearrangement kernel (NNI + SPR). Ordinary (`method = "ols"`) and non-negative (`method = "nnls"`) least squares are supported, with optional Fitch-Margoliash (`weight = "fm"`) or custom weighting. This provides the @@ -836,7 +845,7 @@ faster; inapplicable character handling (Brazeau _et al._ 2019) is built in. converged searches at no score cost. - `pruneReinsertCycles`, `pruneReinsertDrop`, `pruneReinsertSelection` — taxon pruning-reinsertion perturbation: drop a fraction of leaves, let - the backbone re-optimise with TBR, then reinsert taxa greedily. + the backbone re-optimize with TBR, then reinsert taxa greedily. Complementary to the ratchet (which perturbs character weights). - `nniPerturbCycles`, `nniPerturbFraction` — stochastic NNI-perturbation: randomly apply NNI swaps to a fraction of internal branches and diff --git a/R/AdditionTree.R b/R/AdditionTree.R index ee1ce13cc..ddd06a6d0 100644 --- a/R/AdditionTree.R +++ b/R/AdditionTree.R @@ -40,7 +40,7 @@ #' parsimony scores are unaffected by rooting, so root the result yourself with #' [`TreeTools::RootTree()`](https://ms609.github.io/TreeTools/reference/RootTree) #' if the position of the root matters to you. With fewer than four -#' taxa there is nothing to optimise, and a pectinate tree of the dataset's taxa +#' taxa there is nothing to optimize, and a pectinate tree of the dataset's taxa #' is returned without consulting `sequence` or `constraint`. #' @importFrom TreeTools PectinateTree Renumber #' @family tree generation functions diff --git a/R/Bremer.R b/R/Bremer.R new file mode 100644 index 000000000..fd6de90e4 --- /dev/null +++ b/R/Bremer.R @@ -0,0 +1,457 @@ +#' Bremer support (decay index) +#' +#' `Bremer()` calculates the Bremer support (decay index) +#' \insertCite{Bremer1988,Bremer1994}{TreeSearch} of each clade in a reference +#' tree: the number of extra steps required before the clade is no longer +#' present in an optimal tree. Formally, for a clade _C_, +#' \deqn{\textrm{Bremer}(C) = L(\neg C) - L^\star}{Bremer(C) = L(not C) - L\*} +#' where \eqn{L(\neg C)}{L(not C)} is the score of the shortest tree that does +#' not contain _C_ and \eqn{L^\star}{L\*} is the score of the most-parsimonious +#' tree. Larger values indicate better-supported clades. +#' +#' Two engines are available: +#' +#' \describe{ +#' \item{`method = "constraint"` (default, rigorous)}{For each clade, runs +#' a tree search under the constraint that that clade is absent, taking +#' the score of the shortest resulting tree.} +#' \item{`method = "pool"` (fast, loose)}{Runs a single search and collects +#' a pool of suboptimal trees with [`SuboptimalTrees()`], then for each +#' clade, take the best score found in a tree that lacks that clade. +#' Provides an upper bound on the support: better trees that lack the clade +#' may exist outside the sampled trees. Useful for exploration, but not +#' recommended for publication values.} +#' } +#' +#' Ensure that search options (`concavity`, `inapplicable`, ...) match those +#' used to find `tree`. +#' +#' @inheritParams JackLabels +#' @param tree A tree of class `phylo` whose clades are to be evaluated. If a +#' `multiPhylo` object is provided, its strict consensus will be evaluated. +#' @param method Character: `"constraint"` for negative-constraint searches, +#' or `"pool"` for approximation via the suboptimal pool. +#' @param maxBremer Numeric specifying the largest decay value to resolve when +#' `method = "pool"`. Ignored under `method = "constraint"`. +#' @param optimalScore Optional numeric: the score of the optimal tree, +#' \eqn{L^*}{L*}. If `NULL`, it is read from `attr(tree, "score")`, or computed +#' by search. +#' @inheritParams MaximizeParsimony +#' @param concavity,extended_iw,xpiwe_r,xpiwe_max_f,hierarchy,hsj_alpha,inapplicable Tree +#' scoring parameters passed to `MaximizeParsimony()`. +#' These will usually match the parameters used to obtain `tree`. +#' @param \dots Further arguments passed to [`MaximizeParsimony()`] / +#' [`SuboptimalTrees()`], e.g. `maxReplicates`, `maxSeconds`, `effort`, +#' `nThreads`, `verbosity`. +#' +#' @return `Bremer()` returns a numeric vector (or, if `format = "character"`, a +#' `phylo$node.label`-shaped character vector) giving the Bremer support of each +#' resolved clade in `tree`, named by node number (the row names of +#' [`TreeTools::as.Splits()`]). +#' Annotate a plot with [`TreeTools::LabelSplits()`], or assign to +#' `tree$node.label`. +#' Under `method = "pool"` the numeric result carries a logical `censored` +#' attribute marking clades whose support exceeds `maxBremer`. +#' +#' @examples +#' data("inapplicable.phyData", package = "TreeSearch") +#' dataset <- inapplicable.phyData[["Vinther2008"]] +#' \donttest{ +#' # `set.seed()` with `nThreads = 1` makes the heuristic search reproducible +#' set.seed(0) +#' trees <- MaximizeParsimony(dataset, maxReplicates = 8, nThreads = 1, +#' verbosity = 0) +#' +#' # Fast, approximate decay index for the strict consensus topology, +#' # from the suboptimal-tree pool. +#' decay <- Bremer(trees, dataset, method = "pool", maxBremer = 4, +#' maxSeconds = 20, nThreads = 1, verbosity = 1) +#' decay +#' +#' # Annotate the reference tree +#' reference <- TreeTools::Consensus(trees, p = 1) +#' plot(reference) +#' TreeTools::LabelSplits(reference, decay, adj = c(0, -0.5)) +#' +#' # `method = "constraint"` is rigorous but may be slower. +#' # Bounding each search with `maxSeconds` increases both speed, and the chance +#' # of over-estimating support through missing a better score. +#' slower <- Bremer(trees, dataset, maxReplicates = 8, maxSeconds = 2, +#' verbosity = 0) +#' TreeTools::LabelSplits(reference, slower, bg = "gold", adj = c(0, 1.5)) +#' } +#' @references \insertAllCited{} +#' @template MRS +#' @seealso +#' Other clade support measures: [`JackLabels()`], [`SiteConcordance`]; +#' [`SuboptimalTrees()`] collects the pool used by `method = "pool"`. +#' @family split support functions +#' @importFrom TreeTools as.Splits NTip SplitFrequency TipLabels +#' @export +Bremer <- function(tree, dataset, + method = c("constraint", "pool"), + maxBremer = Inf, optimalScore = NULL, + format = "numeric", + concavity = Inf, extended_iw = TRUE, xpiwe_r = 0.5, + xpiwe_max_f = 5, hierarchy = NULL, inapplicable = "bgs", + hsj_alpha = 1.0, + ...) { + method <- match.arg(method) + + # `optimalScore = NULL` is "not supplied" sentinel. + # A non-NULL value must be a single finite number. + if (!is.null(optimalScore) && + (length(optimalScore) != 1L || !is.numeric(optimalScore) || + !is.finite(optimalScore))) { + stop("`optimalScore` must be a single finite number, or NULL.") + } + + scoringArgs <- list(concavity = concavity, extended_iw = extended_iw, + xpiwe_r = xpiwe_r, xpiwe_max_f = xpiwe_max_f, + hierarchy = hierarchy, inapplicable = inapplicable, + hsj_alpha = hsj_alpha) + + # --- Resolve the reference tree, its clades, and L* --- + ref <- .BremerReference(tree, dataset, optimalScore) + + if (length(ref$splitNames) == 0L) { + warning("Reference tree has no resolved internal clades; ", + "nothing to calculate.") + return(.BremerFormat(setNames(numeric(0), character(0)), + logical(0), ref$reference, format)) + } + + # Scoring-units guard: L* must be measured with the same scoring arguments as + # the converse searches / pool re-scores, or the decay mixes two optimality + # criteria (e.g. an equal-weights length minus an implied-weights optimum). + .BremerCheckScoring(tree, dataset, scoringArgs, optimalScore) + + res <- if (method == "pool") { + .BremerPool(ref, dataset, scoringArgs, maxBremer, list(...)) + } else { + .BremerConstraint(ref, dataset, scoringArgs, maxBremer, list(...)) + } + + .BremerFormat(res$bremer, res$censored, ref$reference, format) +} + +# Resolve reference tree + clades (Splits) + optimal score L*. +#' @importFrom TreeTools as.Splits Consensus NTip +.BremerReference <- function(tree, dataset, optimalScore) { + Lstar <- optimalScore + if (inherits(tree, "multiPhylo")) { + if (is.null(Lstar)) { + s <- attr(tree, "score") + if (!is.null(s) && is.finite(s)) Lstar <- s + } + reference <- if (length(tree) == 1L) tree[[1L]] else Consensus(tree, p = 1) + } else if (inherits(tree, "phylo")) { + reference <- tree + } else { + stop("`tree` must be a `phylo` or `multiPhylo` object.") + } + + splits <- as.Splits(reference, tipLabels = names(dataset)) + splitNames <- rownames(as.matrix(splits)) + if (is.null(splitNames)) splitNames <- character(0) + + list(reference = reference, splits = splits, splitNames = splitNames, + Lstar = Lstar) +} + +# Sanity-check a SUPPLIED optimal score L* against the reference's length under +# the scoring arguments now in effect. We trust the user to keep the scoring mode +# consistent; in case of disagreement we proceed as instructed, with a warning. +#' @importFrom TreeTools MakeTreeBinary +.BremerCheckScoring <- function(tree, dataset, scoringArgs, optimalScore) { + # Exact check: a MaximizeParsimony() result records the scoring conditions it + # is optimal under (attr "scoring"). When present, compare that signature + # DIRECTLY to the arguments now in effect. + recorded <- attr(tree, "scoring", exact = TRUE) + if (!is.null(recorded)) { + current <- do.call(.ScoringSignature, scoringArgs) + if (!.ScoringSignatureMatch(recorded, current)) { + warning("The reference trees were found under ", .DescribeScoring(recorded), + " but Bremer() is scoring with ", .DescribeScoring(current), ".") + return(invisible(NULL)) + } + # A matching signature validates the scoring MODE, but not the VALUE. + } + + # Compare the supplied optimal score to the reference's length under the + # current scoring arguments. + suppliedLstar <- if (!is.null(optimalScore)) { + optimalScore + } else if (inherits(tree, "multiPhylo")) { + s <- attr(tree, "score") + if (!is.null(s) && is.finite(s)) s else NULL + } else { + NULL + } + if (is.null(suppliedLstar)) { + return(invisible(NULL)) + } + + # Length of the reference under the CURRENT scoring arguments. + # TreeLength requires binary trees, so resolve zero-length polytomies first; + # take a minimum over arbitrary resolutions to find best. + # TODO verify that there's no non-binary configuration that could result in + # a higher score when resolved arbitrarily, and thus throw a warning. + resolved <- MakeTreeBinary(tree) + refLen <- min(suppressWarnings( + do.call(TreeLength, c(list(tree = resolved, dataset = dataset), scoringArgs)))) + if (!is.finite(refLen)) { + return(invisible(NULL)) + } + + # A material difference signals a probable scoring-mode mismatch. + # Warn and proceed. + tol <- 1e-6 * max(1, abs(refLen), abs(suppliedLstar)) + if (abs(suppliedLstar - refLen) > tol) { + warning("`optimalScore` (", signif(suppliedLstar, 7), + ") differs from ", signif(refLen, 7), ", the score of `tree` ", + "under the supplied arguments (`concavity`, `inapplicable`, ...).") + } + invisible(NULL) +} + +# Format a node-indexed numeric result, mirroring JackLabels(). +#' @importFrom TreeTools NTip +.BremerFormat <- function(values, censored, reference, format) { + numericFmt <- c("numeric", "number", "double") + characterFmt <- c("character", "text") + returnMode <- c(rep("numeric", length(numericFmt)), + rep("character", length(characterFmt)))[ + pmatch(tolower(format), c(numericFmt, characterFmt))] + if (is.na(returnMode)) returnMode <- "numeric" + + switch(returnMode, + "character" = { + ret <- character(reference[["Nnode"]]) + if (length(values)) { + idx <- as.integer(names(values)) - NTip(reference) + ret[idx] <- as.character(values) + # Inf renders as the literal "Inf"; preserve the censoring flag as an + # attribute so pool-method callers can still distinguish "> maxBremer" + # (censored) from a genuine value. Attach it whenever there ARE entries + # (not only when something is censored), so the character format carries + # the same fixed-length `censored` attribute the numeric format does -- + # otherwise downstream code assuming its presence breaks on the character + # path whenever no clade happens to be censored. + if (length(censored)) { + censVec <- logical(reference[["Nnode"]]) + censVec[idx] <- as.logical(censored) + attr(ret, "censored") <- censVec + } + } + ret + }, + { + if (length(censored)) attr(values, "censored") <- censored + values + }) +} + +# Approximate Bremer from a pool of suboptimal trees. +# Returns list(bremer = named numeric, censored = logical). +#' @importFrom TreeTools SplitFrequency +.BremerPool <- function(ref, dataset, scoringArgs, maxBremer, dots) { + poolK <- if (is.finite(maxBremer)) maxBremer else 10 + if (!is.finite(maxBremer)) { + message("method = \"pool\": using maxBremer = ", poolK, + " for the pool depth (set `maxBremer` to control it).") + } + + pool <- do.call(SuboptimalTrees, + c(list(dataset = dataset, maxSuboptimal = poolK), + scoringArgs, dots)) + poolScores <- attr(pool, "scores") + if (is.null(poolScores)) { + poolScores <- do.call(TreeLength, + c(list(tree = pool, dataset = dataset), scoringArgs)) + } + poolBest <- min(poolScores) + + Lstar <- ref$Lstar + tol <- 1e-8 + if (is.null(Lstar)) { + Lstar <- poolBest + } else if (poolBest < Lstar - tol) { + warning("Pool search found a tree (length ", signif(poolBest, 7), + ") shorter than the supplied optimalScore (", signif(Lstar, 7), + "); adopting the shorter length as L*.") + Lstar <- poolBest + } + + # Per-clade displaying matrix: refFreq gives the node names; then test each + # pool tree individually (SplitFrequency of a length-1 forest is 0/1). + refFreq <- SplitFrequency(ref$reference, pool) + nSplits <- length(refFreq) + displayMat <- vapply(seq_along(pool), function(j) { + SplitFrequency(ref$reference, pool[j]) + }, double(nSplits)) + dim(displayMat) <- c(nSplits, length(pool)) + + bremer <- numeric(nSplits) + censored <- logical(nSplits) + for (i in seq_len(nSplits)) { + nonDisplaying <- displayMat[i, ] < 0.5 + if (any(nonDisplaying)) { + bremer[i] <- min(poolScores[nonDisplaying]) - Lstar + } else { + bremer[i] <- Inf + censored[i] <- TRUE + } + } + names(bremer) <- names(refFreq) + names(censored) <- names(refFreq) + list(bremer = bremer, censored = censored) +} + +# Convert one converse-search result `out` = list(score, tree) to a decay- +# eligible score, or NA. Two ways to NA: a non-finite/negative score is the +# engine's "no tree lacking the clade found" sentinel (empty pool -> budget +# shortfall); a returned tree that STILL displays the clade is an engine +# regression the C++ guard + pool backstop should preclude -- surface it as NA + +# a warning rather than a silently deflated decay (the worst outcome for a +# published statistic). +#' @importFrom TreeTools SplitFrequency +.BremerProcessResult <- function(out, refReference, splitName) { + s <- out$score + if (!is.finite(s) || s < 0) { + return(NA_real_) + } + if (!is.null(out$tree)) { + disp <- SplitFrequency(refReference, + structure(list(out$tree), class = "multiPhylo")) + # Index by the reference node number (robust to split ordering). + if (isTRUE(unname(disp[splitName]) >= 0.5)) { + warning("The converse search for clade ", splitName, + " returned a tree that still displays it; reported as NA.") + return(NA_real_) + } + } + s +} + +# Rigorous Bremer by converse-constraint search: for each clade, the shortest +# tree forced to LACK it. Uses the negative-constraint engine wired into +# MaximizeParsimony() via the internal `.negativeConstraint` argument. +# +# The converse search keeps the full search machinery -- Wagner starts, TBR, +# ratchet, sectorial search and NNI perturbation -- which all re-optimize +# through the negative-constraint-guarded TBR and so stay in the space of trees +# lacking the clade; the tree pool additionally rejects any tree that displays +# it. Only the phases that accept score-worsening moves through an unguarded +# path (drift, in-sector drift, simulated annealing) are disabled -- they would +# otherwise wander onto the clade and, being worse-accepting, report it as +# unsupported. Runs serially: the pool guard is on the serial search path. +.BremerConstraint <- function(ref, dataset, scoringArgs, maxBremer, dots, + .runConverse = NULL) { + disabled <- list( + driftCycles = 0L, sectorGoDrift = 0L, sectorDriftCycles = 0L, + annealCycles = 0L + ) + # The worse-accepting phases (drift, in-sector drift, annealing) reach trees + # through a path the negative-constraint hill-climb does not guard, so they + # are ALWAYS disabled here. A user value would only reintroduce the + # unsoundness the converse search exists to avoid, so warn rather than honour + # it (previously `modifyList` let the user win, silently emptying the pool). + reenabled <- intersect(names(dots), names(disabled)) + if (length(reenabled)) { + warning("Ignoring ", paste(reenabled, collapse = ", "), + " in the converse-constraint search: drift, in-sector drift and ", + "annealing are always disabled here for soundness.") + } + converseFixed <- disabled + # `collapse` and `.negativeConstraint` are managed by Bremer(): the converse + # search fixes both, so a user copy arriving through `...` would collide with + # the explicit argument in do.call ("matched by multiple actual arguments"). + # Strip them with a warning, mirroring SuboptimalTrees()'s handling of its own + # managed arguments. `nThreads` is forced to 1 (the negative-constraint pool + # guard is on the serial search path only); everything else flows through. + managed <- intersect(names(dots), c("collapse", ".negativeConstraint")) + if (length(managed)) { + warning("Ignoring `", paste(managed, collapse = "`, `"), + "`: Bremer() manages ", if (length(managed) > 1L) "these" else "this", + " to run the converse-constraint search.") + } + passthrough <- dots[setdiff(names(dots), + c(names(disabled), "nThreads", "collapse", + ".negativeConstraint"))] + + # L* (the unconstrained optimum) is found by a full-strength search -- the + # worse-accepting phases are safe here because there is no forbidden clade. + Lstar <- ref$Lstar + if (is.null(Lstar)) { + res0 <- do.call(MaximizeParsimony, + c(list(dataset = dataset, collapse = TRUE, nThreads = 1L), + scoringArgs, passthrough)) + Lstar <- attr(res0, "score") + } + + # One converse-constraint search: the shortest tree forced to lack `negSplit`. + # Returns the score AND the tree, so the caller can verify the clade really is + # absent (defence in depth around the C++ move-guard + pool backstop). + runConverse <- function(negSplit) { + args <- c(list(dataset = dataset, collapse = TRUE, nThreads = 1L, + .negativeConstraint = negSplit), + scoringArgs, converseFixed, passthrough) + res <- do.call(MaximizeParsimony, args) + # MaximizeParsimony() may return a single `phylo` or a `multiPhylo`; take the + # first tree either way (never `res[[1L]]` on a bare phylo, which would grab + # its `edge` matrix). + tree <- if (inherits(res, "phylo")) { + res + } else if (inherits(res, "multiPhylo") && length(res) >= 1L) { + res[[1L]] + } else { + NULL + } + list(score = attr(res, "score"), tree = tree) + } + # Test seam: an injected search function (same `(negSplit) -> list(score, tree)` + # contract) bypasses the real engine, so the NA / displays-clade handling and + # the aggregate NA warning can be exercised deterministically. + if (!is.null(.runConverse)) { + runConverse <- .runConverse + } + + splitNames <- ref$splitNames + + # Per-clade work unit. + processConverse <- function(i) { + .BremerProcessResult(runConverse(ref$splits[[i]]), ref$reference, + splitNames[i]) + } + + # Per-clade searches are independent, but each converse search must itself + # run with `nThreads = 1` (the negative-constraint pool guard is on the + # serial search path only), so there is currently no parallelism to fan + # them out over. + scores <- vapply(seq_along(splitNames), processConverse, double(1)) + + if (anyNA(scores)) { + # Two causes map to NA: no clade-free tree was found (a budget shortfall), or + # a returned tree unexpectedly displayed the clade (an engine regression the + # BR-6 check caught). The per-clade warning that distinguishes them + # already fired above, so this aggregate keeps the summary honest about + # both. + warning(sum(is.na(scores)), " converse-constraint search(es) returned NA: ", + "either no tree lacking the clade was found (increase `maxReplicates`)", + ", or a returned tree still displayed the clade (an engine bug worth ", + "reporting).") + } + + trueBest <- suppressWarnings(min(c(Lstar, scores), na.rm = TRUE)) + if (is.finite(trueBest) && trueBest < Lstar - 1e-8) { + warning("A converse-constraint search found a tree (length ", + signif(trueBest, 7), ") shorter than the optimal score (", + signif(Lstar, 7), "); the original search was suboptimal. ", + "Adopting the shorter length as L*.") + Lstar <- trueBest + } + + bremer <- setNames(scores - Lstar, splitNames) + list(bremer = bremer, censored = logical(length(bremer))) +} diff --git a/R/LeastSquares.R b/R/LeastSquares.R index ceb0d034e..7a17ad1d1 100644 --- a/R/LeastSquares.R +++ b/R/LeastSquares.R @@ -1,6 +1,6 @@ # Least-squares distance tree fitting and search. # -# A sibling to MaximizeParsimony() that optimises a least-squares fit to a +# A sibling to MaximizeParsimony() that optimizes a least-squares fit to a # target distance matrix instead of a parsimony score, using the same fast # C++ rearrangement kernel (NNI + SPR). Built for Lapointe & Cucumel's (1997) # average consensus procedure, where the averaged patristic distance matrix is @@ -96,7 +96,7 @@ #' Fit branch lengths to a distance matrix on a fixed topology #' -#' Fits branch lengths on a fixed tree topology that minimise the (optionally +#' Fits branch lengths on a fixed tree topology that minimize the (optionally #' weighted) least-squares discrepancy between the tree's patristic distances #' and a target distance matrix, using the package's C++ kernel. This is the #' fixed-topology counterpart of [`LeastSquaresTree()`], and the direct analogue @@ -155,7 +155,7 @@ LeastSquaresFit <- function(tree, dist, method = c("nnls", "ols"), #' Searches tree topologies for the one whose patristic distances best fit a #' target distance matrix under a least-squares criterion, fitting branch #' lengths on each candidate and minimising the residual sum of squares. The -#' heuristic uses the package's optimised C++ kernel, alternating \acronym{NNI} +#' heuristic uses the package's optimized C++ kernel, alternating \acronym{NNI} #' and \acronym{SPR} rearrangements, exactly as the parsimony search does — but #' driven by the least-squares score rather than tree length. #' diff --git a/R/MaximizeParsimony.R b/R/MaximizeParsimony.R index d2f39209d..5c22ad4b7 100644 --- a/R/MaximizeParsimony.R +++ b/R/MaximizeParsimony.R @@ -89,6 +89,38 @@ } # Internal helper: prepare constraint data for C++ engine. +# Build the forbidden-clade (negative / converse constraint) matrix for the +# C++ engine. A tree that displays any of these bipartitions will be rejected. +# @param negConstraint A `Splits` object, or anything `as.Splits()` accepts +# (e.g. a `phylo`), giving the clade(s) to forbid. +# @param dataset A phyDat whose names define the tip ordering. +# @return An integer n_neg x n_tips membership matrix (columns in dataset tip +# order), or NULL if there is nothing to forbid. +# @keywords internal +#' @importFrom TreeTools as.Splits +.PrepareNegativeConstraint <- function(negConstraint, dataset) { + if (is.null(negConstraint)) return(NULL) + tipLabels <- names(dataset) + if (!inherits(negConstraint, "Splits")) { + negConstraint <- as.Splits(negConstraint, tipLabels = tipLabels) + } + if (length(negConstraint) == 0L) return(NULL) + + # as.logical() yields an n_splits x n_tips logical membership matrix. + membership <- as.logical(negConstraint) + if (is.null(dim(membership))) { + membership <- matrix(membership, nrow = 1L) + } + splitLabels <- attr(negConstraint, "tip.label") + colOrder <- match(tipLabels, splitLabels) + if (anyNA(colOrder)) { + stop("Forbidden-clade taxa do not match the dataset taxa.") + } + membership <- membership[, colOrder, drop = FALSE] + storage.mode(membership) <- "integer" + membership +} + # Returns a named list of constraint arguments (empty list if no constraint). # @param constraint A phyDat, phylo, or NULL. # @param dataset A phyDat whose names define the tip ordering. @@ -500,7 +532,7 @@ # Rungs 1-3 are the provisioning presets. Rung 4 is `thorough`'s provisioning # with a raised replicate cap -- which is exactly what `large` already was # (`presets$large <- presets$thorough`, plus `maxReplicates = 500`). So the -# ladder generalises an axis the package was already using; it does not invent +# ladder generalizes an axis the package was already using; it does not invent # one. Above rung 4 only the BUDGET climbs, because provisioning saturates at # `thorough`: there is nothing further to provision. # @@ -796,7 +828,7 @@ #' lose it -- so a tree's length depends on where its root sits, whereas #' parsimony under the other methods does not. Lengths are therefore #' reported at a canonical rooting, on the first taxon of `dataset`, which -#' is the rooting the returned trees carry; `TreeLength()` canonicalises +#' is the rooting the returned trees carry; `TreeLength()` canonicalizes #' identically, so it reproduces the reported score and one topology has #' one length. That value is an upper bound on the rooting-free minimum, #' exceeding it by at most the total number of secondary characters across @@ -1051,12 +1083,25 @@ #' collapse, so an enforced-but-unsupported clade (a zero-length branch) stays #' visible (a constraint encodes external evidence the matrix does not #' capture); unsupported non-constraint branches still collapse. +#' @param .negativeConstraint Internal. A splits object (or `NULL`) naming +#' clades that returned trees must *not* display; used by [`Bremer()`] to run +#' converse-constraint searches. Not intended for direct use. #' @param ... Backward compatibility. #' #' @return A `multiPhylo` object containing the best tree(s) found, with #' attributes: #' \describe{ #' \item{`score`}{Best parsimony score.} +#' \item{`scoring`}{A list recording the scoring conditions the score is +#' optimal under (`concavity`, `inapplicable`, ...), so a saved score +#' remains interpretable. [`Bremer()`][Bremer] checks it against its own +#' scoring arguments and warns on a mismatch.} +#' \item{`scores`}{Present only when `collapse = FALSE`: a numeric vector of +#' the parsimony score of each returned tree, aligned with the returned +#' `multiPhylo` (the same values are attached as a `score` attribute on +#' each individual tree). With `poolSuboptimal > 0` this exposes the +#' retained suboptimal pool for landscape analysis (see +#' [`SuboptimalTrees()`], [`Suboptimality()`]).} #' \item{`replicates`}{Number of replicates completed.} #' \item{`hits_to_best`}{Number of independent discoveries of the best #' score.} @@ -1133,6 +1178,7 @@ MaximizeParsimony <- function( inapplicable = "bgs", hsj_alpha = 1.0, constraint, + .negativeConstraint = NULL, effort = 0L, maxReplicates = 96L, targetHits = NULL, @@ -1166,6 +1212,16 @@ MaximizeParsimony <- function( stop("`maxReplicates` must be a single integer of at least 1.") } + # Capture the scoring conditions the result is optimal under, BEFORE any + # normalization (e.g. inapplicable aliasing, profile -> Inf). Attached to the + # returned trees so a saved optimal score is interpretable -- a bare score is + # meaningless without knowing the criterion it was found under, and Bremer() + # checks this signature against its own scoring arguments. + scoringSignature <- .ScoringSignature( + concavity = concavity, extended_iw = extended_iw, xpiwe_r = xpiwe_r, + xpiwe_max_f = xpiwe_max_f, hierarchy = hierarchy, + inapplicable = inapplicable, hsj_alpha = hsj_alpha) + # --- Set targetHits default if not provided --- # `defaultHits` is retained even when the user supplies `targetHits`: the # implied-weights ratchet depth below scales with the user's *escalation* @@ -1304,7 +1360,7 @@ MaximizeParsimony <- function( # all plateau above the optimum that a deeper ratchet reaches. A 36-matrix # grid over # `ratchetCycles` in {6, 12, 20, 48, 96} (implied weights, k = 10) found - # expected wall-clock-to-optimum minimised at 48: on the 4 cycle-sensitive + # expected wall-clock-to-optimum minimized at 48: on the 4 cycle-sensitive # matrices the mean fell 1435 s -> 709 s, while the 32 others paid a median # +0.2 s with reach unchanged. The curve is flat from ~20 to ~96 and rises # steeply below 20, so 48 is a broad optimum rather than a knife-edge -- @@ -1637,7 +1693,7 @@ MaximizeParsimony <- function( # Derived from T-069 benchmarks: at 225 taxa / 748 chars a single rep takes # ~40s and at least ~34 reps are needed to fill the tree pool reliably. if (userSetReps && nTip >= 30L && verbosity > 0L) { - # `weight` here is the .ScaleWeight()-integerised value used by the C++ + # `weight` here is the .ScaleWeight()-integerized value used by the C++ # engine (up to ~1260x the original for fractional weights); the # recommendation formula is about the number of characters in the # dataset, so it must read `at$weight` (pre-scaling) rather than `weight`. @@ -1664,6 +1720,27 @@ MaximizeParsimony <- function( cli_alert_info("Constraint: {nrow(consArgs$consSplitMatrix)} split{?s}") } + # Negative (converse) constraints: forbidden clades. Internal argument used + # by Bremer() -- the returned trees must NOT display any supplied split. + if (!is.null(.negativeConstraint)) { + # The negative-constraint soundness backstop (TreePool::set_forbidden) is + # wired only into the serial search path; the parallel pool has no such + # guard, so a forbidden clade could slip into the result under nThreads > 1. + # Force serial (Bremer() already does; this protects any other caller). + if (!identical(as.integer(nThreads), 1L)) { + warning("Negative (converse) constraints are supported only in serial ", + "search; forcing `nThreads = 1`.") + nThreads <- 1L + } + negMatrix <- .PrepareNegativeConstraint(.negativeConstraint, dataset) + if (!is.null(negMatrix)) { + consArgs[["consNegSplitMatrix"]] <- negMatrix + if (verbosity > 0L) { + cli_alert_info("Forbidding {nrow(negMatrix)} clade{?s}") + } + } + } + # A start tree that breaks the constraint is not something the search can # rearrange its way out of -- every constrained move from it is rejected, so # it would freeze the replicate on a tree scoring better than any legal one. @@ -1787,6 +1864,11 @@ MaximizeParsimony <- function( resultTrees <- list() } nTopologies <- result$n_topologies + # Per-tree scores aligned with result$trees; surfaced only in the collapse = + # FALSE path (the collapse = TRUE path keeps only best-score trees, so their + # scores are all result$best_score and the collapsed/deduped list no longer + # aligns with result$scores). NULL here -> no "scores" attribute is attached. + perTreeScores <- NULL if (isTRUE(collapse) && length(resultTrees) > 0L) { # Contract zero-length (unsupported) branches into polytomies, à la TNT's # "collapse zero-length branches" -- done entirely in C++ (ts_collapse_pool) @@ -1816,7 +1898,7 @@ MaximizeParsimony <- function( # unsupported non-constraint branches still collapse. consSplitMatrix rows # are the enforced bipartitions in tip_data order, carrying both groups # (1 = together, 0 = apart, NA = free; see .PrepareConstraint). The kernel - # needs both: a tree with free tips generally realises the split at a node + # needs both: a tree with free tips generally realizes the split at a node # whose tip set is wider than the 1 group, which no exact match reaches, so # the enforced grouping would collapse out of the returned tree. consSplits <- if (!is.null(constraintConfig)) { @@ -1843,6 +1925,19 @@ MaximizeParsimony <- function( # C++ edge order may differ from template; renumber to valid preorder Renumber(tr) }) + # Surface the per-tree parsimony scores the engine returns aligned with + # result$trees. Attaching a "score" attribute to each tree lets + # Suboptimality(), SuboptimalTrees() and Bremer(method = "pool") read the + # suboptimal pool's landscape directly without re-scoring. + perTreeScores <- result$scores + if (length(perTreeScores) == length(outTrees)) { + outTrees <- Map(function(tr, sc) { + attr(tr, "score") <- sc + tr + }, outTrees, perTreeScores) + } else { + perTreeScores <- NULL + } } if (length(outTrees) == 0L) { # `treeTpl` is a starting tree, so under a constraint it is exactly what may @@ -1858,6 +1953,10 @@ MaximizeParsimony <- function( "with `maxReplicates` or `maxSeconds`.") } outTrees <- list(treeTpl) + if (!isTRUE(collapse)) { + attr(outTrees[[1]], "score") <- result$best_score + perTreeScores <- result$best_score + } } # --- XFORM: report the score of the tree we are actually returning --- @@ -1868,10 +1967,10 @@ MaximizeParsimony <- function( # of the returned tree does not reproduce -- measured at 178 reported against # 183 returned (T-385; repro in dev/red-team/heavy-tests/). Rescore the # returned pool at the canonical rooting instead: |pool| evaluations, negligible - # against a search, and `TreeLength()` canonicalises identically, so the two + # against a search, and `TreeLength()` canonicalizes identically, so the two # agree by construction. # - # This deliberately does NOT change what the search optimises. The reported + # This deliberately does NOT change what the search optimizes. The reported # value stays a rooting-dependent upper bound on the min-over-rootings # objective, exceeding it by at most the sum of `nSec` over hierarchy blocks # (measured: attained by 87-98% of rootings, mean overstatement 0.02-0.17 @@ -1924,6 +2023,8 @@ MaximizeParsimony <- function( structure( outTrees, score = bestScore, + scoring = scoringSignature, + scores = perTreeScores, replicates = result$replicates, hits_to_best = result$hits_to_best, n_topologies = nTopologies, @@ -1947,6 +2048,57 @@ MaximizeParsimony <- function( ) } +# Canonical scoring identity: the arguments that define the optimality criterion, +# normalized so aliases compare equal. Recorded on a MaximizeParsimony() result +# (attr "scoring") and checked by Bremer() -- a saved optimal score is only +# meaningful alongside the conditions it was optimal under. +.ScoringSignature <- function(concavity = Inf, extended_iw = TRUE, xpiwe_r = 0.5, + xpiwe_max_f = 5, hierarchy = NULL, + inapplicable = "bgs", hsj_alpha = 1.0) { + inap <- tolower(as.character(inapplicable)[[1]]) + if (inap == "brazeau") inap <- "bgs" # documented alias for the bgs kernel + list( + concavity = if (is.numeric(concavity)) concavity else + tolower(as.character(concavity)[[1]]), + extended_iw = isTRUE(extended_iw), + xpiwe_r = as.double(xpiwe_r), + xpiwe_max_f = as.double(xpiwe_max_f), + # Presence only: comparing full hierarchy objects is heavy and brittle, and a + # present-vs-absent difference is the practically important mismatch. + hierarchy = !is.null(hierarchy), + inapplicable = inap, + hsj_alpha = as.double(hsj_alpha) + ) +} + +# TRUE if two scoring signatures denote the same optimality criterion. +.ScoringSignatureMatch <- function(a, b, tol = 1e-8) { + numMatch <- function(x, y) { + if (is.numeric(x) && is.numeric(y)) { + (is.infinite(x) && is.infinite(y) && sign(x) == sign(y)) || + (is.finite(x) && is.finite(y) && abs(x - y) <= tol) + } else { + identical(as.character(x), as.character(y)) + } + } + numMatch(a[["concavity"]], b[["concavity"]]) && + identical(a[["extended_iw"]], b[["extended_iw"]]) && + isTRUE(abs(a[["xpiwe_r"]] - b[["xpiwe_r"]]) <= tol) && + isTRUE(abs(a[["xpiwe_max_f"]] - b[["xpiwe_max_f"]]) <= tol) && + identical(a[["hierarchy"]], b[["hierarchy"]]) && + identical(a[["inapplicable"]], b[["inapplicable"]]) && + isTRUE(abs(a[["hsj_alpha"]] - b[["hsj_alpha"]]) <= tol) +} + +# Compact human-readable rendering of a scoring signature for diagnostics. +.DescribeScoring <- function(sig) { + cn <- sig[["concavity"]] + weight <- if (is.numeric(cn) && is.infinite(cn)) "equal weights" else + paste0("concavity = ", if (is.numeric(cn)) signif(cn, 4) else cn) + paste0(weight, ", inapplicable = \"", sig[["inapplicable"]], "\"") +} + + #' Launch tree search graphical user interface #' #' Opens a "shiny" app for interactive parsimony tree search and results diff --git a/R/Ratchet.R b/R/Ratchet.R index 763878f05..0ee9d6869 100644 --- a/R/Ratchet.R +++ b/R/Ratchet.R @@ -284,7 +284,8 @@ Ratchet <- function(tree, dataset, #' @rdname Ratchet #' @return `MultiRatchet()` returns a list of optimal trees #' produced by `nSearch` `Ratchet()` searches, from which a consensus tree can -#' be generated using [`ape::consensus()`] or [`TreeTools::ConsensusWithout()`]. +#' be generated using [`TreeTools::Consensus()`] or +#' [`TreeTools::ConsensusWithout()`]. #' @param nSearch Number of Ratchet searches to conduct #' (for `RatchetConsensus()`) #' @export @@ -301,7 +302,11 @@ MultiRatchet <- function (tree, dataset, ratchHits = 10, }) scores <- vapply(trees, function (x) attr(x, "score"), double(1)) trees <- .UniqueExceptHits(trees[scores == min(scores)]) - message("Found ", length(trees), " unique trees from ", nSearch, " searches.") + + if (verbosity > 0L) { + message("Found ", length(trees), " unique trees from ", nSearch, + " searches.") + } # Return: structure(trees, class = "multiPhylo") diff --git a/R/RcppExports.R b/R/RcppExports.R index 348ed087b..2bb96e112 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -10,8 +10,8 @@ #' exp(logp) * TreeTools::NUnrooted(7) #' #' @export -MaddisonSlatkin <- function(steps, states) { - .Call(`_TreeSearch_MaddisonSlatkin`, steps, states) +MaddisonSlatkin <- function(steps, states, maxSeconds = 2.0) { + .Call(`_TreeSearch_MaddisonSlatkin`, steps, states, maxSeconds) } #' @export @@ -91,7 +91,7 @@ all_tbr <- function(edge, break_order) { #' Monte Carlo Fitch scores for a single character #' -#' Generates `n_mc` random trees and scores each with a Fitch parsimony +#' Generates `mcSamples` random trees and scores each with a Fitch parsimony #' downpass for a single character defined by `state_counts`. #' Tree generation and scoring are done entirely in C with no R object #' allocation per tree, making this very fast (~0.01 ms per tree). @@ -100,13 +100,13 @@ all_tbr <- function(edge, break_order) { #' state. Length determines the number of states (k); sum determines #' the number of tips (n). For example, `c(13, 13, 12)` defines a #' 3-state character with 38 tips. -#' @param n_mc Number of random trees to generate and score. -#' @return Integer vector of length `n_mc` containing the Fitch parsimony +#' @param mcSamples Number of random trees to generate and score. +#' @return Integer vector of length `mcSamples` containing the Fitch parsimony #' score (number of state changes) for each random tree. #' @keywords internal #' @export -mc_fitch_scores <- function(state_counts, n_mc) { - .Call(`_TreeSearch_mc_fitch_scores`, state_counts, n_mc) +mc_fitch_scores <- function(state_counts, mcSamples) { + .Call(`_TreeSearch_mc_fitch_scores`, state_counts, mcSamples) } ts_fitch_score <- function(edge, contrast, tip_data, weight, levels, min_steps = integer(), concavity = -1.0, infoAmounts = NULL, xpiwe = FALSE, xpiwe_r = 0.5, xpiwe_max_f = 5.0, obs_count = integer()) { diff --git a/R/SPR.R b/R/SPR.R index 619c9bb7c..0a76c09bc 100644 --- a/R/SPR.R +++ b/R/SPR.R @@ -64,8 +64,7 @@ SPRWarning <- function (parent, child, error) { #' \code{RootIrrelevantSPR} will search tree space more efficiently in these cases. #' Branch lengths are not (yet) supported. #' -#' All nodes in a tree must be bifurcating; [ape::collapse.singles] and -#' [ape::multi2di] may help. +#' All nodes in a tree must be bifurcating; [TreeTools::MakeTreeBinary] may help. #' #' @inheritParams TBR #' @param edgeToBreak the index of an edge to bisect, generated randomly if not specified. diff --git a/R/SearchControl.R b/R/SearchControl.R index c3607baac..c822afc5b 100644 --- a/R/SearchControl.R +++ b/R/SearchControl.R @@ -96,7 +96,7 @@ #' tips) above which a sector is solved by tree-drifting (`sectorGoDrift`) or by #' combined analysis (`sectorGoComb`) instead of plain #' \acronym{RAS}+\acronym{TBR}, following \acronym{TNT}'s `sectsch` -#' `godrift`/`gocomb`. Small sectors are cheap to optimise exhaustively; large +#' `godrift`/`gocomb`. Small sectors are cheap to optimize exhaustively; large #' sectors have more reach but need drift to escape their own local optima. #' `sectorGoComb` solves the sector with `sectorCombStarts` \acronym{RAS}+drift #' starts and then *fuses* them (recombines shared clades across the starts over @@ -212,7 +212,7 @@ #' characters are preferentially dropped. High-missingness taxa are #' hardest to score correctly and most likely to be trapped in suboptimal #' positions. -#' - `3` = combined: weight = instability × (1 + normalised missingness). +#' - `3` = combined: weight = instability × (1 + normalized missingness). #' Targets taxa that are both unstably placed and data-poor. #' @param pruneReinsertTbrMoves Integer; maximum number of TBR moves accepted #' during the reduced-tree backbone optimisation phase of each @@ -395,7 +395,7 @@ SearchControl <- function( enumMaxTrees = 0L ) { # Record which fields the caller set explicitly (by name or position; - # `match.call()` normalises positional args to their names). This lets + # `match.call()` normalizes positional args to their names). This lets # `MaximizeParsimony()` distinguish a user-supplied `control` field from a # default, so a partial `control = SearchControl(...)` merges correctly with # a `strategy` preset instead of silently overriding every preset field. diff --git a/R/SuboptimalTrees.R b/R/SuboptimalTrees.R new file mode 100644 index 000000000..c036eec7d --- /dev/null +++ b/R/SuboptimalTrees.R @@ -0,0 +1,118 @@ +#' Collect suboptimal trees for landscape analysis +#' +#' `SuboptimalTrees()` performs a parsimony search with [`MaximizeParsimony()`] +#' and returns every retained tree within a specified number of steps of the +#' optimum, each annotated with its parsimony score. +#' +#' The retained trees represent the search engine's internal tree pool, whose +#' size is bounded by `maxPool`. Once the pool is filled, trees are chosen for +#' eviction so as to preserve topological diversity. +#' +#' @inheritParams MaximizeParsimony +#' @param tree Optional starting tree (of class `phylo`) or `multiPhylo`; if +#' `NULL`, the search begins from random addition sequence trees. +#' @param maxSuboptimal Numeric: only trees whose score is within +#' `maxSuboptimal` of the best score found will be retained. +#' Corresponds to `poolSuboptimal` in [`SearchControl()`]. +#' @param maxPool Integer: maximum number of trees to retain in the pool. +#' Corresponds to `poolMaxSize` in [`SearchControl()`]. +#' @param \dots Further arguments passed to [`MaximizeParsimony()`], including +#' scoring options (`concavity`, `inapplicable`, ...), search effort +#' (`maxReplicates`, `maxSeconds`, `effort`, `nThreads`, `verbosity`), +#' and [`SearchControl()`] fields. +#' +#' @return `SuboptimalTrees()` returns a `multiPhylo` object listing the +#' retained trees, each supplied with a `score` attribute of each tree that +#' records the score it attained. The global `scores` attribute compiles these +#' scores in a numeric vector. +#' [`Suboptimality()`] reports each tree's excess over the optimum. +#' +#' @examples +#' data("inapplicable.phyData", package = "TreeSearch") +#' dataset <- inapplicable.phyData[["Vinther2008"]] +#' \donttest{ +#' set.seed(0) +#' trees <- SuboptimalTrees(dataset, maxSuboptimal = 3, maxReplicates = 5, +#' nThreads = 1, maxPool = 100L, verbosity = 0) +#' table(attr(trees, "scores")) # Parsimony lengths of retained trees +#' table(Suboptimality(trees)) # Excess over the optimum +#' } +#' @seealso +#' [`MaximizeParsimony()`] performs the underlying search; +#' [`Bremer()`][Bremer] uses this pool for approximate decay indices; +#' [`Suboptimality()`] summarizes the excess length of each tree. +#' @template MRS +#' @family split support functions +#' @family tree scoring +#' @export +SuboptimalTrees <- function(dataset, tree = NULL, + maxSuboptimal = 5, maxPool = 1000L, ...) { + if (!is.numeric(maxSuboptimal) || length(maxSuboptimal) != 1L || + maxSuboptimal < 0) { + stop("`maxSuboptimal` must be a single non-negative number.") + } + maxPool <- as.integer(maxPool) + if (is.na(maxPool) || maxPool < 1L) { + stop("`maxPool` must be a single positive integer.") + } + + # SuboptimalTrees() manages `control` (to size the pool) and `collapse` (which + # must stay FALSE to retain the full suboptimal sample). A whole `control=` + # or `collapse=` argument arriving through `...` would otherwise collide with + # these explicit arguments ("matched by multiple actual arguments"). Strip + # them; individual SearchControl() fields (e.g. `ratchetCycles = `) still pass + # through `...` and are merged by MaximizeParsimony(). + dots <- list(...) + if ("control" %in% names(dots)) { + warning("`control` is managed by SuboptimalTrees(); pass individual ", + "SearchControl() fields (e.g. `ratchetCycles = `) via `...` instead.") + dots[["control"]] <- NULL + } + if ("collapse" %in% names(dots)) { + if (!identical(dots[["collapse"]], FALSE)) { + warning("`collapse` is forced to FALSE by SuboptimalTrees() so the full ", + "suboptimal pool is returned.") + } + dots[["collapse"]] <- NULL + } + # `poolSuboptimal`/`poolMaxSize` are the raw SearchControl() fields that + # `maxSuboptimal`/`maxPool` set below. Passed via `...` they would flow into + # MaximizeParsimony()'s dots-override-control merge and SILENTLY win over the + # constructed control (last-writer-wins), defeating `maxSuboptimal`/`maxPool`. + # Strip them with a warning, directing the caller to the blessed arguments. + managed <- intersect(c("poolSuboptimal", "poolMaxSize"), names(dots)) + if (length(managed)) { + warning("Set the suboptimal-pool size via `maxSuboptimal` / `maxPool`, not ", + "`", paste(managed, collapse = "` / `"), + "`; the latter would silently override the former and ", + if (length(managed) > 1L) "are" else "is", " ignored.") + dots[managed] <- NULL + } + + control <- SearchControl(poolSuboptimal = maxSuboptimal, poolMaxSize = maxPool) + + do.call(MaximizeParsimony, + c(list(dataset = dataset, tree = tree, control = control, + collapse = FALSE), dots)) +} + +#' Compare scores of suboptimal trees +#' +#' @param trees List of trees, perhaps generated through `SuboptimalTrees()`, +#' each of which must bear a numeric attribute `score` giving its score. +#' @param normalize Logical stating whether to normalize results to lowest +#' score. +#' @return `Suboptimality()` returns a numeric vector listing, for each tree, +#' its difference in score from the optimal (lowest) within `trees`. +#' @export +Suboptimality <- function (trees, normalize = FALSE) { + scores <- vapply(trees, attr, double(1), "score") + minScore <- min(scores) + + # Return: + if (normalize) { + (scores - minScore) / minScore + } else { + scores - minScore + } +} diff --git a/R/SuccessiveApproximations.R b/R/SuccessiveApproximations.R index 52361f9c5..4ed63b0fc 100644 --- a/R/SuccessiveApproximations.R +++ b/R/SuccessiveApproximations.R @@ -144,28 +144,6 @@ SuccessiveApproximations <- function (tree, dataset, outgroup = NULL, k = 3, ) } -#' Tree suboptimality -#' -#' How suboptimal is a tree? -#' -#' @param trees list of trees, to include an optimal tree -#' @param proportional logical stating whether to normalise results to lowest -#' score -#' @return `Suboptimality()` returns a vector listing, for each tree, how much -#' its score differs from the optimal (lowest) score. -#' @keywords internal -#' @export -Suboptimality <- function (trees, proportional = FALSE) { - scores <- vapply(trees, attr, double(1), "score") - - # Return: - if (proportional) { - (scores - min(scores)) / min(scores) - } else { - scores - min(scores) - } -} - #' @rdname SuccessiveApproximations #' @return `SuccessiveWeights()` returns the score of a tree, given the #' weighting instructions specified in the attributes of the dataset. diff --git a/R/TBR.R b/R/TBR.R index d0bf968ef..cb9469419 100644 --- a/R/TBR.R +++ b/R/TBR.R @@ -23,8 +23,7 @@ TBRWarning <- function (parent, child, error) { #' #' Branch lengths are not (yet) supported. #' -#' All nodes in a tree must be bifurcating; [ape::collapse.singles] and -#' [ape::multi2di] may help. +#' All nodes in a tree must be bifurcating; [TreeTools::MakeTreeBinary] may help. #' #' @param tree A bifurcating tree of class \code{\link[ape]{phylo}}, with all nodes resolved; #' @param edgeToBreak (optional) integer specifying the index of an edge to bisect/prune, diff --git a/R/TaxonInfluence.R b/R/TaxonInfluence.R index 5d6919ead..a79afe4ea 100644 --- a/R/TaxonInfluence.R +++ b/R/TaxonInfluence.R @@ -23,7 +23,7 @@ #' It is possible that a set of trees contains all possible resolutions of a #' particular clade, and a single other topology in which that clade does not #' exist – essentially two distinct solutions, one (_a_) which could be -#' summarised with a summary tree that contains a polytomy, and another (_b_) +#' summarized with a summary tree that contains a polytomy, and another (_b_) #' which could be summarized by a perfectly resolved tree. #' Neither of these scenarios is preferable under the principles of parsimony; #' but summary statistics (e.g. mean, median) will be strongly influenced by the diff --git a/R/data_manipulation.R b/R/data_manipulation.R index 20aff7344..f51f43395 100644 --- a/R/data_manipulation.R +++ b/R/data_manipulation.R @@ -14,18 +14,14 @@ # k=5: n=9 (2,2,2,2,1) sc=35 logB 142 logPVec 4990 <- threshold # # This gate is not the latency control, and should not be tuned as one. It -# skips work hopeless enough not to be worth starting; what a caller actually -# waits is capped by `TIME_BUDGET_S` in MaddisonSlatkin.cpp, which stops the -# recursion mid-flight and falls back to Monte Carlo. Admitting a character -# here therefore costs at most that budget, not the figures below. +# skips work hopeless enough not to be worth starting; callers can set a +# per-character time budget with `maxSeconds` in `StepInformation()`, which +# stops the recursion mid-flight and falls back to Monte Carlo. # -# Consequently these thresholds may be generous without hurting anyone, and -# raising one does not make the package less responsive. Timings, for scale -# only, on a 2021-vintage desktop: (9,9,9) ~12.7 s, (8,7,5) ~1.9 s -- i.e. most -# of the k=3 range is stopped by the clock, not finished. Earlier revisions of -# this comment quoted sub-second figures for the same characters and are -# superseded; they were ~13x optimistic, which is how a 2 s budget came to look -# like a backstop when it was in fact the operative limit. +# Consequently these thresholds may be generous without forcing callers to +# wait: they can select a shorter budget for approximation or a longer one for +# exact results. Timings, for scale only, on a 2021-vintage desktop: +# (9,9,9) ~12.7 s and (8,7,5) ~1.9 s. .MS_SC_THRESHOLD <- c(Inf, Inf, 75L, 50L, 35L) .MSSplitCount <- function(state_counts) { @@ -65,15 +61,19 @@ #' (i.e. `{02}` \ifelse{html}{\out{→}}{\eqn{\rightarrow}{-->}} `?`). #' #' @param dataset dataset of class \code{phyDat} -#' @param approx Character string controlling how profile information amounts -#' are computed for multi-state characters with many tips. +#' @param approx Character string controlling how profile information +#' amounts are computed for multi-state characters with many tips. #' `"auto"` (default) uses the exact Maddison & Slatkin calculation when #' feasible, falling back to a Monte Carlo approximation for large or #' complex characters. #' `"mc"` always uses the Monte Carlo approximation; -#' `"exact"` always uses the exact calculation (may be very slow). -#' @param n_mc Integer; number of Monte Carlo samples for the MC -#' approximation. Default 100 000. +#' `"exact"` always uses the exact calculation regardless of the feasibility +#' gate. +#' @param maxSeconds Non-negative numeric giving the time budget, in seconds, +#' allowed to the exact solver (under `"auto"` or `"exact"`) before falling +#' back to the Monte Carlo approximation. +#' @param mcSamples Integer specifying number of Monte Carlo samples for the MC +#' approximation. #' #' @return An object of class `phyDat`, with additional attributes. #' `PrepareDataProfile` adds the attributes: @@ -103,7 +103,8 @@ #' @family profile parsimony functions #' @encoding UTF-8 #' @export -PrepareDataProfile <- function (dataset, approx = "auto", n_mc = 100000L) { +PrepareDataProfile <- function (dataset, approx = "auto", maxSeconds = 2, + mcSamples = 1e5L) { if ("info.amounts" %fin% names(attributes(dataset))) { # Already prepared return(dataset) @@ -225,7 +226,7 @@ PrepareDataProfile <- function (dataset, approx = "auto", n_mc = 100000L) { # --- Compute StepInformation per unique pattern --- info <- lapply(seq_along(mataset[1, ]), function (i) StepInformation(mataset[, i], ambiguousTokens = AMBIG_TOKEN, - approx = approx, n_mc = n_mc)) + approx = approx, maxSeconds = maxSeconds, mcSamples = mcSamples)) maxSteps <- max(vapply(info, diff --git a/R/pp_info_extra_step.r b/R/pp_info_extra_step.r index eca1cd020..de63b000f 100644 --- a/R/pp_info_extra_step.r +++ b/R/pp_info_extra_step.r @@ -17,7 +17,9 @@ #' When the Maddison & Slatkin computation would be infeasible (exponential #' in the number of tips for a given number of tokens), behaviour depends on #' the `approx` argument. With `"auto"` (default), the exact solver is used -#' where feasible and the Monte Carlo approximation is used otherwise. +#' where feasible and the Monte Carlo approximation is used otherwise; the +#' exact solver is allowed a time budget of `maxSeconds`, past which the +#' calculation falls back to the Monte Carlo approximation. #' With `"mc"`, the Monte Carlo approximation is always used. #' The MC approximation computes the exact #' minimum-steps probability analytically, uses random trees for the @@ -27,18 +29,21 @@ #' beyond ~27 tips, 4-state beyond ~13 tips, and 5-state beyond #' ~9 tips trigger the approximation. #' With `"exact"`, the full Maddison & Slatkin recursion is forced regardless -#' of cost (may be very slow for large or complex characters). +#' of the feasibility gate (may be very slow for large or complex +#' characters), still subject to `maxSeconds`. #' #' @param char Vector of tokens listing states for the character in question. #' @param ambiguousTokens Vector specifying which tokens, if any, correspond to #' the ambiguous token (`?`). -#' @param approx Character string controlling the computation method: -#' `"auto"` (default) uses exact computation when feasible, falling back to -#' Monte Carlo for large or complex characters (see Details); +#' @param approx Character string specifying the computation method: #' `"mc"` always uses the Monte Carlo approximation; -#' `"exact"` forces exact computation regardless of cost (may be very slow -#' for large or complex characters). -#' @param n_mc Integer. Number of random trees used by the MC approximation. +#' `"exact"` forces exact computation regardless of the feasibility gate; +#' `"auto"` (default) uses exact computation when feasible, falling back to +#' Monte Carlo for large or complex characters (see Details). +#' @param maxSeconds Non-negative numeric giving the time budget, in seconds, +#' allowed to the exact solver (under `"auto"` or `"exact"`) before falling +#' back to the Monte Carlo approximation. +#' @param mcSamples Integer. Number of random trees used by the MC approximation. #' Larger values improve accuracy but increase computation time. #' Default: 100 000. #' @@ -58,8 +63,9 @@ #' @importFrom TreeTools Log2Unrooted LnUnrooted NUnrooted NUnrootedMult #' @family profile parsimony functions #' @export -StepInformation <- function (char, ambiguousTokens = c("-", "?"), - approx = "auto", n_mc = 100000L) { +StepInformation <- function(char, ambiguousTokens = c("-", "?"), + approx = c("auto", "mc", "exact"), maxSeconds = 2, + mcSamples = 1e5L) { NIL <- c("0" = 0) char <- char[!char %fin% ambiguousTokens] if (length(char) == 0) { @@ -81,16 +87,21 @@ StepInformation <- function (char, ambiguousTokens = c("-", "?"), k <- length(split) nTips <- sum(split) - + method <- match.arg(approx) + if (!is.numeric(maxSeconds) || length(maxSeconds) != 1L || + is.na(maxSeconds) || !is.finite(maxSeconds) || maxSeconds < 0) { + stop("`maxSeconds` must be a non-negative, finite, numeric value.") + } + # Exact MaddisonSlatkin is only instantiated for k <= 5; larger k always # uses MC (bitmask Fitch in mc_fitch_scores supports up to 32 states). # For k <= 5, use partition-aware split_count to decide feasibility. infeasible <- k > 5L || (k >= 3L && .MSSplitCount(split) > .MS_SC_THRESHOLD[k]) - if (identical(approx, "mc") || - (infeasible && !identical(approx, "exact"))) { - return(.ApproxStepInformation(split, n_mc = n_mc, + if (identical(method, "mc") || + (infeasible && !identical(method, "exact"))) { + return(.ApproxStepInformation(split, mcSamples = mcSamples, nSingletons = nSingletons)) } @@ -111,12 +122,12 @@ StepInformation <- function (char, ambiguousTokens = c("-", "?"), reducedMinSteps <- k - 1L maxSteps <- nTips - 1L logP <- tryCatch( - MaddisonSlatkin(reducedMinSteps:maxSteps, states), + MaddisonSlatkin(reducedMinSteps:maxSteps, states, maxSeconds), error = function(e) NULL ) if (is.null(logP) || anyNA(logP)) { # Exact solver hit capacity limit or timed out; fall back to MC - return(.ApproxStepInformation(split, n_mc = n_mc, + return(.ApproxStepInformation(split, mcSamples = mcSamples, nSingletons = nSingletons)) } } @@ -146,11 +157,11 @@ StepInformation <- function (char, ambiguousTokens = c("-", "?"), # # @param split Integer vector of informative token frequencies (sorted # decreasing, singletons removed). -# @param n_mc Integer. Number of Monte Carlo trees to score. +# @param mcSamples Integer. Number of Monte Carlo trees to score. # @param nSingletons Integer. Number of singleton tokens (for step offset). # @return Named numeric vector of IC (bits) by step count. # @keywords internal -.ApproxStepInformation <- function(split, n_mc = 100000L, nSingletons = 0L) { +.ApproxStepInformation <- function(split, mcSamples = 100000L, nSingletons = 0L) { k <- length(split) n <- sum(split) s_min <- k - 1L @@ -161,7 +172,7 @@ StepInformation <- function (char, ambiguousTokens = c("-", "?"), # 2. MC: generate and score random trees via compiled Fitch downpass. # No R object allocation per tree; ~0.01 ms per tree. - mc_scores <- mc_fitch_scores(split, n_mc) + mc_scores <- mc_fitch_scores(split, mcSamples) mu_hat <- mean(mc_scores) sd_hat <- sd(mc_scores) @@ -186,7 +197,7 @@ StepInformation <- function (char, ambiguousTokens = c("-", "?"), # Fill MC body: all bins from s_lo onward for (i in s_lo_idx:length(mc_tab)) { if (mc_tab[i] > 0L) { - log_p[i] <- log(mc_tab[i] / n_mc) + log_p[i] <- log(mc_tab[i] / mcSamples) } else { # Right tail: normal extrapolation (negligible IC contribution) log_p[i] <- dnorm(s_min + i - 1L, mu_hat, sd_hat, log = TRUE) @@ -231,7 +242,7 @@ StepInformation <- function (char, ambiguousTokens = c("-", "?"), s <- steps[i] cnt <- mc_tab[i] log_p[i] <- if (cnt > 0L) { - log(cnt / n_mc) + log(cnt / mcSamples) } else { dnorm(s, mu_hat, sd_hat, log = TRUE) } @@ -319,7 +330,7 @@ StepInformation <- function (char, ambiguousTokens = c("-", "?"), #' **binary** characters, where _a_ leaves bear one state and _b_ bear the #' other. #' -#' `MaddisonSlatkin()` generalises this result to characters with multiple +#' `MaddisonSlatkin()` generalizes this result to characters with multiple #' states using the recursive approach of #' \insertCite{Maddison1991;textual}{TreeSearch}. #' It returns the **log-probability** (i.e. log of the fraction of unrooted @@ -336,6 +347,8 @@ StepInformation <- function (char, ambiguousTokens = c("-", "?"), #' entry 3 = ambiguous state `{1,2}` (binary `011`), and so on. #' Only observed singleton states need non-zero counts; polymorphic entries #' are typically zero. +#' @param maxSeconds Numeric giving maximum computation time in seconds. +#' `NA` is returned when the budget is exceeded. #' #' @return `Carter1()` returns the number of unrooted binary trees on which a #' binary character with `a` leaves in one state and `b` in the other can be diff --git a/R/tree_length.R b/R/tree_length.R index 7d558d3ed..877fdd60f 100644 --- a/R/tree_length.R +++ b/R/tree_length.R @@ -311,7 +311,7 @@ TreeLength.list <- function(tree, dataset, concavity = Inf, tree[] <- lapply(tree, function(tr) if (TreeIsRooted(tr)) tr else RootTree(tr, 1)) if (useXform) { # XFORM's score is rooting-dependent (asymmetric step matrix), so an - # *already*-rooted tree must be canonicalised too, not just an unrooted one: + # *already*-rooted tree must be canonicalized too, not just an unrooted one: # otherwise the same topology gets different lengths from different rootings # and no length agrees with what `MaximizeParsimony()` reports. See the # single-tree method above for the full rationale. No warning here -- unlike diff --git a/data-raw/Wills2012.nex b/data-raw/Wills2012.nex index cbde191c3..881c84ed2 100644 --- a/data-raw/Wills2012.nex +++ b/data-raw/Wills2012.nex @@ -576,7 +576,7 @@ BEGIN NOTES; TEXT CHARACTER=68 TEXT="Lemburg (1999) recognized this as a synapomorphy of lorciferans and extant priapulids."; TEXT CHARACTER=69 TEXT=" The presence of a flagellum in spermatozoa is a metazoan symplesiomorphy, but a flagellum is lacking from the spermatozoa of nematodes and nematomorphs (Schmidt-Rhaesa 1998)."; TEXT CHARACTER=64 TEXT="Lemburg (1999) recognizes the presence of this character as a synapomorphy of Megaintroverta (Priapulopsis + Acanthopriapulus + Priapulus)."; - TEXT CHARACTER=63 TEXT="Lemburg (1999) recognised the presence of this character as a synapomorphy of (extant) Priapulida. However, it has since been demonstrated that the larvae of nematomorphs also possess six pharyngeal retractor muscles (Kristensen, 2003; Muller et al., 2004). "; + TEXT CHARACTER=63 TEXT="Lemburg (1999) recognized the presence of this character as a synapomorphy of (extant) Priapulida. However, it has since been demonstrated that the larvae of nematomorphs also possess six pharyngeal retractor muscles (Kristensen, 2003; Muller et al., 2004). "; TEXT CHARACTER=56 TEXT="Lemburg (1999) recognized this as a synapomorphy of Introverta (Nematoda + Nematomorpha + Kinorhyncha + Loricifera + Priapulida)."; TEXT CHARACTER=58 TEXT="Lemburg (1999) recognizes the presence of this character as a synapomorphy of (extant) Priapulida."; diff --git a/dev/benchmarks/ab_pr_bench.R b/dev/benchmarks/ab_pr_bench.R index 2ba5505aa..d2b17b195 100644 --- a/dev/benchmarks/ab_pr_bench.R +++ b/dev/benchmarks/ab_pr_bench.R @@ -1,5 +1,5 @@ #!/usr/bin/env Rscript -# A/B: original (converging reduced-TBR) vs optimised (5-move limit + code fixes) +# A/B: original (converging reduced-TBR) vs optimized (5-move limit + code fixes) # Both libraries must already be installed. # orig_lib = .vtune-lib (tbr_max_moves=0, no build_postorder fix) # opt_lib = .agent-Eopt (tbr_max_moves=5, build_postorder deferred) diff --git a/dev/benchmarks/ab_pr_seq.R b/dev/benchmarks/ab_pr_seq.R index 86b055375..82c108855 100644 --- a/dev/benchmarks/ab_pr_seq.R +++ b/dev/benchmarks/ab_pr_seq.R @@ -1,5 +1,5 @@ #!/usr/bin/env Rscript -# Sequential A/B: original vs optimised PR, run in subprocesses one at a time +# Sequential A/B: original vs optimized PR, run in subprocesses one at a time orig_lib <- "C:/Users/pjjg18/GitHub/TreeSearch-a/.vtune-lib" opt_lib <- "C:/Users/pjjg18/GitHub/TreeSearch-a/.agent-Eopt" diff --git a/dev/benchmarks/b2_collapsed_density.R b/dev/benchmarks/b2_collapsed_density.R index dc8c0d8e4..964045044 100644 --- a/dev/benchmarks/b2_collapsed_density.R +++ b/dev/benchmarks/b2_collapsed_density.R @@ -26,7 +26,7 @@ # of zero-length edges => the active merging captures ~nothing and there is no # wall-clock win in B2's main mechanism for this class. If it is substantial, # the win is already being CAPTURED (merging is on) => still nothing new to build, -# but worth confirming the trajectory benefit is realised. +# but worth confirming the trajectory benefit is realized. # # Usage: TS_LIB=.agent-fuse Rscript dev/benchmarks/b2_collapsed_density.R diff --git a/dev/benchmarks/bench_na_certify_cell.R b/dev/benchmarks/bench_na_certify_cell.R index 42d5cf34b..57948da76 100644 --- a/dev/benchmarks/bench_na_certify_cell.R +++ b/dev/benchmarks/bench_na_certify_cell.R @@ -33,7 +33,7 @@ # sub-searches and the fuse cleanup reach the certifier; at tabuSize = 0 # (the `sprint` preset, and every raw ts_tbr_search / ts_ratchet_search call, # which is how the 97.7% was measured) every whole-tree search does. Running -# both is what makes the result generalise beyond one preset. +# both is what makes the result generalize beyond one preset. # # `naDiag` is recorded per run. A null result with n_evs_skipped == 0 means the # flag never fired, which is a different finding from "it fired and bought diff --git a/dev/benchmarks/bench_outer_cycles.R b/dev/benchmarks/bench_outer_cycles.R index ac0f56da2..cd199e060 100644 --- a/dev/benchmarks/bench_outer_cycles.R +++ b/dev/benchmarks/bench_outer_cycles.R @@ -134,7 +134,7 @@ library(dplyr) summary_tbl <- results_df |> filter(!is.na(best_score)) |> group_by(dataset, n_taxa, condition) |> - summarise(median_score = median(best_score), + summarize(median_score = median(best_score), median_reps = median(replicates), .groups = "drop") |> tidyr::pivot_wider(names_from = condition, diff --git a/dev/benchmarks/bench_pr_stage3_mbank.R b/dev/benchmarks/bench_pr_stage3_mbank.R index 9eff3c5f6..d828a84c5 100644 --- a/dev/benchmarks/bench_pr_stage3_mbank.R +++ b/dev/benchmarks/bench_pr_stage3_mbank.R @@ -22,7 +22,7 @@ # 0 = RANDOM uniform random # 1 = INSTABILITY weighted by positional instability in pool # 2 = MISSING weighted by ambiguous/inapplicable character count -# 3 = COMBINED instability × (1 + normalised missingness) +# 3 = COMBINED instability × (1 + normalized missingness) # # Usage: # Rscript bench_pr_stage3_mbank.R [timeout_s] [output_dir] diff --git a/dev/benchmarks/bench_pr_stage4_validation.R b/dev/benchmarks/bench_pr_stage4_validation.R index 6199f03c2..746e7c5ca 100644 --- a/dev/benchmarks/bench_pr_stage4_validation.R +++ b/dev/benchmarks/bench_pr_stage4_validation.R @@ -10,7 +10,7 @@ # PR enabled in large preset: c=5, d=5%, sel=MISSING # # Stage 4 goals: -# 1. Verify PR benefit generalises across 5 independent large matrices +# 1. Verify PR benefit generalizes across 5 independent large matrices # spanning 131-206 tips. # 2. Check whether benefit persists or baseline catches up at 120s budget. # diff --git a/dev/benchmarks/bench_stress_large.R b/dev/benchmarks/bench_stress_large.R index 47308da31..d424b733f 100644 --- a/dev/benchmarks/bench_stress_large.R +++ b/dev/benchmarks/bench_stress_large.R @@ -40,7 +40,7 @@ prep_ds <- function(phyDat) { n_taxa = length(phyDat), n_chars = ncol(tip_data)) } -# ---- Section 1: Load matrices and summarise --------------------------------- +# ---- Section 1: Load matrices and summarize --------------------------------- cat("=== T-069 Large-Matrix Stress Test ===\n\n") cat("=== Section 1: Dataset summary ===\n\n") diff --git a/dev/benchmarks/bench_trajectory.R b/dev/benchmarks/bench_trajectory.R index bb7fef113..1c644e9d3 100644 --- a/dev/benchmarks/bench_trajectory.R +++ b/dev/benchmarks/bench_trajectory.R @@ -371,7 +371,7 @@ ts_replicate_trajectory <- function(ts_result) { library(dplyr) traj |> group_by(replicate) |> - summarise( + summarize( rep_score = last(score), total_phase_ms = sum(phase_ms), n_phases = n(), @@ -401,7 +401,7 @@ ts_phase_effectiveness <- function(ts_result) { ungroup() |> filter(phase != "Wagner", phase != "Reset") |> group_by(phase) |> - summarise( + summarize( n = n(), n_improved = sum(improved), hit_rate = round(mean(improved), 3), diff --git a/dev/benchmarks/data-na/lobo.nex b/dev/benchmarks/data-na/lobo.nex index c22e430a3..4b14f576f 100644 --- a/dev/benchmarks/data-na/lobo.nex +++ b/dev/benchmarks/data-na/lobo.nex @@ -1769,7 +1769,7 @@ TEXT CHARACTER=224 TEXT='(~) inapplicable: loricate stage absent^nCf. WTS62^n'; TEXT CHARACTER=225 TEXT='(~) inapplicable: direct development, or larva without defined neck. ^nCrenulated in the Higgins larva of loriciferans (Neves et al. 2016)^nNot in Halicryptus (Storch and Higgins 1991; Janssen et al. 2009)^n'; TEXT CHARACTER=226 TEXT='(~) inapplicable: direct development^nPliciloricus and Urnaloricus have a collar region in their neck, whereas Nanaloricus and Tenuiloricus do not (Neves and Kristensen 2014).^n'; - TEXT CHARACTER=227 TEXT='(–) inapplicable^nWTS63. Lemburg (1999) recognised the presence of this character as a synapomorphy of (extant) Priapulida. However, it has since been demonstrated that the larvae of nematomorphs also possess six pharyngeal retractor muscles (Kristensen 2003; Müller et al. 2004). Long pharynx retractor muscles are also present in loriciferans (at least within the Nanaloricidae) (Neves et al. 2013).^n'; + TEXT CHARACTER=227 TEXT='(–) inapplicable^nWTS63. Lemburg (1999) recognized the presence of this character as a synapomorphy of (extant) Priapulida. However, it has since been demonstrated that the larvae of nematomorphs also possess six pharyngeal retractor muscles (Kristensen 2003; Müller et al. 2004). Long pharynx retractor muscles are also present in loriciferans (at least within the Nanaloricidae) (Neves et al. 2013).^n'; TEXT CHARACTER=228 TEXT='(~) inapplicable: no larval form (direct development; trans. ser. ###)^nWTS65.^nA proposed synapomorphy of scalidophora, but also present in nematomorphs ^nPresent in Priapulid caudatus, Tubiluchis corallicola; absent in Kinorhyncha, Loricifera ; probably absent in Nectonema larvae yet present in adults (Schmidt-Rhaesa 1997a)^n'; TEXT CHARACTER=229 TEXT='(~) inapplicable^nProposed as a synapomorphy between nematomorphs and loriciferans (Kristensen 1983)^nNot reported in Halicryptus, Maccabeus or Priapulus (van der Land 1970; Por and Bromley 1974; Storch and Higgins 1991), but present in Tubiluchus (Higgins and Storch 1989)^n'; TEXT CHARACTER=230 TEXT='(~) inapplicable: proboscis and abdomen undivided^nThe larvae of Orstenoloricus (Maas and Waloszek 2009) possess a pair of spines at the anterior of the trunk. These may correspond to the anteroventral setae of Tenuiloricus (Neves and Kristensen 2014). Similar spines are present in Nanaloricidae and Pliciloricidae (Neves et al. 2016)^nThe tubuli present in the Halicryptus hatching larva are not paired, and disappear in the Higgins larval stage; paired spines are coded as absent in this taxon (Storch and Higgins 1991; Janssen et al. 2009) similar structures present in Tubiluchus (Kirsteuer 1976)'; @@ -1809,4 +1809,4 @@ BEGIN ASSUMPTIONS; TYPESET * UNTITLED = unord: 1 - 251; ENDBLOCK; - \ No newline at end of file + diff --git a/dev/benchmarks/data/README.md b/dev/benchmarks/data/README.md index 4141c5c5c..84abe74c8 100644 --- a/dev/benchmarks/data/README.md +++ b/dev/benchmarks/data/README.md @@ -41,7 +41,7 @@ To rebuild `zhu2013_2island_ref.rds` from scratch: 1. Run TreeSearch (`inapplicable = "missing"`, `concavity = Inf`) and TNT (`mult = replic 100 tbr; bbreak = tbr;`, gaps→missing) on `zhu2013_2island.nex`; keep the unique length-598 topologies from each. -2. Canonicalise every tree as `write.tree(SortTree(RootTree(t, outgroup)))`. +2. Canonicalize every tree as `write.tree(SortTree(RootTree(t, outgroup)))`. 3. `island2` = the TNT-only set; confirm it is one disconnected component by seeding TreeSearch from any one of them with ratchet/sector/drift/fuse disabled (`accept_equal` TBR walk only) and checking the closure contains diff --git a/dev/benchmarks/diag_tbr_falseconv_check.R b/dev/benchmarks/diag_tbr_falseconv_check.R index 3f00f548a..ed06b787a 100644 --- a/dev/benchmarks/diag_tbr_falseconv_check.R +++ b/dev/benchmarks/diag_tbr_falseconv_check.R @@ -1,7 +1,7 @@ # Is our default TBR's "convergence" genuine, on the SHIPPING cpp-search build # (post directional-vroot fix, commit 2b299e4b)? We run TBR to convergence via # ts_tbr_diagnostics, then enumerate the FULL unrooted canonical-TBR neighbourhood -# of the result with the SEPARATE, unoptimised enumerator TBRMoves (-> all_tbr in +# of the result with the SEPARATE, unoptimized enumerator TBRMoves (-> all_tbr in # rearrange.cpp, a different code path). >0 improving neighbour => the kernel # falsely declared convergence (the competent-chaum move-incompleteness finding). # @@ -26,7 +26,7 @@ d <- list(phy = phy, contrast = at$contrast, weight = at$weight, levels = at$levels, nTip = length(phy)) norm <- function(tr) Preorder(RenumberTips(tr, names(d$phy))) -# Run our default (rooted, optimised) TBR to convergence from a warm start. +# Run our default (rooted, optimized) TBR to convergence from a warm start. tsTbr <- function(start, seed) { set.seed(seed) res <- TreeSearch:::ts_tbr_diagnostics(norm(start)[["edge"]], d$contrast, d$tip_data, diff --git a/dev/benchmarks/diag_treespace_sampling.R b/dev/benchmarks/diag_treespace_sampling.R index 5a628d4fb..c204cc11a 100644 --- a/dev/benchmarks/diag_treespace_sampling.R +++ b/dev/benchmarks/diag_treespace_sampling.R @@ -38,7 +38,7 @@ wd <- file.path(tempdir(), "tntsamp"); dir.create(wd, showWarnings = FALSE, recu if (length(trees) == 1L) return(trees[[1]]) ape::consensus(trees, p = 1) } -.asMP <- function(tr) { # normalise to multiPhylo list of phylo +.asMP <- function(tr) { # normalize to multiPhylo list of phylo if (is.null(tr)) return(list()) if (inherits(tr, "phylo")) return(list(tr)) unclass(tr) diff --git a/dev/benchmarks/frac-quart/chance_baseline.R b/dev/benchmarks/frac-quart/chance_baseline.R index 38187da82..9bea2aac4 100644 --- a/dev/benchmarks/frac-quart/chance_baseline.R +++ b/dev/benchmarks/frac-quart/chance_baseline.R @@ -16,7 +16,7 @@ pos <- function(z) { z[z < 0] <- 0; z } -# Observed per-pair triple for one realised cell vector. +# Observed per-pair triple for one realized cell vector. pairTriple <- function(p, q, r, s) { nI <- p + q; nJ <- r + s mA <- p + r; tP <- nI + nJ diff --git a/dev/benchmarks/mbank_X30754.nex b/dev/benchmarks/mbank_X30754.nex index 4e6507490..a56cd42ad 100644 --- a/dev/benchmarks/mbank_X30754.nex +++ b/dev/benchmarks/mbank_X30754.nex @@ -3035,7 +3035,7 @@ TEXT CHARACTER=251 TEXT='Microdictyon and Onychodictyon plates exhibit mushroom-like bosses at the junction of the net-like pattern.'; TEXT CHARACTER=252 TEXT='Arthropodization is thought to happen first in pre-ocular appendages, then co-opted to the rest of the appendages [@Chipman2019]. This character reflects this hypothesised event. Sclerotization is thought to occur simultaneously in all trunk appendages (as they have been co-opted from the pre-ocular appendages), therefore we code this in one character. This is treated as a neomorphic character as the trunk appendages" co-option of sclerotization from pre-ocular appendages requires additional genetic control.^n'; TEXT CHARACTER=253 TEXT='(~) inapplicable: lateral flaps (trans. ser. 55) not present^nTransformation series 38 in Daley et al. (2009).'; - TEXT CHARACTER=254 TEXT='Treated as neomorphic.^n^nTo summarise @Daley2009 and @VanRoy2015:^nExopods, the outer branch of a true biramous limb, are unique to Mandibulata. The outer appendage branch of chelicerates and many stem-group euarthropods is interpreted as an exite, a lateral flap which is not homologous to the mandibulate exopod [see also @Bruce2020]. ^n@Daley2009 treated this flap as homologous with the lateral flaps of anomalocaridids and gilled lobopodians, which often bear dorsal lanceolate blades (= setal blades). ^n@VanRoy2015 considered the setal blades themselves to represent the exite, homologizing the wrinkling on Kerygmachela and Pambdelurion flaps, the setal blades of Opabinia and anomalocaridids, and the exites of upper stem euarthropods. Dorsal flaps are therefore not necessarily present in addition to the setal blades; indeed @VanRoy2015 code them as absent in Amplectobeluids and Anomalocaris (as well as euarthropods).^n^nAdapted from character 31 in @Daley2009. Character 55 in @Smith2015 and 53 in @Yang2015.^n'; + TEXT CHARACTER=254 TEXT='Treated as neomorphic.^n^nTo summarize @Daley2009 and @VanRoy2015:^nExopods, the outer branch of a true biramous limb, are unique to Mandibulata. The outer appendage branch of chelicerates and many stem-group euarthropods is interpreted as an exite, a lateral flap which is not homologous to the mandibulate exopod [see also @Bruce2020]. ^n@Daley2009 treated this flap as homologous with the lateral flaps of anomalocaridids and gilled lobopodians, which often bear dorsal lanceolate blades (= setal blades). ^n@VanRoy2015 considered the setal blades themselves to represent the exite, homologizing the wrinkling on Kerygmachela and Pambdelurion flaps, the setal blades of Opabinia and anomalocaridids, and the exites of upper stem euarthropods. Dorsal flaps are therefore not necessarily present in addition to the setal blades; indeed @VanRoy2015 code them as absent in Amplectobeluids and Anomalocaris (as well as euarthropods).^n^nAdapted from character 31 in @Daley2009. Character 55 in @Smith2015 and 53 in @Yang2015.^n'; TEXT CHARACTER=255 TEXT='The description of Aegirocassis @VanRoy2015 clarifies the relationship of the dorsal lanceolate (setal) blades in gilled lobopodians and radiodontans, and establishes their homology with setae borne on the outer appendage branches (i.e. exites) of upper-stem Euarthropoda.^n^nAdapted from characters 51, 56 and 68 from @Smith2015 and 54 in @Yang2015.'; TEXT CHARACTER=256 TEXT='The dorsal flaps of anomalocaridids and gilled lobopodians are considered as homologous with euarthropod exites.^nSee character 57 in @VanRoy2015. Treated as neomorphic.^n^nCharacter 57 in @Smith2015 and @Yang2015.'; TEXT CHARACTER=257 TEXT='@VanRoy2015 consider the setal blades to represent exites. Dorsal flaps are not always present in addition to the setal blades: the (ventral) flaps of amplectobeluids and Anomalocaris correspond to the euarthropod endopod. Dorsal flaps are considered to represent an elaboration of the setal blades, and thus treated as a neomorphic character.^n^nModified from character 21 in @VanRoy2015. Character 67 in @Smith2015 and 55 in @Yang2015.'; @@ -3178,7 +3178,7 @@ TEXT CHARACTER=394 TEXT='WTS60. Biphasic encompasses the multiple phases of priapulid larvae (e.g. Wennberg et al. 2009), also documented in Sirilorica (Peel et al. 2013)^nThe nematode larva is morphologically similar to the adult, but lacks reproductive functions.^nLarvae of Priapulopsis are poorly known but are understood to be similar to Priapulus (van der Land 1970), and are coded equivalently herein.'; TEXT CHARACTER=395 TEXT='Cf. WTS62'; TEXT CHARACTER=396 TEXT='Crenulated in the Higgins larva of loriciferans (Neves et al. 2016)^nNot in Halicryptus (Storch and Higgins 1991; Janssen et al. 2009)'; - TEXT CHARACTER=397 TEXT='WTS63. Lemburg (1999) recognised the presence of this character as a synapomorphy of (extant) Priapulida. However, it has since been demonstrated that the larvae of nematomorphs also possess six pharyngeal retractor muscles (Kristensen 2003; Müller et al. 2004). Long pharynx retractor muscles are also present in loriciferans (at least within the Nanaloricidae) (Neves et al. 2013).'; + TEXT CHARACTER=397 TEXT='WTS63. Lemburg (1999) recognized the presence of this character as a synapomorphy of (extant) Priapulida. However, it has since been demonstrated that the larvae of nematomorphs also possess six pharyngeal retractor muscles (Kristensen 2003; Müller et al. 2004). Long pharynx retractor muscles are also present in loriciferans (at least within the Nanaloricidae) (Neves et al. 2013).'; TEXT CHARACTER=398 TEXT='WTS65.^nA proposed synapomorphy of scalidophora, but also present in nematomorphs ^nPresent in Priapulid caudatus, Tubiluchis corallicola; absent in Kinorhyncha, Loricifera ; probably absent in Nectonema larvae yet present in adults (Schmidt-Rhaesa 1997a)'; TEXT CHARACTER=399 TEXT='Proposed as a synapomorphy between nematomorph larvae and loriciferan adults [@Kristensen1983], but not seemingly ^nNot reported in Halicryptus, Maccabeus or Priapulus (van der Land 1970; Por and Bromley 1974; Storch and Higgins 1991), but present in Tubiluchus (Higgins and Storch 1989)'; TEXT CHARACTER=400 TEXT='(~) inapplicable: proboscis and abdomen undivided^nThe larvae of Orstenoloricus (Maas and Waloszek 2009) possess a pair of spines at the anterior of the trunk. These may correspond to the anteroventral setae of Tenuiloricus (Neves and Kristensen 2014). Similar spines are present in Nanaloricidae and Pliciloricidae (Neves et al. 2016)^nThe tubuli present in the Halicryptus hatching larva are not paired, and disappear in the Higgins larval stage; paired spines are coded as absent in this taxon (Storch and Higgins 1991; Janssen et al. 2009) similar structures present in Tubiluchus (Kirsteuer 1976)'; @@ -5237,4 +5237,4 @@ BEGIN ASSUMPTIONS; TYPESET * UNTITLED = unord: 1 - 425; ENDBLOCK; - \ No newline at end of file + diff --git a/dev/benchmarks/ratchet_race.R b/dev/benchmarks/ratchet_race.R index 4e310a30b..695397526 100644 --- a/dev/benchmarks/ratchet_race.R +++ b/dev/benchmarks/ratchet_race.R @@ -2,7 +2,7 @@ # from an IDENTICAL Wagner start. Answers "does TNT reach the optimum in fewer # reweight cycles?" Unit = rearrangements (TS total_tbr_moves <-> TNT "Total # rearrangements examined"); score-parity = validity gate; seed distributions. -# Both optimise the same Fitch objective (inapplicable -> '?'). +# Both optimize the same Fitch objective (inapplicable -> '?'). # # Env (tbr_shared_start_lib.R reads TS_LIB / TNT_EXE / T0_DIR): SHARED_LIB (path # to tbr_shared_start_lib.R), RACE_DATASETS, RACE_SEEDS, RACE_WAGSEED, diff --git a/dev/benchmarks/stress_large_findings.md b/dev/benchmarks/stress_large_findings.md index 23d5dcdb0..7155a2bd2 100644 --- a/dev/benchmarks/stress_large_findings.md +++ b/dev/benchmarks/stress_large_findings.md @@ -19,7 +19,7 @@ Agent F, 2026-03-18 | `indirect_us` | **n^2.73** | — | | `clip_incr_us` | **n^1.50** | — | -Candidate count scales slightly super-quadratically (larger pruned subtrees give more valid regraft positions). Indirect scoring tracks candidates closely. Clip/incremental is sub-linear relative to candidates — incremental state amortises well. +Candidate count scales slightly super-quadratically (larger pruned subtrees give more valid regraft positions). Indirect scoring tracks candidates closely. Clip/incremental is sub-linear relative to candidates — incremental state amortizes well. Both exponents are consistent with the existing AGENTS.md note (~n^2.8 TBR cost). diff --git a/dev/benchmarks/t289e_stage4_hamilton.sh b/dev/benchmarks/t289e_stage4_hamilton.sh index 2d7a89a83..c7cf7e2a6 100644 --- a/dev/benchmarks/t289e_stage4_hamilton.sh +++ b/dev/benchmarks/t289e_stage4_hamilton.sh @@ -9,7 +9,7 @@ # T-289e: Prune-reinsert Stage 4 — multi-dataset validation # -# Validates that PR (c=5, d=5%, MISSING) benefit generalises across 5 large-tree +# Validates that PR (c=5, d=5%, MISSING) benefit generalizes across 5 large-tree # matrices (131-206 tips) and persists at 120s budget. # # Grid: 5 datasets × 2 configs × 2 budgets × 10 seeds = 200 runs diff --git a/dev/benchmarks/t364_FINDINGS.md b/dev/benchmarks/t364_FINDINGS.md index d86aa1fad..4ddaa01a5 100644 --- a/dev/benchmarks/t364_FINDINGS.md +++ b/dev/benchmarks/t364_FINDINGS.md @@ -41,7 +41,7 @@ budget, +3 to +66 steps. Absence from one MP tree was *not* the acceptance test split absent from one MP tree can sit in another tree of the same island at zero cost. **Stratified on which side holds dataset tip 0**, because that is what decides -whether arm 2 can fail at all: `build_constraint()` canonicalises the mask so tip 0 +whether arm 2 can fail at all: `build_constraint()` canonicalizes the mask so tip 0 is outside, so the canonical side is always the tip-0-excluding one, and the pre-T-384 mapping needs *that* side to be a rooted clade — which fails exactly when the root position lies inside it. diff --git a/dev/benchmarks/t364_analyze.R b/dev/benchmarks/t364_analyze.R index 58617aecd..258a2d2ff 100644 --- a/dev/benchmarks/t364_analyze.R +++ b/dev/benchmarks/t364_analyze.R @@ -253,7 +253,7 @@ if (length(tm_files)) { # ---- Budget exhaustion: the budget-INDEPENDENT statement of arm 2's cost. # A wall ratio partly reflects maxReplicates, which is a chosen setting; "did the -# search recognise convergence at all" does not. +# search recognize convergence at all" does not. ex <- M ex$exhausted <- ex$reps_done >= ex$maxrep cat("\n-- replicate-budget exhaustion (reps_done >= maxReplicates) --\n") diff --git a/dev/benchmarks/t364_common.R b/dev/benchmarks/t364_common.R index b524fcbfc..d99f6ef9f 100644 --- a/dev/benchmarks/t364_common.R +++ b/dev/benchmarks/t364_common.R @@ -4,7 +4,7 @@ # Deliberately free of TreeSearch calls so it can be sourced against any arm. # # ROOTING VOCABULARY, used throughout. A constraint split is an *unrooted* -# bipartition. build_constraint() (ts_constraint.cpp) canonicalises every mask +# bipartition. build_constraint() (ts_constraint.cpp) canonicalizes every mask # so that dataset tip 0 -- R tip names(dataset)[1] -- is OUTSIDE it, and the # pre-T-384 map_constraint_nodes() then demanded a node whose descendant tip set # EQUALS that canonical mask. So a tree can display the constraint yet still be @@ -169,7 +169,7 @@ tip_depths_from <- function(tree, focal) { # want_tip0 STRATIFIES on which SIDE of the bipartition holds dataset tip 0, and # that is the axis that decides whether arm 2 can express its pathology at all. # The reasoning, which is worth not re-deriving: -# * build_constraint() canonicalises the mask so tip 0 is OUTSIDE it, so the +# * build_constraint() canonicalizes the mask so tip 0 is OUTSIDE it, so the # canonical mask is ALWAYS the tip-0-excluding side -- which of the two sides # I hand in as the "1" group is immaterial, it gets flipped anyway. # * The pre-T-384 mapping needs that canonical side to be a rooted CLADE. It diff --git a/dev/benchmarks/t364_patch_findings.R b/dev/benchmarks/t364_patch_findings.R index e1670697d..4d11af4c9 100644 --- a/dev/benchmarks/t364_patch_findings.R +++ b/dev/benchmarks/t364_patch_findings.R @@ -10,7 +10,7 @@ raw <- paste(x, collapse = "\n") reps <- list( list( old = "Diagnostic signature worth reusing: every phase (TBR, XSS, ratchet) slower on a score-IDENTICAL trajectory — 226→220 and 233→223 at both builds — which means moves being rejected, not a different search being run.", - new = "Diagnostic signature — CORRECTED 2026-07-29, do not reuse the original form: the claim as filed was \"every phase (TBR, XSS, ratchet) slower on a score-IDENTICAL trajectory ⇒ moves being rejected\". Cumulative per-phase totals do rise (ratchet 1.20x, xss 1.50x, final_tbr 1.24x, arm 2 vs pre-fix), but those totals are summed over however many replicates ran, and a blocked arm runs MORE of them. Normalised PER REPLICATE, arm 2 is CHEAPER in nearly every phase — tbr 0.61x, wagner 0.70x, ratchet 0.83x, fuse 0.45x — which is what rejected moves must do: less work per replicate, not more. Measured from the engine's own `timings` attribute (microsecond resolution, so not a clock artefact), 13 cells x 3 arms. The replicate counts are the whole effect: median `reps_done` 17 pre-fix / **24 = budget exhausted** / 16 merged. **Correct signature: MORE replicates, each CHEAPER, identical score** — a blocked replicate never registers a hit on the best score, so the convergence rule never trips and the search burns its entire budget." + new = "Diagnostic signature — CORRECTED 2026-07-29, do not reuse the original form: the claim as filed was \"every phase (TBR, XSS, ratchet) slower on a score-IDENTICAL trajectory ⇒ moves being rejected\". Cumulative per-phase totals do rise (ratchet 1.20x, xss 1.50x, final_tbr 1.24x, arm 2 vs pre-fix), but those totals are summed over however many replicates ran, and a blocked arm runs MORE of them. Normalized PER REPLICATE, arm 2 is CHEAPER in nearly every phase — tbr 0.61x, wagner 0.70x, ratchet 0.83x, fuse 0.45x — which is what rejected moves must do: less work per replicate, not more. Measured from the engine's own `timings` attribute (microsecond resolution, so not a clock artefact), 13 cells x 3 arms. The replicate counts are the whole effect: median `reps_done` 17 pre-fix / **24 = budget exhausted** / 16 merged. **Correct signature: MORE replicates, each CHEAPER, identical score** — a blocked replicate never registers a hit on the best score, so the convergence rule never trips and the search burns its entire budget." ), list( old = "Not shown identical (different battery), but the mechanism matches: **re-measure before citing 1.43x.**", diff --git a/dev/benchmarks/t364_patch_findings4.R b/dev/benchmarks/t364_patch_findings4.R index 9b865a657..176257b2b 100644 --- a/dev/benchmarks/t364_patch_findings4.R +++ b/dev/benchmarks/t364_patch_findings4.R @@ -10,7 +10,7 @@ raw <- paste(readLines(f, warn = FALSE, encoding = "UTF-8"), collapse = "\n") old <- paste0( "Cumulative per-phase totals do rise (ratchet 1.20x, xss 1.50x, final_tbr 1.24x, ", "arm 2 vs pre-fix), but those totals are summed over however many replicates ran, ", - "and a blocked arm runs MORE of them. Normalised PER REPLICATE, arm 2 is CHEAPER ", + "and a blocked arm runs MORE of them. Normalized PER REPLICATE, arm 2 is CHEAPER ", "in nearly every phase \u2014 tbr 0.61x, wagner 0.70x, ratchet 0.83x, fuse 0.45x ", "\u2014 which is what rejected moves must do: less work per replicate, not more. ", "Measured from the engine's own `timings` attribute (microsecond resolution, so ", diff --git a/dev/benchmarks/tbr_grid.R b/dev/benchmarks/tbr_grid.R index c64c3f5a9..b7e7d8f17 100644 --- a/dev/benchmarks/tbr_grid.R +++ b/dev/benchmarks/tbr_grid.R @@ -36,7 +36,7 @@ buildStartLadder <- function(d) { w <- TreeSearch:::ts_random_wagner_tree(d$contrast, d$tip_data, d$weight, d$levels) ladder[[paste0("wagner", i)]] <- norm(asPhylo(w$edge, d)) } - # 1 partially-TBR-optimised tree (medium): a Wagner pushed ~15 accepted moves + # 1 partially-TBR-optimized tree (medium): a Wagner pushed ~15 accepted moves set.seed(3001) w <- TreeSearch:::ts_random_wagner_tree(d$contrast, d$tip_data, d$weight, d$levels) partial <- TsTbr(d, norm(asPhylo(w$edge, d)), seed = 3001, diff --git a/dev/benchmarks/tbr_missing_move_characterise.R b/dev/benchmarks/tbr_missing_move_characterise.R index 5df091af2..713927329 100644 --- a/dev/benchmarks/tbr_missing_move_characterise.R +++ b/dev/benchmarks/tbr_missing_move_characterise.R @@ -46,7 +46,7 @@ spNb <- TreeTools::as.Splits(nb, tipLabels = TipLabels(opt)) # Count splits in one but not the other (RF-style raw difference). inOpt <- apply(as.logical(spOpt), 1, function(r) paste(as.integer(r), collapse = "")) inNb <- apply(as.logical(spNb), 1, function(r) paste(as.integer(r), collapse = "")) -# Normalise complement (a split and its complement are the same bipartition). +# Normalize complement (a split and its complement are the same bipartition). canon <- function(s) { v <- as.integer(strsplit(s, "")[[1]]) if (v[1] == 1) paste(1L - v, collapse = "") else s } inOptC <- vapply(inOpt, canon, ""); inNbC <- vapply(inNb, canon, "") diff --git a/dev/benchmarks/tbr_shared_start_lib.R b/dev/benchmarks/tbr_shared_start_lib.R index f0bd4dbc4..97b41b340 100644 --- a/dev/benchmarks/tbr_shared_start_lib.R +++ b/dev/benchmarks/tbr_shared_start_lib.R @@ -4,7 +4,7 @@ # TNT 1.6 from IDENTICAL starting trees. Loaded by the pilot and the full # grid driver. See dev/plans/2026-06-18-tbr-shared-start.md for the design. # -# Both engines optimise the SAME Fitch objective because the matrices have +# Both engines optimize the SAME Fitch objective because the matrices have # inapplicable tokens replaced by '?'. Lengths are therefore directly # comparable (TreeLength vs TNT `length`). @@ -49,7 +49,7 @@ ToTntTree <- function(tr) { } # Run a TNT script (character vector of lines) in a fresh temp dir that -# already contains a data.tnt for `phy`. Returns sanitised stdout lines. +# already contains a data.tnt for `phy`. Returns sanitized stdout lines. # `files` is a named list of extra files to write into the working dir # (name = filename, value = character vector of lines). RunTnt <- function(phy, scriptLines, tag = "tnt", files = list()) { @@ -76,7 +76,7 @@ GrepNum <- function(out, pat) { # Run TNT TBR (bbreak) from `startTree`, save result, read it back, score in # R with TreeLength. Returns a one-row data.frame. # mulpars/hold : equal-tree buffer controls (Mode A: FALSE/1; Mode B: TRUE/1000) -# randclip : randomise clip order using rseed (the stochasticity knob) +# randclip : randomize clip order using rseed (the stochasticity knob) TntTbr <- function(d, startTree, seed, mulpars, hold, randclip = TRUE) { swap <- paste0("bbreak = tbr ", if (randclip) "randclip " else "norandclip ", diff --git a/dev/benchmarks/tbr_verify.R b/dev/benchmarks/tbr_verify.R index 26e6947fc..1b8bd19de 100644 --- a/dev/benchmarks/tbr_verify.R +++ b/dev/benchmarks/tbr_verify.R @@ -27,7 +27,7 @@ b2 <- TntTbr(d, wagTree, seed=2, mulpars=FALSE, hold=1, randclip=TRUE) cat("randclip seed1:", b1$final_len, " seed2:", b2$final_len, "\n") # (3) Sanity: TNT bbreak from the OPTIMAL T0 (1271) must NOT do RAS (stay <=1271). -cat("\n--- TNT bbreak from T0=1271 (must not re-randomise) ---\n") +cat("\n--- TNT bbreak from T0=1271 (must not re-randomize) ---\n") t0 <- ape::read.tree(file.path(T0_DIR, "Zanol2014.tre")) c1 <- TntTbr(d, t0, seed=1, mulpars=FALSE, hold=1, randclip=TRUE) cat("T0 start:", c1$start_len, " final:", c1$final_len, "\n") diff --git a/dev/benchmarks/vine-embedding-ls-search.md b/dev/benchmarks/vine-embedding-ls-search.md index 3790fb21a..c48fe65b7 100644 --- a/dev/benchmarks/vine-embedding-ls-search.md +++ b/dev/benchmarks/vine-embedding-ls-search.md @@ -78,7 +78,7 @@ Reliability splits by **difficulty, not size**. Untuned ε ~ U(0.05, 0.30) vs tu 730 s default vs 695 s seeded. **A better start tree buys nothing.** See the follow-up lead below. -## Does it transfer to parsimony? No — and the reason generalises +## Does it transfer to parsimony? No — and the reason generalizes Everything above is a least-squares objective, where the decoder and the objective are natively aligned (NJ *is* a distance method; the objective *is* distance fit), so the @@ -90,7 +90,7 @@ perturbed-NJ (pNJ) vs random-addition-sequence Wagner as the start generator in 16 sectors (8–37 tips) carved as clades from parsimony trees of Sansom2010, Wortley2006, OLeary1999, Griswold1999; K = 5 restarts; identical fixed TBR budget (`maxIter = 100`); equal weights. Excess = steps above the best score known for that -sector; diversity = mean pairwise normalised clustering-information distance across +sector; diversity = mean pairwise normalized clustering-information distance across restarts. | generator | start excess | post-TBR excess | best-of-5 | P(hit sector best) | start diversity | @@ -143,7 +143,7 @@ winning topology, rescored under NNLS, is **worse than the untouched NJ start** Mathematically expected — OLS admits negative branch lengths, so the search chases topologies that only fit well if lengths may go negative (on one target the OLS winner -reached RSS = 0.00000, an exact signed fit). Not a code defect; the function optimises +reached RSS = 0.00000, an exact signed fit). Not a code defect; the function optimizes what it is asked to. But the documented purpose is Lapointe & Cucumel average consensus, which is NNLS/Fitch–Margoliash, so a user switching to `"ols"` for speed silently gets a topology worse than doing nothing. Worth a sentence in `?LeastSquaresTree`. diff --git a/dev/benchmarks/vtune_pr_driver.R b/dev/benchmarks/vtune_pr_driver.R index 7f5d8a369..fe70442a9 100644 --- a/dev/benchmarks/vtune_pr_driver.R +++ b/dev/benchmarks/vtune_pr_driver.R @@ -27,7 +27,7 @@ for (ds_name in names(datasets)) { ds <- datasets[[ds_name]] cat(sprintf("Running %s ...\n", ds_name)) - # Maximise PR time share: high cycle count, no ratchet/drift/NNI-perturb, + # Maximize PR time share: high cycle count, no ratchet/drift/NNI-perturb, # enough time for ~100+ replicates worth of PR work. set.seed(7531) MaximizeParsimony( diff --git a/dev/briefings/briefing-multistate-profile.md b/dev/briefings/briefing-multistate-profile.md index 71141492b..e50127405 100644 --- a/dev/briefings/briefing-multistate-profile.md +++ b/dev/briefings/briefing-multistate-profile.md @@ -253,10 +253,10 @@ the existing "keep top 5" heuristic as an option. #' #' @param split Integer vector of token frequencies (sorted decreasing, #' singletons removed). -#' @param n_mc Number of Monte Carlo trees to sample (default 50000). +#' @param mcSamples Number of Monte Carlo trees to sample (default 50000). #' @return Named numeric vector of information content (bits) per step count. #' @keywords internal -.ApproxStepInformation <- function(split, n_mc = 50000L) { +.ApproxStepInformation <- function(split, mcSamples = 50000L) { k <- length(split) n <- sum(split) s_min <- k - 1L @@ -273,7 +273,7 @@ the existing "keep top 5" heuristic as an option. matrix(char_vec, ncol = 1, dimnames = list(labels, "c1")) ) mc_scores <- vapply( - seq_len(n_mc), + seq_len(mcSamples), function(i) RandomTreeScore(dat), double(1) ) @@ -297,7 +297,7 @@ the existing "keep top 5" heuristic as an option. mc_count <- sum(mc_scores == s) if (mc_count > 0) { # Direct empirical estimate - log_p[i] <- log(mc_count / n_mc) + log_p[i] <- log(mc_count / mcSamples) } else { # Normal extrapolation for unobserved step counts log_p[i] <- dnorm(s, mu_hat, sd_hat, log = TRUE) diff --git a/dev/expertise/profiling.md b/dev/expertise/profiling.md index ce83d0ae3..a58df162e 100644 --- a/dev/expertise/profiling.md +++ b/dev/expertise/profiling.md @@ -388,7 +388,7 @@ VTune top functions (TreeSearch.dll self, total 2.70 s; names via `nm`): **Conclusion:** per-candidate scoring is at the AVX2/compiler limit. Standard-Fitch is **bookkeeping- + strategy-bound**: per-clip O(n) work (postorder rebuild + incremental passes + edge/from_above/vroot construction ≈ 18%) and ratchet (63%) -are the levers — exactly what TNT minimises (Goloboff 1996). This corroborates +are the levers — exactly what TNT minimizes (Goloboff 1996). This corroborates `.positai/plans/2026-03-21-tnt-outperformance-analysis.md` (strategy > code). **Two build gotchas for VTune on this MinGW/Windows toolchain** (cost an hour): diff --git a/dev/expertise/red-team.md b/dev/expertise/red-team.md index 076be0b2b..c436c2b35 100644 --- a/dev/expertise/red-team.md +++ b/dev/expertise/red-team.md @@ -37,7 +37,7 @@ over breadth: one focused review that finds a real bug beats a broad "all green" | **"Missing `skip_on_cran()`" that's actually a deliberate Tier-1 file** | Check `tests/testing-strategy.md` tier lists FIRST. Tier-1 `test-ts-*` (`constraint-small`, `memory-layout`, `pool`, `simd`, `splits`, `rep-warning`, `start-tree`) are *intentionally* guardless (run on CRAN); only Tier-2 (any `test-ts-*` NOT in the Tier-1/Tier-3 lists) needs `skip_on_cran()`; Tier-3 stress/bench needs `skip_extended()`. Don't report a Tier-1 file for "missing skip" | 2026-06-16 area 8 (T-322 round refuted 6 false "missing skip" hits, confirmed 2 real: `impose-constraint`, `strategy`) | | Bare dataset symbol used without `data()` — usually NOT a bug | `inapplicable.phyData` etc. are lazy-loaded (`DESCRIPTION: LazyData: true` + `data/*.rda`), so a bare reference resolves the moment TreeSearch is attached. Only a real bug if the dataset genuinely isn't lazy-loaded | 2026-06-16 area 8 (TS-8-02 refuted) | | Cross-check test omits a production-required arg → tautology | Both sides then compute the same *non-production* formula, so the test always passes and can't catch a regression. `test-ts-wagner.R` NA+IW omitted `min_steps` (prod always passes `as.integer(MinimumLength(ds, compress=TRUE))`). Verify the test feeds the SAME args production does | T-322 (2026-06-16 area 8) | -| **Cross-engine "discrepancy" that's really mismatched defaults — NOT a bug** | Before filing "engine A ≠ engine B", confirm both compute the *same variant with the same defaults*. `TreeLength()` defaults `extended_iw = TRUE` → **XPIWE** (Goloboff 2014 Ext-3: `f=1+r·missing/obs`, `eff_k=k/f`, `phi=(1+eff_k)/(1+k)`, `Σ fit·w·phi`), but the kernel `ts_fitch_score(min_steps, concavity)` computes **plain IW** (no XPIWE args). They differ by design on NA datasets (irrational per-char values are the `phi`/`eff_k` scaling, not a min-steps bug). Production is consistent: `MaximizeParsimony` sets `useXpiwe <- isTRUE(extended_iw) && is.finite(concavity) && !useProfile` (`MaximizeParsimony.R:813`) so it optimises the same XPIWE `TreeLength` reports. **Verified equalities (Vinther2008):** `TreeLength(extended_iw=FALSE)` == kernel plain IW exactly; `MaximizeParsimony(conc=k)$score` == `TreeLength(extended_iw=TRUE)` rescore exactly. To compare engines, match `extended_iw` on both sides. | area-9 false signal, refuted 2026-06-16 | +| **Cross-engine "discrepancy" that's really mismatched defaults — NOT a bug** | Before filing "engine A ≠ engine B", confirm both compute the *same variant with the same defaults*. `TreeLength()` defaults `extended_iw = TRUE` → **XPIWE** (Goloboff 2014 Ext-3: `f=1+r·missing/obs`, `eff_k=k/f`, `phi=(1+eff_k)/(1+k)`, `Σ fit·w·phi`), but the kernel `ts_fitch_score(min_steps, concavity)` computes **plain IW** (no XPIWE args). They differ by design on NA datasets (irrational per-char values are the `phi`/`eff_k` scaling, not a min-steps bug). Production is consistent: `MaximizeParsimony` sets `useXpiwe <- isTRUE(extended_iw) && is.finite(concavity) && !useProfile` (`MaximizeParsimony.R:813`) so it optimizes the same XPIWE `TreeLength` reports. **Verified equalities (Vinther2008):** `TreeLength(extended_iw=FALSE)` == kernel plain IW exactly; `MaximizeParsimony(conc=k)$score` == `TreeLength(extended_iw=TRUE)` rescore exactly. To compare engines, match `extended_iw` on both sides. | area-9 false signal, refuted 2026-06-16 | | Arg-count mismatch in `TreeSearch-init.c`; `R_PosInf` in Rcpp defaults | After adding/removing Rcpp params; `R/RcppExports.R` after `compileAttributes()` | — | | Stale state arrays after a rejected move | Restore only covers the clip-to-root path; regraft-to-root nodes keep regrafted values — conservative (screening only) but real | T-235 (SPR); A 2026-03-19 | | Stale metadata after **equal-score / tabu** rejection | Constraint `map_constraint_nodes`/`compute_dfs_timestamps` re-synced on the `!accepted` path but not the tabu path | T-316 (possible P1) | diff --git a/dev/plans/2026-06-16-closing-the-tnt-gap.md b/dev/plans/2026-06-16-closing-the-tnt-gap.md index 22d73248b..43b8c4f46 100644 --- a/dev/plans/2026-06-16-closing-the-tnt-gap.md +++ b/dev/plans/2026-06-16-closing-the-tnt-gap.md @@ -175,7 +175,7 @@ targets wall-clock. - The cited StateSnapshot ~23% comes from a VTune doc that **predates T-261** (it *recommended* the fill removal T-261 then made) and likely T-300's incremental-SPR accept path — so the figure is **stale and the share has probably shrunk**. The remaining lever (selective `StateSnapshot` - save/restore) is intricate, correctness-critical surgery on the most-optimised code in the repo. + save/restore) is intricate, correctness-critical surgery on the most-optimized code in the repo. - **Decision:** it must be **re-profiled in a fresh `/profile` (VTune) round** to confirm it is still a meaningful hotspot *before* the surgery — not done on stale data at the tail of this round. Verification when pursued: behaviour-neutral via **candidate-identity** (a correct timing optimisation must leave @@ -223,7 +223,7 @@ fair reference; scores / rearrangement counts are. R0-vs-R12 wall-clock is compa ## Methodology guardrails -- **Optimise against candidates-per-improvement** (continuous, low-variance), not +- **Optimize against candidates-per-improvement** (continuous, low-variance), not score-at-fixed-time (±2-step lottery on a small panel). - **Authoritative wall-clock**: Hamilton 64-bit Linux TNT (matches the on-disk `t264`/`t249` reference scores). The local `tnt.exe` is **32-bit** (PE32/i386) — diff --git a/dev/plans/2026-06-17-tnt-algorithm-audit.md b/dev/plans/2026-06-17-tnt-algorithm-audit.md index 6e48f5a27..9860cb318 100644 --- a/dev/plans/2026-06-17-tnt-algorithm-audit.md +++ b/dev/plans/2026-06-17-tnt-algorithm-audit.md @@ -261,7 +261,7 @@ documented +1..+3 EW gap, so this install behaves as the gap reports describe. - Hypothesis: even rebuild-alone is null from a converged T0, but rebuild is a *precondition* for D1 — you cannot exploit a floated HTU without re-resolving the sector. With the HTU frozen, `rasStarts=3` is null (already shown). With the HTU - floated (D1), `rasStarts>=3` becomes necessary to realise the joint move, and the + floated (D1), `rasStarts>=3` becomes necessary to realize the joint move, and the "+3 extra starts on score disagreement" (`combstarts`) matters. So D2 is *coupled* to D1: the only meaningful test of D2 is rasStarts>=3 *with* a floating HTU. - Experiment: factorial on `bench_sectorial_shared.R` — {HTU frozen, HTU floated} × diff --git a/dev/plans/2026-06-18-tbr-shared-start.md b/dev/plans/2026-06-18-tbr-shared-start.md index 874d0293b..794d9cd0a 100644 --- a/dev/plans/2026-06-18-tbr-shared-start.md +++ b/dev/plans/2026-06-18-tbr-shared-start.md @@ -33,7 +33,7 @@ engines and running TBR to convergence. (The Wagner half is a separate task.) ## Method & comparability controls - Datasets are EW-Fitch-converted (inapplicable tokens → `?`), so **both engines - optimise the identical Fitch objective**; `TreeLength` (TS) and TNT `length` + optimize the identical Fitch objective**; `TreeLength` (TS) and TNT `length` are directly comparable. Verified: T0 round-trips at 1271 in both. - **TS entry point:** `TreeSearch:::ts_tbr_diagnostics(edge, ...)` — runs TBR to convergence from a warm-start edge matrix, returns final score + per-pass @@ -41,7 +41,7 @@ engines and running TBR to convergence. (The Wagner half is a separate task.) optimum; `acceptEqual=TRUE` = single-tree plateau-walk. - **TNT entry point:** `bbreak = tbr [no]randclip [no]mulpars;` with `tread` of the shared start tree and `rseed N`. `bbreak` swaps the *in-memory* tree — it - does **not** re-randomise (verified: bbreak from T0=1271 stays 1271). + does **not** re-randomize (verified: bbreak from T0=1271 stays 1271). - **Two modes:** - **Mode A — strict single tree:** TS `acceptEqual=F`; TNT `nomulpars hold 1`. - **Mode B — buffer / plateau:** TS `acceptEqual=T`; TNT `mulpars hold 1000`. @@ -69,7 +69,7 @@ engines and running TBR to convergence. (The Wagner half is a separate task.) Final length over **6 seeds** per (start tree × engine), from the IDENTICAL shared start. Six start trees per dataset spanning a quality ladder (two random -topologies, two RAS Wagner, one partially-TBR-optimised, one near-optimal +topologies, two RAS Wagner, one partially-TBR-optimized, one near-optimal anchor). `gap` = median(TS) − median(TNT). Raw rows: `dev/benchmarks/tbr_results/tbr_grid_raw.csv`; shared starts: `_starts.nwk`. diff --git a/dev/plans/2026-06-19-component-isolation-profiling.md b/dev/plans/2026-06-19-component-isolation-profiling.md index 30f945212..4764f1c82 100644 --- a/dev/plans/2026-06-19-component-isolation-profiling.md +++ b/dev/plans/2026-06-19-component-isolation-profiling.md @@ -169,7 +169,7 @@ mixed-state case col-reduce helps). Default stays `0L`. Recorded in the switches ## Why this reframe -The previous round optimised "the expensive phase of the current recipe" +The previous round optimized "the expensive phase of the current recipe" (ratchet, ~60%). That produced a real recipe win — `ratchetCycles` 12→6 is ~20–38% wall with no quality loss (findings T-P5d) — **but recipe tuning only reshuffles component *proportions*; it cannot address the core belief that TNT diff --git a/dev/plans/2026-06-19-lazy-precompute-incremental-length.md b/dev/plans/2026-06-19-lazy-precompute-incremental-length.md index d2d00ed0f..0a7e98052 100644 --- a/dev/plans/2026-06-19-lazy-precompute-incremental-length.md +++ b/dev/plans/2026-06-19-lazy-precompute-incremental-length.md @@ -156,7 +156,7 @@ Two viable designs — pick the simpler that M1 supports: - **On-demand single-pass:** the scorer requests block `b` of candidate `D`; a per-clip "is block b's full-tree up-pass done?" bitmap triggers a one-time O(N) up-pass+combine for block `b` across all nodes the first time any candidate needs - it. Subsequent candidates reuse it. (This is the clean version — amortises each + it. Subsequent candidates reuse it. (This is the clean version — amortizes each consumed block's up-pass once per clip.) The on-demand single-pass is preferred: it makes the saving exactly "skip the @@ -290,7 +290,7 @@ cpp-search), I created my OWN worktree: blocks (the up-pass + the edge-set pass). Called once per clip at `src/ts_tbr.cpp:1504` (gated `use_directional = !has_na`). - **THE CRUX (kills design (a) a priori, pending measurement):** the SPR - candidate loop at `src/ts_tbr.cpp:1537` initialises `best_candidate = HUGE_VAL` + candidate loop at `src/ts_tbr.cpp:1537` initializes `best_candidate = HUGE_VAL` (ts_tbr.cpp:1529), so the **FIRST SPR candidate every clip is scored with `cutoff = INT_MAX`** (ts_tbr.cpp:1578-1580) → it never bails → it walks ALL blocks. The TBR reroot loop (ts_tbr.cpp:1819) and all later SPR candidates are @@ -329,7 +329,7 @@ instrumentation (M1). **Advisor (pre-build):** crux holds; gate on the RAW union (the spec's literal metric); bounded-union is a diagnostic only; do NOT build M2 even if bounded-union is small (a bit-identical-trajectory lazy prototype still eats the full -first-candidate scan); minimise build surface (avoid RcppExports/init.c codegen); +first-candidate scan); minimize build surface (avoid RcppExports/init.c codegen); run Zanol first; document the "seed best_candidate" spin-off but don't implement. **Built (worktree `claude/lazy-precompute-m46`, off cpp-search da0f203f):** diff --git a/dev/plans/2026-06-19-lever3-incremental-edgeset.md b/dev/plans/2026-06-19-lever3-incremental-edgeset.md index 6d09da38e..7f70d4ce2 100644 --- a/dev/plans/2026-06-19-lever3-incremental-edgeset.md +++ b/dev/plans/2026-06-19-lever3-incremental-edgeset.md @@ -194,7 +194,7 @@ returns on ascend): - **Ascend** child c → parent p: inverse — re-remove that chunk. - **Suppress-node** (clip parent nx removed, sibling ns joined to grandparent nz): the single most bug-prone step — `up[]` at ns's new position and everything it - newly exposes must be refreshed (advisor). The amortised total over a full DFS + newly exposes must be refreshed (advisor). The amortized total over a full DFS is O(N) (each node exposed/hidden O(1) times). **Gate (hard, non-negotiable):** under `-DNDEBUG`-off, assert the incrementally- diff --git a/dev/plans/2026-06-19-na-directional-feasibility.md b/dev/plans/2026-06-19-na-directional-feasibility.md index 80427697d..83d72cc1d 100644 --- a/dev/plans/2026-06-19-na-directional-feasibility.md +++ b/dev/plans/2026-06-19-na-directional-feasibility.md @@ -77,7 +77,7 @@ richer than a single set: - the region term *can* be charged locally in a **directed/rooted** formulation (charge +1 at each applicable node whose parent is inapplicable = one charge per region's top), which is fixed-size; -- combine at a join would re-optimise the boundary in O(table²). +- combine at a join would re-optimize the boundary in O(table²). **But three things make this research-grade, not a quick win:** diff --git a/dev/plans/2026-06-22-pairwise-fuse-strategy-briefing.md b/dev/plans/2026-06-22-pairwise-fuse-strategy-briefing.md index f95cd6f6b..d9870a20e 100644 --- a/dev/plans/2026-06-22-pairwise-fuse-strategy-briefing.md +++ b/dev/plans/2026-06-22-pairwise-fuse-strategy-briefing.md @@ -14,7 +14,7 @@ anything you already measure. TreeSearch's tree-fusing (`tree_fuse`) recombines clades between pool trees, but in the production driven search it does **zero productive recombination**: the -recipient is hard-wired to `pool.best()` (the already-optimised global best) and +recipient is hard-wired to `pool.best()` (the already-optimized global best) and acceptance requires a *strict* improvement, so a clade swap from a suboptimal donor can never strictly improve it (root cause: `dev/plans/2026-06-22-tree-fuse-zero-exchanges.md`). The **pairwise-fuse** diff --git a/dev/plans/2026-06-22-tree-fuse-zero-exchanges.md b/dev/plans/2026-06-22-tree-fuse-zero-exchanges.md index 122e97f10..b72742e3c 100644 --- a/dev/plans/2026-06-22-tree-fuse-zero-exchanges.md +++ b/dev/plans/2026-06-22-tree-fuse-zero-exchanges.md @@ -13,7 +13,7 @@ all pool sizes 2..10. No crash; not size-specific. preset (`fuseAcceptEqual = FALSE`) requires a **strict score improvement** (`new_score < score`, `ts_fuse.cpp:466`/`481-484`). On this dataset class a clade swap **never strictly improves** the recipient, because the recipient is always -`pool.best().tree` (the already-TBR-optimised global best) and the donors are +`pool.best().tree` (the already-TBR-optimized global best) and the donors are within `poolSuboptimal` steps. So every trial lands in one of three rejected buckets and `n_exchanges` stays 0. diff --git a/dev/plans/2026-07-14-lever6-incremental-edgeset-land.md b/dev/plans/2026-07-14-lever6-incremental-edgeset-land.md index a4deab6fc..b72d944af 100644 --- a/dev/plans/2026-07-14-lever6-incremental-edgeset-land.md +++ b/dev/plans/2026-07-14-lever6-incremental-edgeset-land.md @@ -38,11 +38,11 @@ That doc is the spec; summary: refresh `up[]`/`edge_set[]` at ns's new position + everything newly exposed. - Re-materialise only the O(changed) `edge_set` entries per move; score this clip's regrafts immediately in place; do **NOT** cache all clips' arrays (that is O(n²) memory/copy and kills the win). -- Amortised **O(N) total** directional work per pass (vs O(N²)); each node exposed/hidden O(1)× over a +- Amortized **O(N) total** directional work per pass (vs O(N²)); each node exposed/hidden O(1)× over a full DFS. ## 3. Obstacles (from the design doc — plan for them) -- **Abandon restore-between-clips + the clip SHUFFLE.** `order_clips` randomises clip order to seed tight +- **Abandon restore-between-clips + the clip SHUFFLE.** `order_clips` randomizes clip order to seed tight cutoffs early; DFS order is the opposite → cutoffs may loosen → MORE candidates scored. The B-vs-A kill-test measured **~1.2× candidate cost** (acceptable; inherited by both B and C). - **Feature interactions:** gate the new path to plain EW (`use_directional && !has_na` + no diff --git a/dev/plans/2026-07-14-lever7-scorer-monomorphization.md b/dev/plans/2026-07-14-lever7-scorer-monomorphization.md index a64144598..b729d18d9 100644 --- a/dev/plans/2026-07-14-lever7-scorer-monomorphization.md +++ b/dev/plans/2026-07-14-lever7-scorer-monomorphization.md @@ -7,7 +7,7 @@ Land an EXACT (byte-identical) per-candidate speedup in the TBR scorer by lifting per-candidate **criterion-flavour dispatch out of the hot loop** — instantiate one monomorphic candidate-scan loop per flavour, chosen once at dispatch, so the dead branches compile away and the compiler can inline / -pick a specialised kernel. +pick a specialized kernel. - **Measured ceiling:** a *runtime* branch-strip alone ≈ **5% whole-search**; the **template** version additionally unlocks a weight-aware "flat" kernel worth **~17% of the SPR scan** on unit-weight data @@ -26,8 +26,8 @@ pick a specialised kernel. toggle)**. A first A/B falsely read ~0% because the fast path never fired (the collapsed-vector is always sized, so the `!use_collapsed` gate was BASE-vs-BASE) — a fire-counter + a deliberately-wrong mode caught it. **Replicate that positive control.** -- Prototype to generalise from: the `TS_EW_STRIP` runtime probe (default-OFF, uncommitted) in - `src/ts_tbr.cpp` on that worktree branch — a specialised branch-free plain-EW SPR loop, proven +- Prototype to generalize from: the `TS_EW_STRIP` runtime probe (default-OFF, uncommitted) in + `src/ts_tbr.cpp` on that worktree branch — a specialized branch-free plain-EW SPR loop, proven byte-identical (BASE==STRIP==307). #7 turns this from a runtime toggle into compile-time specialisation. ## 2. The lever (design — this is the user's framing, and it's the right one) @@ -58,7 +58,7 @@ Monomorphization is a pure reorganization ⇒ final `score` AND `attr(result,"ca per-pass `n_candidates_evaluated` MUST be **BYTE-IDENTICAL** vs baseline on Wortley2006 / Zhu2013 / Zanol2014 × ≥2 seeds (EW: gaps `-`→`?`), via BOTH `TreeSearch:::ts_tbr_diagnostics` (kernel-level) AND full `MaximizeParsimony` (exercises the ratchet `use_flat=false` + `sector_mask` regimes the kernel -oracle can't reach). **Replicate the positive control:** a fire-counter proving the specialised path +oracle can't reach). **Replicate the positive control:** a fire-counter proving the specialized path actually executed, plus a deliberately-wrong instantiation that DIVERGES — otherwise a no-op passes the gate falsely (the agent's ~0% trap). Any drift = bug: STOP, do not proceed to timing. diff --git a/dev/plans/2026-07-29-t374b-xform-rooting-policy.md b/dev/plans/2026-07-29-t374b-xform-rooting-policy.md index aebf3333d..9eaedfe94 100644 --- a/dev/plans/2026-07-29-t374b-xform-rooting-policy.md +++ b/dev/plans/2026-07-29-t374b-xform-rooting-policy.md @@ -10,11 +10,11 @@ ## Recommendation, up front -**Option 3: document XFORM as an approximation whose reported score is a rooting-dependent upper bound, canonicalise the rooting at the single place the user-visible discrepancy is produced, and do not pin `sankoff_forced_root`.** +**Option 3: document XFORM as an approximation whose reported score is a rooting-dependent upper bound, canonicalize the rooting at the single place the user-visible discrepancy is produced, and do not pin `sankoff_forced_root`.** Concretely, and in priority order: -1. **Fix the reported-score/`TreeLength()` discrepancy, which is the actual P1 user-visible defect**, by making the report path and `TreeLength()` agree on one rooting — not by making the objective rooted. This is the lead evidence (Q-E): the quantity the search already optimises is a **valid upper bound** on the well-defined unrooted objective (min over rootings), it is **tight for 87–98% of rootings**, mean overstatement 0.02–0.17 steps, and the worst case is bounded by `nSec` per hierarchy block (Q-B). So agreement at the boundary is cheap and provably close, and nothing about the search needs to change to get it. +1. **Fix the reported-score/`TreeLength()` discrepancy, which is the actual P1 user-visible defect**, by making the report path and `TreeLength()` agree on one rooting — not by making the objective rooted. This is the lead evidence (Q-E): the quantity the search already optimizes is a **valid upper bound** on the well-defined unrooted objective (min over rootings), it is **tight for 87–98% of rootings**, mean overstatement 0.02–0.17 steps, and the worst case is bounded by `nSec` per hierarchy block (Q-B). So agreement at the boundary is cheap and provably close, and nothing about the search needs to change to get it. 2. **Do not set `forced_root_state = 0` as a standalone change.** This was the attractive cheap fix, and it is **incoherent on its own** — not merely suboptimal. Pinning the root *state* makes the root meaningful while leaving the root *position* arbitrary and, at four sites, actively moving (Q3). Measured: the pinned-state criterion's value varies across root positions on **84–117/120** random 9-tip topologies, with spread up to 5 — so the one-liner does not remove root-sensitivity from a rerooting pipeline, it relocates and enlarges it. **Read that number correctly** (Q-C): `forced_root_state = 0` defines a *different, explicitly rooted* criterion, perfectly well-defined at any fixed rooting, so this is not evidence that a rooted criterion is wrong. It is evidence that pinning the state is only meaningful **together with** pinning the position — i.e. it is Option 2, not a one-line fix. 3. **Do not "rethink TBR", and do not run the rooting-pinned A/B.** See Q3 and §"The A/B" — TBR's fragment reroot is not in conflict with a pinned root, the whole-tree rerooting sites are a short enumerable list, and the already-filed **T-377** is a first-order defect upstream that makes any second-order rooting measurement uninterpretable (see the "Blocker" section). 4. **Document XFORM as rooting-sensitive** in `?MaximizeParsimony` and `?RecodeHierarchy`, stating the bound. @@ -67,7 +67,7 @@ Asymmetric whenever `nSec ≥ 1`; **symmetric when `nSec = 0`** (gain = loss = 1 **But "the maths is asymmetric" is not "the user's tree is rooted".** The engine's rooting is an *artefact*, not a hypothesis: - `MaximizeParsimony` supplies `TreeTools::RandomTree(nTip, root = TRUE)` when no start tree is given (`R/MaximizeParsimony.R:1219`) — the root position is whatever the RNG produced. -- `TreeLength()` scores at whatever rooting the user's `phylo` happens to carry (`R/tree_length.R:198-204`, `:347-354`); it does not canonicalise. +- `TreeLength()` scores at whatever rooting the user's `phylo` happens to carry (`R/tree_length.R:198-204`, `:347-354`); it does not canonicalize. - Wagner addition, sector search, fusing and `ts_collapse_pool` all move the root freely, on the stated premise that length is root-invariant (`ts_tbr.cpp:123-124`, `:2999`, and the already-recorded `ts_collapse_pool` comment). - `forced_root_state` is `-1` for every block (`R/recode_hierarchy.R:176`), i.e. "min over root states" — a deliberate attempt to *neutralise* the rooting rather than to honour it. @@ -105,9 +105,9 @@ Three code comments assert root-invariance and are false for HSJ/XFORM: `ts_tbr. ## Q4. If unrooted: is min-over-root-states enough? -**No.** `ts_sankoff.cpp:74-87` minimises over the root's own *state* at a fixed root *position*. That removes one degree of freedom out of two; the position remains, and the position is what varies. +**No.** `ts_sankoff.cpp:74-87` minimizes over the root's own *state* at a fixed root *position*. That removes one degree of freedom out of two; the position remains, and the position is what varies. -What *would* restore invariance is **min over rootings as well as root states**: for an unrooted topology, minimise the total over all `2n − 3` placements of the root and all root states. This is a well-defined unrooted objective, and the oracle shows the current single-rooting score is always an **upper bound** on it (Q-E) — an overstatement, never an understatement. That matters: it means the existing kernel is a *sound but sometimes loose* evaluator of the correct unrooted objective, and it is tight for 87–98% of rootings. +What *would* restore invariance is **min over rootings as well as root states**: for an unrooted topology, minimize the total over all `2n − 3` placements of the root and all root states. This is a well-defined unrooted objective, and the oracle shows the current single-rooting score is always an **upper bound** on it (Q-E) — an overstatement, never an understatement. That matters: it means the existing kernel is a *sound but sometimes loose* evaluator of the correct unrooted objective, and it is tight for 87–98% of rootings. For an asymmetric matrix that satisfies the triangle inequality this is the standard construction (equivalently: attach a hypothetical ancestor of unspecified state, and let the DP place it). Its cost is a factor `2n − 3` on the Sankoff term, which is why it belongs on the **report path only**, not in the search loop. @@ -115,7 +115,7 @@ For an asymmetric matrix that satisfies the triangle inequality this is the stan Yes, and it is the recommendation. The **actual P1 defect** in T-374 is not "the objective is rooting-dependent" — it is "`MaximizeParsimony` reports a best score that `TreeLength()` of its own returned trees does not reproduce", and "4 of 6 trees in one MPT set do not share a score under a common rooting". Both are *reporting* defects, and both are fixed by making one rooting authoritative at the boundary: -- Have the report path and `TreeLength()` agree — either both canonicalise (tip-0 rooting, as `ts_collapse_pool` already does) or, better, both evaluate the min-over-rootings objective for the returned pool only (`|pool| × (2n − 3)` Sankoff evaluations, negligible against a search). +- Have the report path and `TreeLength()` agree — either both canonicalize (tip-0 rooting, as `ts_collapse_pool` already does) or, better, both evaluate the min-over-rootings objective for the returned pool only (`|pool| × (2n − 3)` Sankoff evaluations, negligible against a search). - The min-over-rootings variant additionally makes the MPT set internally consistent by construction, since the score no longer depends on the representation. - Document the bound: for a single block the search-time score can exceed the min-over-rootings objective by at most `nSec` (measured, below), and does so for 2–13% of rootings. - Do **not** error out on unrooted input. XFORM has no root to demand, and requiring one would ask users to supply a hypothesis the criterion does not actually use. @@ -171,7 +171,7 @@ total = Σ_edges s(u,v) − Σ_{internal} f(u) − f(root) + Σ_tips f **Option 2 — full rooted criterion: require a rooted input tree, pin state and position, gate the four whole-tree reroot sites.** *Rejected on cost/benefit, not feasibility.* Q3 shows the mechanical cost is moderate. But it (i) asks users for a root the criterion does not need, (ii) makes the root position an unsearched nuisance parameter whose choice changes the answer by up to `Σ nSec_b`, (iii) shrinks the reachable move set at the four gated sites — including fusing, whose whole value is topological diversity, and `ts_collapse_pool`, whose canonicalisation the collapse logic depends on, and (iv) delivers no user-visible benefit over Option 3, because Option 3 already removes the reported discrepancy. Reconsider only if a user presents a hierarchy where the root state is genuinely known *and* the tree genuinely rooted. -**Option 3 — recommended: unrooted reading, canonicalised reporting, documented bound.** Keep `forced_root_state = -1`. Keep the search as-is (an upper-bound evaluator). Make `MaximizeParsimony`'s reported score and `TreeLength()` agree on one rooting; prefer min-over-rootings on the returned pool, which additionally makes the MPT set self-consistent. Document XFORM as rooting-sensitive with the `nSec`-per-block bound. Fix the three false root-invariance comments as part of whatever touches those files. +**Option 3 — recommended: unrooted reading, canonicalized reporting, documented bound.** Keep `forced_root_state = -1`. Keep the search as-is (an upper-bound evaluator). Make `MaximizeParsimony`'s reported score and `TreeLength()` agree on one rooting; prefer min-over-rootings on the returned pool, which additionally makes the MPT set self-consistent. Document XFORM as rooting-sensitive with the `nSec`-per-block bound. Fix the three false root-invariance comments as part of whatever touches those files. **Option 4 — implement min-over-rootings inside the search.** Correct but `(2n−3)×` on the Sankoff term. Not warranted while the Sankoff term is absent from the candidate screen (blocker). Revisit only after that is fixed and only if measurement shows the loose bound misleads the search. diff --git a/dev/plans/2026-08-03-concave-branch-criterion-is-aml.md b/dev/plans/2026-08-03-concave-branch-criterion-is-aml.md index 8cf1ffe4e..aeb56b04c 100644 --- a/dev/plans/2026-08-03-concave-branch-criterion-is-aml.md +++ b/dev/plans/2026-08-03-concave-branch-criterion-is-aml.md @@ -50,10 +50,10 @@ which also fails. Prior art for the general programme: Wheeler & Varón 2025, ## 1. The criterion Give every edge its own free substitution probability under a symmetric -$m$-state (Mk) process, maximise the likelihood over topology, edge +$m$-state (Mk) process, maximize the likelihood over topology, edge probabilities **and ancestral states**, and take negative log-likelihood: -$$\text{minimise}\quad \Phi(T) \;=\; \min_{\text{reconstruction}} \sum_{e \in E(T)} f_m(d_e)$$ +$$\text{minimize}\quad \Phi(T) \;=\; \min_{\text{reconstruction}} \sum_{e \in E(T)} f_m(d_e)$$ where $d_e$ is the number of characters changing on edge $e$, $k$ the number of characters, and @@ -61,7 +61,7 @@ characters, and $$f_m(d) \;=\; \begin{cases} k\,H_b\!\left(\dfrac{d}{k}\right) + d\log(m-1), & \dfrac{d}{k} \le \dfrac{m-1}{m}\\[2ex] k\log m, & \text{otherwise}\end{cases}$$ with $H_b$ the binary entropy in nats. Verified: $f_m(d)$ equals the negated -maximised log-likelihood of a symmetric Mk edge whose change probability is +maximized log-likelihood of a symmetric Mk edge whose change probability is constrained to the identifiable range $p \le (m-1)/m$, to $\mathbf{5.7\times10^{-14}}$ across 30 $(m,k,d)$ cells. @@ -153,7 +153,7 @@ one, so MM freezes and can never move a change onto an empty edge. Krichevsky–Trofimov / Jeffreys smoothing $\hat p = (d+\frac12)/(k+1)$ removes the singularity ($\lambda(0)=\log(2k+1)$), is the proper MDL two-part code rather than a hack, and adds a $\frac12\log k$ per-edge parameter cost that correctly -penalises resolution. **This is not merely an algorithmic detail — it is the +penalizes resolution. **This is not merely an algorithmic detail — it is the mechanism of the published shrinkage pathology** (§2), so the smoothed variant was tested as a candidate repair (§5). @@ -232,7 +232,7 @@ not a package feature. The chain runs: NS-2026 information metric → chance-corrected agreement → weighted parsimony → concave-branch penalty → **AML** → NP-hard *and* statistically inconsistent → and the principled repair for inconsistency is to -*integrate out* ancestral states rather than maximise over them, which lands on +*integrate out* ancestral states rather than maximize over them, which lands on ordinary Mk ML, already implemented consistently elsewhere. The incidental- parameters problem (Neyman–Scott 1948) is structural here: the number of ancestral states grows with $k$, so no amount of smoothing makes joint diff --git a/dev/plans/frac-quart-weight-agreement.md b/dev/plans/frac-quart-weight-agreement.md index 7fe19b3d4..b0efc7ec4 100644 --- a/dev/plans/frac-quart-weight-agreement.md +++ b/dev/plans/frac-quart-weight-agreement.md @@ -23,7 +23,7 @@ verified by brute-force enumeration / GF(2) rank (scratch scripts, this session) - **`weight` (TRUE/FALSE) is retained and orthogonal to `unit`.** It controls the *pooling amount*, not the per-pair quality. See §5. - **`return` (edge/char/tree) retained;** it selects which unit is "reported" and - therefore which weight normalises. See §4–5. + therefore which weight normalizes. See §4–5. - This slots into the package's existing **quality × amount** decomposition (`ConcordanceTable()` / `QACol()`): quality = per-pair coverage ratio, amount = N&L fractional weight (trit content). @@ -264,7 +264,7 @@ char FQ(c) = Σ_k M(k,c)·Q(k,c) / Σ_k M(k,c) edge/char × weight × binary/multistate (`test-Concordance.R`), and `normalize = FALSE` is byte-identical to the published raw measure. The flat `1/3` baseline stays rejected (it does not adapt to marginals; the - fixed-marginal null generalises it). + fixed-marginal null generalizes it). - **`M = min(W_c,W_k)`** (settled): the pooling amount is the *shared* information, so it must be symmetric between character and split (both `return`s pool by the same `M`); `M = W_k` would make the char return weight a diff --git a/dev/profiling/drivers/fitch-tnt.R b/dev/profiling/drivers/fitch-tnt.R index 4105b34f5..7bc20205b 100644 --- a/dev/profiling/drivers/fitch-tnt.R +++ b/dev/profiling/drivers/fitch-tnt.R @@ -1,7 +1,7 @@ # Standard-Fitch TNT-parity profiling driver — Area #5 # GOAL: profile the *standard Fitch* path that the TNT-parity benchmark uses. # TNT-parity replaces inapplicable "-" with missing "?" so both engines -# optimise the identical Fitch objective (no Brazeau-Guillerme-Smith NA +# optimize the identical Fitch objective (no Brazeau-Guillerme-Smith NA # handling). Removing the "-" level makes the C++ engine take has_na=FALSE # and use the flat / 4-wide (T-245) kernels — a code path NEVER profiled # before (all prior rounds used the NA path on raw inapplicable.phyData). diff --git a/dev/profiling/drivers/framing.R b/dev/profiling/drivers/framing.R index 7b16e2e3e..02c066b51 100644 --- a/dev/profiling/drivers/framing.R +++ b/dev/profiling/drivers/framing.R @@ -68,7 +68,7 @@ run_tnt <- function(phy, seed, timeout_s) { # as a run-file argument. `tnt run.run;` makes 64-bit TNT try to EXECUTE the # filename as a command ("Must read data before generating random trees") and # `proc` never runs; piping commands to stdin is the canonical non-interactive - # mode. No curses is initialised when stdin is not a TTY, so TERM=dumb (set in + # mode. No curses is initialized when stdin is not a TTY, so TERM=dumb (set in # the sbatch) suffices — the earlier TERM=xterm / `log`-only theory was wrong. out <- tryCatch( system2(TNT_EXE, stdin = basename(runfile), diff --git a/dev/profiling/drivers/ham_step0_rerun.sh b/dev/profiling/drivers/ham_step0_rerun.sh index 8ee2630fd..a33173166 100644 --- a/dev/profiling/drivers/ham_step0_rerun.sh +++ b/dev/profiling/drivers/ham_step0_rerun.sh @@ -13,7 +13,7 @@ export OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 # THE FIX (verified 2026-06-19): the earlier "curses needs TERM" theory was WRONG. # 64-bit TNT starts fine headless; it died because `tnt run.run;` makes TNT try to # EXECUTE the filename as a command and `proc` never runs. framing.R now feeds the -# script via STDIN (the canonical non-interactive mode); no curses is initialised +# script via STDIN (the canonical non-interactive mode); no curses is initialized # when stdin is not a TTY, so TERM=dumb suffices. Results land in the `log` file. export TERM=dumb LIB=/nobackup/$USER/TreeSearch/lib diff --git a/dev/profiling/drivers/sector-rss.R b/dev/profiling/drivers/sector-rss.R index 788a94ae5..936c0bd13 100644 --- a/dev/profiling/drivers/sector-rss.R +++ b/dev/profiling/drivers/sector-rss.R @@ -5,7 +5,7 @@ # (dev/plans/2026-06-19-component-isolation-profiling.md) and the advisor: # * Crank rssPicks HIGH and use FEW calls so the trailing global TBR and the # per-call make_dataset/init_from_edge marshaling (a DRIVER ARTIFACT) are -# amortised — otherwise the profile is mostly TBR, not sectorial. +# amortized — otherwise the profile is mostly TBR, not sectorial. # * EW-Fitch only ('-' -> '?'); the NA path is owned by another workstream. # # Start tree: a Wagner addition tree (non-optimal) so many sectors improve and diff --git a/dev/profiling/exactness-gate.R b/dev/profiling/exactness-gate.R index eb5c97483..2480227f3 100644 --- a/dev/profiling/exactness-gate.R +++ b/dev/profiling/exactness-gate.R @@ -151,7 +151,7 @@ fitchUpEngine <- function(tree, prelim) { # Regraft subtree S onto the edge above node D of `base` (a new node w splits # that edge; w's children = D and S's root). Explicit edge-matrix surgery, then -# normalise numbering via write/read newick. Branch lengths dropped (Fitch is +# normalize numbering via write/read newick. Branch lengths dropped (Fitch is # length-invariant). Returns a phylo. regraft <- function(base, S, D) { nTb <- length(base$tip.label); nTs <- length(S$tip.label) diff --git a/dev/profiling/findings.md b/dev/profiling/findings.md index 486dd7b44..bbd189a34 100644 --- a/dev/profiling/findings.md +++ b/dev/profiling/findings.md @@ -6,13 +6,13 @@ reproduces the predicted delta. Tags: - `[Port]` — R loop on the hot path that should move to C++. -- `[Optimise]` — C++ change with verified expected speedup. +- `[Optimize]` — C++ change with verified expected speedup. - `[AT-LIMIT]` — function is at a hardware ceiling; record so the rotation skips it in future rounds. | ID-suggest | P? | Status | Depends | Headline | Detail (% time, mechanism, verified Δ, micro-bench path) | |------------|----|--------|---------|----------|---------------------------------------------------------| -| T-300 | P1 | DONE | — | [Optimise] `full_rescore` after accepted TBR move (ts_tbr.cpp:1138): replace with incremental rescore | LANDED (commits f531bbcd EW + 014ccdea NA dirty-set). 19.2 % of NA-path DLL CPU; 15.2 % wall speedup on Zhu2013 NA (3.88→3.29 s). | +| T-300 | P1 | DONE | — | [Optimize] `full_rescore` after accepted TBR move (ts_tbr.cpp:1138): replace with incremental rescore | LANDED (commits f531bbcd EW + 014ccdea NA dirty-set). 19.2 % of NA-path DLL CPU; 15.2 % wall speedup on Zhu2013 NA (3.88→3.29 s). | | #38 | P3 | DONE / AT-LIMIT | T-300 | [AT-LIMIT] the residue T-300 left: `full_rescore` on TBR-**rerooting** accepts | The rerooting arm is now incremental too (third dirty seed at `clip_node`), but the lever is spent. Post-T-300 the WHOLE accept branch is **0.18–0.51 %** of `tbr_search` wall (Vinther2008 / Agnarsson2004 / Zanol2014 × EW,IW; in-DLL `na_t_accept_ms`, 15 paired cells, `dev/profiling/drivers/tbr-accept-ab-cell.R`). The patch cuts that slice to ×0.901 median (12/15 cells) ⇒ **≈0.03 % of wall e2e — undetectable**. Merge case is codepath unification, not speed. Rerooting accepts are ~18 % of accepts, so this is not a coverage artefact. | ## Round 3 (2026-06-16) — standard-Fitch TNT-parity path (Zhu2013 `-`→`?`, auto→thorough) @@ -22,8 +22,8 @@ DLL self-CPU total 2.70 s. Names resolved via `nm` (VTune reporter shows | ID-suggest | P? | Status | Depends | Headline | Detail (% time, mechanism, verified Δ, micro-bench path) | |------------|----|--------|---------|----------|---------------------------------------------------------| -| T-S3a | P3 | **DONE (verified)** | — | [Optimise] Per-clip allocation churn in TBR helpers — Tier 1 | **IMPLEMENTED 2026-06-16** (uncommitted, in working tree). Converted per-clip scratch vectors to `static thread_local` + clear/assign: `fitch_incremental_uppass` `dirty` (`std::vector`→`char`, allocated EVERY clip — ts_fitch.cpp:225), `collect_main_edges`/`collect_subtree_edges` DFS stacks, `compute_from_above` preorder+stack (ts_tbr.cpp). Per-thread-safe (each search thread owns its TreeState; none re-entrant). **VERIFIED:** Zhu2013 `-`→`?` score 627 unchanged (10/10 runs); ts- suite 3061 assertions, 0 fail; wall-clock 4.00 s→3.84 s = **~4.0%** (non-overlapping medians, TS_REPS=8 TS_SEED=1, identical build flags). Tier 2 (dedup table) now **DONE — see T-S3d (~3%)**. Tier 3a (gate `validate_topology` under NDEBUG, ~2-3%) NOT done — a safety/policy call (removes a release-build invariant check). | -| T-S3d | P3 | **DONE (verified)** | T-S3a | [Optimise] Per-clip rerooting dedup table — Tier 2 | **IMPLEMENTED 2026-06-16** (uncommitted, in working tree). Replaced the per-clip `std::unordered_set seen_vp_hashes` (ts_tbr.cpp ~946 — a bucket-array alloc + a per-insert node malloc on every internal-node clip) with a reusable open-addressed `VpHashSet`: power-of-two table, generation-stamped O(1) `reset()` (bump a counter, no zeroing), Fibonacci-mixed probe (fast_hash is FNV-1a, weak low bits), linear probing. Dedups on the exact 64-bit key ⇒ semantics identical to `unordered_set`. **KEY LESSON (emutls):** first written `static thread_local` → measured ~neutral/slight-regression, because MinGW resolves `thread_local` in a *loaded DLL* via **emutls** (a function call per access) and `insert()` runs per reroot candidate. Re-implemented as a **plain local declared once before the clip loop** (per-thread-safe via the call stack; zero TLS) → **~3% wall-clock** win: interleaved A/B floor **3.72 s vs 3.83 s** Tier1-only, clean-block median 3.75 vs 3.86, score 627 identical (dev/profiling/bench_tier2.R). **Behaviour-neutral:** identical scores on 6 datasets × {NA three-pass, standard Fitch} (dev/profiling/bench_equiv.R, diff empty). NB: the full ts- suite can't be a gate from a temp lib — relative `lib.loc` breaks data lazy-load under testthat's CWD switch (use absolute), and there is a PRE-EXISTING **flaky `nThreads=2` crash** (exit 127, reproduces on Tier1-only — NOT Tier 2; passes on re-run). | +| T-S3a | P3 | **DONE (verified)** | — | [Optimize] Per-clip allocation churn in TBR helpers — Tier 1 | **IMPLEMENTED 2026-06-16** (uncommitted, in working tree). Converted per-clip scratch vectors to `static thread_local` + clear/assign: `fitch_incremental_uppass` `dirty` (`std::vector`→`char`, allocated EVERY clip — ts_fitch.cpp:225), `collect_main_edges`/`collect_subtree_edges` DFS stacks, `compute_from_above` preorder+stack (ts_tbr.cpp). Per-thread-safe (each search thread owns its TreeState; none re-entrant). **VERIFIED:** Zhu2013 `-`→`?` score 627 unchanged (10/10 runs); ts- suite 3061 assertions, 0 fail; wall-clock 4.00 s→3.84 s = **~4.0%** (non-overlapping medians, TS_REPS=8 TS_SEED=1, identical build flags). Tier 2 (dedup table) now **DONE — see T-S3d (~3%)**. Tier 3a (gate `validate_topology` under NDEBUG, ~2-3%) NOT done — a safety/policy call (removes a release-build invariant check). | +| T-S3d | P3 | **DONE (verified)** | T-S3a | [Optimize] Per-clip rerooting dedup table — Tier 2 | **IMPLEMENTED 2026-06-16** (uncommitted, in working tree). Replaced the per-clip `std::unordered_set seen_vp_hashes` (ts_tbr.cpp ~946 — a bucket-array alloc + a per-insert node malloc on every internal-node clip) with a reusable open-addressed `VpHashSet`: power-of-two table, generation-stamped O(1) `reset()` (bump a counter, no zeroing), Fibonacci-mixed probe (fast_hash is FNV-1a, weak low bits), linear probing. Dedups on the exact 64-bit key ⇒ semantics identical to `unordered_set`. **KEY LESSON (emutls):** first written `static thread_local` → measured ~neutral/slight-regression, because MinGW resolves `thread_local` in a *loaded DLL* via **emutls** (a function call per access) and `insert()` runs per reroot candidate. Re-implemented as a **plain local declared once before the clip loop** (per-thread-safe via the call stack; zero TLS) → **~3% wall-clock** win: interleaved A/B floor **3.72 s vs 3.83 s** Tier1-only, clean-block median 3.75 vs 3.86, score 627 identical (dev/profiling/bench_tier2.R). **Behaviour-neutral:** identical scores on 6 datasets × {NA three-pass, standard Fitch} (dev/profiling/bench_equiv.R, diff empty). NB: the full ts- suite can't be a gate from a temp lib — relative `lib.loc` breaks data lazy-load under testthat's CWD switch (use absolute), and there is a PRE-EXISTING **flaky `nThreads=2` crash** (exit 127, reproduces on Tier1-only — NOT Tier 2; passes on re-run). | | T-S3b | — | AT-LIMIT | — | [AT-LIMIT] `simd::any_hit_reduce(3)_avx2` (21 % DLL) | Core Fitch reduce. Disasm of `hor_or256` confirms GCC already emits register-only horizontal reduce (vextracti128/vpsrldq/vpor/vmovq) — the store-reload anti-pattern is elided. Compiler-optimal at -O2. No win. | | T-S3c | — | AT-LIMIT | — | [AT-LIMIT] `uppass_node` scalar state-update loop (13 % DLL) | The update loop (ts_fitch.cpp:54-61) is scalar vs the vectorised `fitch_combine` in downpass. Micro-bench `dev/profiling/microbench/bench_uppass_combine.cpp`: AVX2 version bit-identical (value+changed flag, 0 mismatches) but only **1.22×** at n_states=4, and the 4-wide path does NOT trigger for 2-state (binary) morphology → ~1 % wall-clock. Not worth the incremental-uppass correctness risk (see fitch-scoring memory: dirty-flag invariant is delicate). | | — | — | NOTE | — | [Strategic] Standard-Fitch is bookkeeping/strategy-bound, not scoring-bound | Per-candidate scoring at AVX2/compiler limit. Parity levers: (a) reduce per-clip O(n) bookkeeping — `build_postorder_prealloc` 5.2 % (rebuilt per clip+accept) + incremental down/uppass 6.4 %; (b) ratchet (63 %) evaluation economy. Aligns with `.positai/plans/2026-03-21-tnt-outperformance-analysis.md` (strategy > code). | @@ -38,20 +38,20 @@ names via `nm`): | ID-suggest | P? | Status | Depends | Headline | Detail (% time, mechanism, verified Δ, micro-bench path) | |------------|----|--------|---------|----------|---------------------------------------------------------| -| T-P5a | **P0** | **CONFIRMED dominant (full-EW)** | — | [Optimise/Algorithm] `compute_insertion_edge_sets` = **27.3 % of total CPU (≈47 % of TS self-CPU)** on the REAL full-EW workload — THE per-iteration deficit vs TNT | Full-EW VTune (dev/profiling/drivers/full-ew-vtune.R, Zanol2014 fitch ×2 reps3, symboled .vtune-lib-20260618212528, resolve_syms.R maps the `combine` lambdas back to the fn by Start Address): **1.96 s / 7.19 s = 27.3 %**, #1 by 2.5×. Breakdown: two scalar `combine` operator() lambdas 0.81+0.67 s + self ≈ 1.96 s; PLUS it drives most ucrtbase memory traffic (memset func@0x180020b2c 0.89 s + vector::assign 0.24 + malloc 0.18 + memcpy 0.08). Core scoring (any_hit_reduce_avx2 0.77 + fitch_indirect_length_cached 0.47 + …) ≈ AT-LIMIT (Round 3). This per-clip **O(N)** directional edge-set recompute is the recently-added unrooted-TBR code; TNT avoids it via INCREMENTAL length updates (Goloboff 1996) ⇒ it IS "what we're missing" per iteration (matches framing: throughput 1.3-2.3×, efficiency~1, so same candidates evaluated more expensively). Explains Step-1's mere 2 %: it removed only the small malloc, NOT the 1.48 s compute or 0.9 s zero-fill. **LEVERS:** (1) skip per-clip zero-fill [Step-2; needs write-before-read invariant + correctness gate; ~targets the 0.9 s memset]; (2) vectorize the combine [1.48 s/21 %; re-examine despite earlier memory-bound/state-poor caveat — absolute cost now too large to dismiss]; (3) **ALGORITHM: amortize/incremental edge-set across clips** [the real Goloboff fix; attacks the whole 27 %+; correctness-critical on the quality-fix fn]. | -| T-P5b | P2 | **Step 1 verified (neutral); small Δ** | T-P5a | [Optimise] `compute_insertion_edge_sets` per-clip alloc + zero-fill (~11–14 %) | **Step 1 (malloc/free hoist, KEEPS zero-fill):** `up`/`pre` now CALLER-owned plain-local buffers passed by ref (emutls lesson T-S3d: NOT thread_local), reused across clips. **Behaviour-neutral VERIFIED 2026-06-19:** score AND `candidates_evaluated` BIT-IDENTICAL base(.agent-p0) vs step1(.bench-step1), 6 EW(fitch) runs × {Wortley,Zhu,Zanol}×seed{1,2} (dev/profiling/drivers/ab.R + ab_compare.R). **End-to-end EW wall ratio 0.981** (~2 %; Wortley 0.887/tiny-abs, Zhu 0.969, Zanol 0.998/flat) — far below the 31 % clip-loop share (see dilution note). **Step 2 (skip zero-fill)** = pending; bisects memset cost from the malloc cost Step 1 just measured as ~2 %. | +| T-P5a | **P0** | **CONFIRMED dominant (full-EW)** | — | [Optimize/Algorithm] `compute_insertion_edge_sets` = **27.3 % of total CPU (≈47 % of TS self-CPU)** on the REAL full-EW workload — THE per-iteration deficit vs TNT | Full-EW VTune (dev/profiling/drivers/full-ew-vtune.R, Zanol2014 fitch ×2 reps3, symboled .vtune-lib-20260618212528, resolve_syms.R maps the `combine` lambdas back to the fn by Start Address): **1.96 s / 7.19 s = 27.3 %**, #1 by 2.5×. Breakdown: two scalar `combine` operator() lambdas 0.81+0.67 s + self ≈ 1.96 s; PLUS it drives most ucrtbase memory traffic (memset func@0x180020b2c 0.89 s + vector::assign 0.24 + malloc 0.18 + memcpy 0.08). Core scoring (any_hit_reduce_avx2 0.77 + fitch_indirect_length_cached 0.47 + …) ≈ AT-LIMIT (Round 3). This per-clip **O(N)** directional edge-set recompute is the recently-added unrooted-TBR code; TNT avoids it via INCREMENTAL length updates (Goloboff 1996) ⇒ it IS "what we're missing" per iteration (matches framing: throughput 1.3-2.3×, efficiency~1, so same candidates evaluated more expensively). Explains Step-1's mere 2 %: it removed only the small malloc, NOT the 1.48 s compute or 0.9 s zero-fill. **LEVERS:** (1) skip per-clip zero-fill [Step-2; needs write-before-read invariant + correctness gate; ~targets the 0.9 s memset]; (2) vectorize the combine [1.48 s/21 %; re-examine despite earlier memory-bound/state-poor caveat — absolute cost now too large to dismiss]; (3) **ALGORITHM: amortize/incremental edge-set across clips** [the real Goloboff fix; attacks the whole 27 %+; correctness-critical on the quality-fix fn]. | +| T-P5b | P2 | **Step 1 verified (neutral); small Δ** | T-P5a | [Optimize] `compute_insertion_edge_sets` per-clip alloc + zero-fill (~11–14 %) | **Step 1 (malloc/free hoist, KEEPS zero-fill):** `up`/`pre` now CALLER-owned plain-local buffers passed by ref (emutls lesson T-S3d: NOT thread_local), reused across clips. **Behaviour-neutral VERIFIED 2026-06-19:** score AND `candidates_evaluated` BIT-IDENTICAL base(.agent-p0) vs step1(.bench-step1), 6 EW(fitch) runs × {Wortley,Zhu,Zanol}×seed{1,2} (dev/profiling/drivers/ab.R + ab_compare.R). **End-to-end EW wall ratio 0.981** (~2 %; Wortley 0.887/tiny-abs, Zhu 0.969, Zanol 0.998/flat) — far below the 31 % clip-loop share (see dilution note). **Step 2 (skip zero-fill)** = pending; bisects memset cost from the malloc cost Step 1 just measured as ~2 %. | | — | — | **NOTE (2026-06-19)** | — | [Dilution] The 31 % is clip-loop-ISOLATED, not full-search | T-P5a's 31 % came from the `ts_tbr_diagnostics` clip-loop driver (pure TBR-to-convergence). Full `MaximizeParsimony` (the MISSION workload) also runs Wagner + sectorial + ratchet reweight + consensus + R glue, so `compute_insertion_edge_sets` is a much smaller slice end-to-end — hence Step 1's ~2 %. RE-PROFILE DONE → see phase table below. | | T-P5c | **P0** | OPEN | — | [Strategic] FULL-EW phase attribution: **RATCHET = ~60 %**, sectorial ~30 %, all TBR <8 % | Per-phase `result$timings` (driver dev/profiling/drivers/framing-phases.R, traces `ts_driven_search`, `.agent-p0` release DLL, no rebuild). Zhu2013 + Zanol2014 fitch × seed{1,2}, maxReplicates=3: **ratchet 57.9/59.9/60.2/62.4 %**; rss 14.4–17.6 %; xss 8.0–10.5 %; css 7.2–8.5 %; initial tbr 1.5–3.1 %; final_tbr 2.3–2.5 %; wagner 1.7–1.9 %; fuse 0–2.5 %. The mission's center of gravity is the **ratchet phase** (reweight→TBR-re-search cycles), NOT the isolated TBR clip loop. NB the Round-3 "63 % ratchet" was NOT stale — fresh build confirms ~60 %. CAVEAT: the per-clip cluster (compute_insertion_edge_sets + compute_from_above + vroot_cache) is CROSS-CUTTING — TBR runs inside ratchet+sectorial too — so its true inclusive cost needs VTune on a FULL run, not the 5 % `tbr_ms`. NEXT: read ts_ratchet.cpp for per-cycle redundant recompute; then VTune full-EW run, inclusive time on the cluster + ratchet internals. | | T-P5d | **P0** | OPEN (validate) | — | [Recipe] ratchet is LOAD-BEARING but OVER-PROVISIONED: `ratchetCycles` 12→6 ≈ 20–38 % wall win, no quality loss | Ablation `dev/profiling/drivers/ratchet-ablation.R` + cycles sweep `ratchet-sweep.R`, `.agent-p0`, maxReplicates=6, seeds 1-3. (1) **Ratchet OFF** (`ratchetCycles=0L` — now GENUINELY disables; the [[ratchet-not-disableable]] gotcha is FIXED in-code, ts_driven.cpp:212+319) reaches best-known score only on TINY trees (Longrich 20t 0/3 miss); MISSES by +1..+4 on 37–75t (Wortley 2/3, Wills 1/3, Zanol 3/3, Zhu 3/3). ⇒ sectorial+TBR does NOT substitute for ratchet post-fix; "drop ratchet" refuted except small-dataset (matches TNT dropping it for small n). (2) **`ratchetCycles=6` ≥ 12 on quality at 0.62–0.79× wall** on all 4 (Wortley/Wills 0/3 @0.62–0.63×; Zhu 6c BEATS 12c, 1/3 vs 2/3 miss; Zanol wash at +1, cheaper). (3) Matched-wall off+4×reps substitution works small (Wills 0/3 @0.76 s) but FAILS large (Zanol/Zhu still miss). RECOMMEND `ratchetCycles=6` default + ratchet OFF for n_tip<~30 — VALIDATE at realistic maxReplicates (default 96, not 6) + Hamilton time-matched gate before flipping a user-facing default. | | — | — | NOTE | — | [Strategic, fresh] The residual gap is per-clip state RECOMPUTATION TNT amortizes | `compute_insertion_edge_sets` + `compute_from_above` + `vroot_cache` rebuilt every clip = the per-clip overhead (~half of per-candidate time, perclip.R). T-P5a/b are the quick behaviour-neutral wins; the high-order lever is incremental cross-clip state maintenance (Goloboff 1996) — bigger, correctness-critical, scope separately. | -| T-P5e | **P1** | **BANKED (validate-merge)** | T-P5a L1 | [Optimise] **Lever 1 — skip per-clip zero-fill** (the big within-clip win) | Subagent (worktree), RELEASE libs, **bit-identical gate PASS** (score + `candidates_evaluated` identical base vs mod, {Wortley,Zhu,Zanol}×seed{1,2} @reps3 AND a reps10 heavy run). Wall: **Zanol2014 −16.4 %, sum −9.4 %** (heavy 10-rep, median of 3); gain concentrates on largest data (zero-fill is O(n_node×total_words)/clip), exactly as predicted. Change: `up`/`pre` caller-owned scratch (NOT thread_local — emutls T-S3d) + non-zeroing size-ensure; `#ifndef NDEBUG` write-before-read completeness guard; 3 call sites threaded (ts_tbr.cpp, ts_sector.cpp, ts_wagner.cpp). **COORDINATION:** parallel session `claude/perclip-edgeset-buf` (worktree TS-perclip) has UNCOMMITTED the SAME signature change but KEEPS the zero-fill (its own comment: "follow-up skips it — see T-P5b"). Lever 1 = that follow-up (superset). User must sequence the merge: perclip hoist → lever-1 zero-fill skip. Neither committed. | +| T-P5e | **P1** | **BANKED (validate-merge)** | T-P5a L1 | [Optimize] **Lever 1 — skip per-clip zero-fill** (the big within-clip win) | Subagent (worktree), RELEASE libs, **bit-identical gate PASS** (score + `candidates_evaluated` identical base vs mod, {Wortley,Zhu,Zanol}×seed{1,2} @reps3 AND a reps10 heavy run). Wall: **Zanol2014 −16.4 %, sum −9.4 %** (heavy 10-rep, median of 3); gain concentrates on largest data (zero-fill is O(n_node×total_words)/clip), exactly as predicted. Change: `up`/`pre` caller-owned scratch (NOT thread_local — emutls T-S3d) + non-zeroing size-ensure; `#ifndef NDEBUG` write-before-read completeness guard; 3 call sites threaded (ts_tbr.cpp, ts_sector.cpp, ts_wagner.cpp). **COORDINATION:** parallel session `claude/perclip-edgeset-buf` (worktree TS-perclip) has UNCOMMITTED the SAME signature change but KEEPS the zero-fill (its own comment: "follow-up skips it — see T-P5b"). Lever 1 = that follow-up (superset). User must sequence the merge: perclip hoist → lever-1 zero-fill skip. Neither committed. | | T-P5f | P2 | **AT-LIMIT** | T-P5a L2 | [AT-LIMIT] **Lever 2 — vectorize the `combine` lambda** — no real win | Subagent (worktree), RELEASE libs, bit-identical gate PASS (same 6 runs). Genuine AVX2 (runtime `cpu_has_avx2`), 4-wide. Wall: **sum −1.2 % (in ±2.5 % jitter), sign flips per-dataset** (Zhu2013 went SLOWER) — the noise signature. Mechanism: `n_states` tiny (Zhu 4, Wortley 8, Zanol 9; most chars fewer) → ~1 vector iter/block; the `set1`/horizontal-OR setup cancels the vector gain; the two combine sweeps are memory-bandwidth-bound (confirms T-S3c). Change is correct but DO NOT merge as a perf win. Any future win here is structural (fuse the two sweeps to halve buffer traffic — see L3a), not wider vectors. | | T-P5g | **P0** | OPEN (advisor+data gated) | T-P5a L3 | [Algorithm] **Lever 3 — cross-clip amortization is the ONLY route to the asymptotic win; root-caused** | The named "incremental edge sets across clips." **Proof there is no within-clip shortcut:** for a clip of subtree c, `up_e[D]` (divided-tree directional msg) = `up_full[D]` IFF D is an **ancestor of c** (then c ⊆ subtree(D), inside D's complement-of-view, so unchanged); for ALL non-ancestors of c (≈ O(N) nodes), c leaves D's view → `up[D]` changes. So a per-pass full-tree precompute reuses only the O(depth) ancestor-path; the per-clip directional pass is irreducibly O(N). The genuine Goloboff amortization (O(N) **total** directional work/pass vs O(N²)) requires moving the clip **incrementally in tree-order** so adjacent clips share O(1) topology — which means ABANDONING restore-between-clips (Phase 2 `restore_prealloc_undo`+`spr_unclip`) and the cutoff-tightening clip SHUFFLE (`order_clips`). High-risk restructure of a function ≥2 parallel sessions are editing now. Design: dev/plans/2026-06-19-lever3-incremental-edgeset.md. **L3a (fuse the two `combine` sweeps into one preorder pass — `up[D]` then `edge_set[D]` inline while hot) is the safe within-clip remnant: bit-identical, composes w/ Lever 1, smaller marginal win (zero-fill already took the big chunk); offered, not yet measured.** | | T-P5h | **P0** | **REFRAMES L3b → likely NO-GO** | T-P5g | [Strategic, DECISIVE] **TS per-candidate cost is FLAT in N (W-driven), not an O(N²) blowup ⇒ L3b is constant-factor, not asymptotic** | Hamilton 64-bit (job 17528864, COMPLETED) `framing_64bit.csv`: **TNT side FAILED** (tnt_rate/tnt_wall/throughput all NA — binary produced no output; gold-standard TNT comparison still owed, RE-RUN needed). BUT the TS-side 64-bit `ts_rate` (cand/ms) came through and is **flat across tips**: Wortley 37t≈13.9, Wills 55t≈13.4, Zanol 74t≈12.4, Zhu 75t≈20.1, Giles 78t≈20.9 — variation tracks **W/char-complexity** (heavy Zanol 12.4 vs light Zhu/Giles ~20), NOT tip count. If the per-clip O(N) directional pass were a per-candidate penalty, ts_rate would COLLAPSE with N; it does not (it's O(N) pass ÷ O(N) candidates/clip = O(1)/cand, already amortized). **Reconciles the local-32bit "throughput grows with N" (1.36@37→2.3@75, framing_latest.csv):** that trend is TNT-side (tnt_rate grew 18→44 with N — TNT speeds up per-candidate on bigger trees via cache/amortization) while TS stayed flat. So L3b's cross-clip restructure attacks only the directional pass's CONSTANT-FACTOR per-candidate share (~½ per perclip.R) — same order as L1(zero-fill, banked −16 %)/L3a(fuse) — at high restructure risk and shuffle-loss. **RECOMMEND NO-GO on the L3b restructure;** bank L1 (+measure L3a), treat the per-candidate deficit as constant-factor near-limit, shift to recipe composition (#39/#40). REVISIT L3b only if a SUCCESSFUL 64-bit TNT re-run shows tnt_rate asymptotically diverging from ts_rate in a way ONLY cross-clip amortization could match. | | T-P5h2 | **P0** | **64-bit TNT CAPTURED — prize is gold-standard 2–3.4×, REOPENS L3b tension** | T-P5h | [Strategic] **Successful 64-bit head-to-head (Hamilton job 17529081, framing_64bit_log.csv)** | The TNT-headless harness fix (stdin-pipe + TERM=dumb, see [[profiling]]) WORKED — `tnt_rearr`/`tnt_rate` now populated (only `tnt_score` regex still misses this xmult format — immaterial). **64-bit rates (cand-or-rearr/ms):** Wortley 37t ts14/tnt17 = **1.2×**; Wills 55t ts13/tnt44 = **3.4×**; Zanol 74t ts12/tnt30 = **2.5×**; Zhu 75t ts20/tnt40 = **2.1×**; Giles 78t ts20/tnt41 = **2.1×**. Confirms T-P5h SHAPE (TS flat/W-driven, TNT rises with N), but the magnitude is **NOT soft 32-bit — it's a gold-standard ~2–3× per-candidate prize**, gapB=0 (all best-known). **TENSION:** T-P5h NO-GO'd L3b as "constant-factor near-limit" — but that constant factor is ~2–3×, and `perclip.R` measured the per-clip cluster (compute_insertion_edge_sets + compute_from_above + vroot_cache, the L3b target) as ~½ of per-candidate time = the bulk of this prize. "Flat in N" rules out an ASYMPTOTIC argument for L3b, NOT a 2× constant-factor one. **NEXT (settles it):** shared-start TBR race (dev/benchmarks/tbr_shared_start_lib.R) from identical t0 — if TNT reaches the same score per candidate ~2× faster (throughput) it's the bookkeeping (L3b reopens); if it's per-candidate parity but fewer candidates (efficiency) the gap is search-trajectory, not bookkeeping. Foundation under both ratchet (#43→#39) and sectorial races. | | T-P5h4 | **P0** | **RESOLVED: efficiency gap = COUNTING ARTIFACT; only ~2.5× throughput is real, and it's per-candidate OVERHEAD (not scoring, not bound)** | T-P5h3 | [Diagnostic, DECISIVE] Bail-fraction counter (discriminator #2, worktree-isolated -DTS_SCORE_STATS build) | Instrumented `fitch_indirect_length_cached` (bail at ts_fitch.cpp:470). **95–99 % of scorer calls bail early** (~2.5 blocks touched on Zanol; ~1.8 on Giles — **the original "of 210 / of 236" was a UNITS ERROR corrected by T-P5l: 210/236 are the PATTERN counts, NOT block counts; Zanol packs into 4 blocks, so ~2.5 of 4 = ~71 % read, NOT ~1 %; the ~2.5-blocks-touched measurement itself stands**); **fully-scored candidates <1.2 % of n_evaluated** every dataset/seed. Bonus: Giles fic_calls 100K ≪ n_evaluated 585K → ~485K positions rejected UPSTREAM of the scorer (zero_skip=0). ⇒ TS's 2–4× `n_evaluated` lead over TNT (T-P5h3) is a **counting artifact** (TS counts every regraft incl. cheap early-exit; real work ≈ 5–7.5K/descent ≈ TNT's). **EFFICIENCY GAP DISSOLVES.** The ONLY real per-iteration gap is the **~2.5× throughput** (framing 64-bit; BOTH builds 64-bit so NOT bitness — genuine per-candidate cost). Since scoring is 99 %-cheaply-bailed (the bound/cutoff is EXCELLENT — so NOT bound-tightness, advisor's gate #2 → it's bookkeeping-side), the 2.5× lives in per-candidate OVERHEAD *around* the cheap scoring: edge_set_buf lookup + cutoff/divided_length arithmetic + regraft-loop machinery — the cluster TNT avoids via incremental length. **LAST CHECK = discriminator #3:** re-measure per-candidate cost breakdown POST-L1 (perclip.R) — if the edge-set cluster is ~30 % even perfect amortization caps at ~1.4× (can't close 2.5× → L3b stays NO-GO); if ~50 %+ the overhead is the lever. | | T-P5h3 | **P0** | **SUPERSEDED by T-P5h4 (comparability RESOLVED)** | T-P5h2 | [Diagnostic] Shared-start strict-descent TBR race (dev/benchmarks/tbr_throughput_race.R, .agent-l1, local 32-bit TNT), discriminator #1 (advisor) aimed at LOSERS | From an IDENTICAL Wagner start, TS reaches an equal-or-better optimum but examines **2–4× MORE counted candidates** than TNT: Zanol ts 0.8M/tnt 0.2M (~4×), Zhu 0.6/0.3 (~2×), Giles 0.6/0.2 (~3×); scores parity (TS ≤ TNT by 0–3 steps — validity gate PASS). **TS per-candidate rate is clean & fast (16–21 Mcand/s, kernel-only timing) — matches framing 64-bit ts_rate (~12–20), so TS is NOT per-candidate-crippled.** TNT wall here is pure startup (~0.23 s flat, single-descent too short) ⇒ local isolated *throughput* is unusable; take throughput (~2.5×) from framing xmult instead. **Contradiction forcing discriminator #2:** if throughput(2.5×) AND efficiency(2–4×) were BOTH real, compound wall ≈ 5–10×, but framing whole-search wall_ratio is only ~2.5× ⇒ the candidate counts are almost certainly **NOT comparable** — TS `n_evaluated` counts every regraft incl. cutoff-bails (ts_tbr.cpp:1574); TNT likely omits bound-pruned. **NEXT = bail-fraction counter** in `fitch_indirect_length_cached` (bail at ts_fitch.cpp:470): if TS bails on ~50–75 % of candidates, real fully-scored count ≈ TNT's ⇒ efficiency dissolves, only the ~2.5× throughput remains (then chase WHY: bitness ~2× + at-limit kernel vs bookkeeping). Instrumented in ISOLATED WORKTREE (concurrency hazard: ts_fitch/ts_rcpp also touched by NA/IW agent). | -| T-P5e2 | **P1** | **MERGED to cpp-search 00d73d6a** | T-P5e | [Optimise] Lever 1 landed + re-verified on main checkout | Applied the validated diff to the main checkout (clean apply, both at 78b74147). **Re-gate on this checkout: all 6 EW runs bit-identical** to the documented values (score+cand exact, verify_l1.R); **NA single-threaded bit-identical** (Vinther2008 seeds1–4, na_serial_cmp.R); **276 kernel search tests PASS** (tbr/wagner/sector/ratchet/drift/fitch, NOT_CRAN). Committed 00d73d6a (NOT pushed) + NEWS bullet. **perclip resolved:** branch `claude/perclip-edgeset-buf` is ORPHANED (tip=25e35be7 is a cpp-search ancestor, no unique commits; uncommitted files last touched 2026-06-18 21:42; user confirms no live agent on it) and its changes are a strict SUBSET of Lever 1 (hoist only, keeps zero-fill) → nothing to redeem. **FOUND (out-of-scope):** a pre-existing intermittent CRASH in the **parallel (nThreads≥2) NA** path (Vinther2008) — reproduces on UNMODIFIED baseline (crashed iter 6/8), timing-race, present with TS_EV_NOCACHE=1 too; Lever 1 exonerated (baseline crashes; per-thread-safe). Filed as spawn_task task_3eda6e75 for the NA/IW workstream; repro = dev/profiling/drivers/repro_par.R. | +| T-P5e2 | **P1** | **MERGED to cpp-search 00d73d6a** | T-P5e | [Optimize] Lever 1 landed + re-verified on main checkout | Applied the validated diff to the main checkout (clean apply, both at 78b74147). **Re-gate on this checkout: all 6 EW runs bit-identical** to the documented values (score+cand exact, verify_l1.R); **NA single-threaded bit-identical** (Vinther2008 seeds1–4, na_serial_cmp.R); **276 kernel search tests PASS** (tbr/wagner/sector/ratchet/drift/fitch, NOT_CRAN). Committed 00d73d6a (NOT pushed) + NEWS bullet. **perclip resolved:** branch `claude/perclip-edgeset-buf` is ORPHANED (tip=25e35be7 is a cpp-search ancestor, no unique commits; uncommitted files last touched 2026-06-18 21:42; user confirms no live agent on it) and its changes are a strict SUBSET of Lever 1 (hoist only, keeps zero-fill) → nothing to redeem. **FOUND (out-of-scope):** a pre-existing intermittent CRASH in the **parallel (nThreads≥2) NA** path (Vinther2008) — reproduces on UNMODIFIED baseline (crashed iter 6/8), timing-race, present with TS_EV_NOCACHE=1 too; Lever 1 exonerated (baseline crashes; per-thread-safe). Filed as spawn_task task_3eda6e75 for the NA/IW workstream; repro = dev/profiling/drivers/repro_par.R. | | T-P5i | P2 | **AT-LIMIT** | T-P5g L3a | [AT-LIMIT] **L3a — fuse the two combine sweeps** — no measurable win | Built fused single-preorder-pass (up[D] then edge_set[D] inline), **bit-identical** (all 6 gate values exact, .agent-l3a). Wall A/B vs committed Lever 1 (.agent-l1), 4 interleaved rounds, full auto search reps10: **Zhu2013 +0.4 %, Zanol2014 +0.2 % (both SLOWER, in noise)**. Mechanism: at mission tree sizes (37–88t) the `up[]` buffer is tiny (~10 KB for Zanol = n_node×W×8) and stays L1/L2-resident, so the two-pass form's second read of up[] is ALREADY a cache hit — fusing saves no memory traffic (it never went to memory). Would only help at vastly larger trees (up[] > cache). Reverted (Lever 1 two-pass form kept). Same lesson as L2: don't merge a non-win as perf. ⇒ within-clip edge-set kernel is now AT-LIMIT post-L1; only L3b (cross-clip, T-P5g/h) remains, and T-P5h leans NO-GO. | | T-P5j | **P0** | **L3b CLOSED — DEAD by direct footprint+Euler measurement (this dataset class)** | T-P5g/h/i | [Algorithm, DECISIVE] Both incremental schemes fail the realizable-saving gate | Advisor measure-first reframe: 24–33 % is the SHARE; realizable saving is bounded by the per-clip CHANGED-VALUE footprint. Instrumented the from-scratch path (`-DTS_EDGESET_FOOTPRINT`, `src/ts_edgeset_footprint.h`, `ts_edgeset_footprint_report()`; driver `dev/profiling/drivers/l3b_footprint.R`; rows `l3b_footprint.csv`+`l3b_euler_gate.csv`, Wortley/Zanol/Zhu ×3 reps, 11–44k clips). **(1) Scheme-1 (patch-from-full-tree): footprint = 41–68 % of ALL edges change per clip** (Wortley 0.66, Zanol 0.47, Zhu 0.46; GO needed <0.3). Fitch combine does NOT saturate on these EW morphological matrices. Memory-traffic floor even at 0.41: recompute+undo-save+undo-restore ≈ 0.41×3 > 1.0 vs the bandwidth-bound clean sweep ⇒ LOSS. **(2) Euler-tour (cross-clip): per-descend-step delta = 1.14–1.24× the footprint** — one parent→child boundary move flips AS MANY OR MORE views (delta = tiny chunk 5–7 newly-exposed + LARGE common-changes 44–67 pre-existing) ⇒ NO cross-clip locality; a small topological change near the clip propagates view-changes across ~½ the tree via the non-saturating intersect-else-union. This delta is a true FLOOR on any Euler kernel (advisor-verified) and is *optimistic* (omits early-term boundary checks + the measured ~1.2× DFS candidate cost + reroot/restore integration risk). **Edge-set hotspot AT-LIMIT; L3b ABANDONED.** Confirms T-P5h4's "~30 % cluster ⇒ amortization caps ~1.4×" prediction and resolves the T-P5h2 tension (the 2–3× prize is NOT recoverable via incremental edge_set). Worktree code MEASUREMENT-ONLY — nothing to merge; instrumentation KEPT (fp_frac data-dependent, lower on bigger/denser ⇒ molecular/large-N is the REOPEN condition, a ~10-min rerun). **NOT killed:** bound-then-verify/lazy-exact (cheap admissible screen → exact for survivors = TNT quick-TBR; distinct, untested, NOT NOW — scoring already at-limit T-P5h4, and no admissible Fitch-insertion lower bound established). ⇒ per-candidate throughput declared at-limit; pivot to recipe composition + sectorial (#39/#40). **CORRECTED by T-P5k: the closing 'per-candidate throughput at-limit' line OVER-CLAIMED — at-limit was proved only for (a) combine throughput + (b) cross-clip reuse, NOT the production per-candidate path. Scorer thread REOPENED (#46).** | | T-P5k | **P0** | **REOPENS scorer — per-candidate is NOT at-limit (T-P5j over-claimed)** | T-P5j | [Diagnostic, DECISIVE] VTune per-candidate split, post-L1 symboled (worktree `.vtune-lib-sym`, footprint `#ifdef`-compiled-out → clean production binary; `DLLFLAGS=-static-libgcc` to defeat the `-s` strip), Zanol2014 full-EW | Discriminator #3 (advisor) — queued in T-P5h4, finally run. Flat self + call-tree CSV, module=`TreeSearch.dll`, 6.4s CPU. **Per-candidate cluster ≈ ½ of EW CPU, split ~56% PRECOMPUTE / ~44% CONSUMPTION:** PRECOMPUTE = `compute_insertion_edge_sets` building the full ~210-block directional views = two combine `operator()` lambdas (ts_fitch.cpp, addrs adjacent to the fn) **0.731+0.683=1.41s** + self 0.156 + `uppass_node` 0.154 ≈ **1.8s**; CONSUMPTION = bail-fast scorer `fitch_indirect_length_cached` self 0.424 + `any_hit_reduce_avx2` 0.292 + horiz-reduce intrinsics (`_mm_or_si128` 0.123 + `_mm256_extracti128` 0.048) + `popcount64` 0.046 + regraft-loop machinery (`tbr_search` self 0.340) + `fitch_join_states` 0.063 ≈ **1.4s**. **NEITHER is at-limit as T-P5j asserted:** the combine THROUGHPUT is at-limit (T-P5f/S3b) but it is AVOIDABLE work, not irreducible; consumption is dominated by per-candidate FIXED overhead (SIMD setup paid ~2.5×/cand for a ~2.5-block bail = T-P5f's 'set1/horizontal-OR setup cancels the gain' signal, MIS-FILED at-limit; + loop machinery), NOT the at-limit combine kernel. **Reconciles the 2.5×:** the eager 210-block view-build (T-P5h4: 99% of scorings consume only ~2.5 blocks) ≈ the per-candidate THROUGHPUT gap TNT skips via incremental length. **CAVEATS (honest, advisor guard):** (1) realizable WALL win ~1.2-1.5×, NOT 2.5× — the per-candidate cluster is only ~½ of EW CPU (rest = ratchet reweight, sectorial, hashing, R glue, memory); 2.5× is the per-candidate THROUGHPUT metric, not end-to-end wall. (2) MECHANISM still ambiguous: avoidable-precompute (lazy / incremental-length = TNT quick-TBR) **vs** pervasive constant-factor (TNT does the same work ~2.5× faster via layout/scalar). The lazy route is NOT a slam-dunk — L3b (T-P5j) showed incremental FULL-view maintenance lacks locality here (footprint 41-68%), and lazy-per-block trades amortized O(all-blocks)/edge for un-amortized O(depth×bail-blocks)/cand, an untested tradeoff (the bound-then-verify route, still needs an admissible Fitch-insertion bound). **NEXT:** discriminator #2 micro-bench — scalar-inlined vs SIMD `fitch_indirect_length_cached` in the bail regime (real Zanol clip, millions of calls, ≥3 medians) settles the cheap CONSUMPTION lever; separately scope the bigger PRECOMPUTE/incremental-length prize. `result_p5k_consume` deleted post-round; symboled lib KEPT for #2. **CORRECTED by T-P5l (below): the "~210-block / 98%-unread" model is WRONG — Zanol2014 EW DataSet = 4 blocks × n_states=9 (total_words=36); the scorer reads ~71% of each node's view (mean bail 2.85/4), NOT ~1%. CONSUMPTION scalar-inline lever DEAD; PRECOMPUTE lazy ceiling shrinks to ~29%-unread.** | @@ -81,8 +81,8 @@ setup-vs-loop split (ts_sector.cpp + ts_tbr.cpp, MEASUREMENT-ONLY, gated). |------------|----|--------|---------|----------|---------------------------------------------------------| | T-S6a | **P0** | **AT-LIMIT (by inheritance) — but kernel had a 22 % getenv fat, see T-S6d** | — | [AT-LIMIT] Isolated sectorial wall is ~96 % the TBR clip loop; sectorial-specific scaffolding ≤2 % | chrono decomposition (`sector-rss.R` TS_SECT_TIMING, 5 datasets × {Wagner,TBR-conv} start): **inner `tbr_search` loop + global TBR = 96–97 %** of isolated sectorial wall. **Sectorial-SPECIFIC scaffolding** (`build_reduced_dataset` + reduced `score_tree` + accept-path reinsert/full-rescore) = **1.3–2.0 % stable across datasets** (Zanol-wagner 0.047/2.97 s=1.6 %; Zhu 1.3 %; Wortley 1.5 %; Wills 2.0 %). `compute_from_above_for_sector` ≤0.4 %; accept-path 0.000–0.004 s (full tree 37–78 t small ⇒ Lever 2 dirty-set rescore = noise, DROPPED). `tbr_search` per-call SETUP = 8.7 µs/call = ~1 % (the "short TBR calls don't amortize setup" hypothesis REFUTED). ⇒ NO sectorial-SPECIFIC throughput lever above ~1 %. **CORRECTION (T-S6d): the measured `tbr_search` LOOP (2.88 s) was INFLATED ~0.69 s by a per-clip diagnostic getenv hidden in VTune's ucrtbase self-time; the TRUE at-limit loop is ~2.19 s.** So "at-limit by inheritance" stands structurally, but the kernel itself carried 22 % removable getenv fat. Material sectorial speedups: (a) the getenv hoist [T-S6d, ~22 %, TBR-kernel-wide], (b) faster TBR kernel [closed T-P5l/m], (c) fewer/smaller sector TBR calls [recipe/quality, closed at parity — sector-resolve-status]. | | T-S6b | — | **DEAD (measured)** | T-S6a | [Dead] Inner-sector unrooted-reroot sweep is wall-neutral; not a removable cost | The inner sector `tbr_search` defaults `unrooted=true` (no sector_mask/cd/tabu/pool ⇒ `do_reroot` active), and `search_sector`'s `root_ok` guard REVERTS any result that floats the HTU — hypothesised as wasted reroot work. MEASURED (`TS_SECT_NOREROOT` toggle + `SECT_REVERT` counter, Zanol-wagner 48×80): reroot ON loop=2.99 s vs OFF 3.01 s (**wall-neutral, in noise**); revert rate only **3.4 %** (130/3840). The reroot sweep activates only at convergence and exits fast when the first re-descent finds nothing ⇒ negligible cost. Disabling it also perturbs scores (improved 209→206) for zero wall gain. NOT a lever. | -| T-S6c | P3 | **DONE (verified, byte-identical, ~2.8 %)** | T-S6a | [Optimise] Sectorial-specific byte-identical micro-levers (ts_sector.cpp) | THREE byte-identical levers banked (gate: score + n_sectors_searched/improved BIT-IDENTICAL base vs mod across {Zanol,Zhu,Wortley}×{wagner,tbr}×seed{1,2} = 12/12; 8 test files PASS). (1) **`search_sector` `ras_starts==1` fast path** (the default): the `best_left/right/parent` snapshot + post-loop restore + `build_postorder` are a provable no-op round-trip when there is one start — `rd.subtree` already holds the result and `reinsert_sector` never reads postorder. Skip them. **~0.05 s** (biggest of the three — advisor-flagged residual). (2) **`compute_from_above_for_sector` alloc hoist**: `new_from_above(tw)` was heap-allocated INSIDE the path loop (ts_sector.cpp:72); now allocated once + `std::swap` each step (padding words stay 0 in both ⇒ byte-identical). **~0.006 s**. (3) **getenv hoist** (see T-S6d for why): `TS_FREE_HTU_PROBE` was read 2–3×/pick + `TS_SECT_DEBUG` per accept → cached `static const bool`. **~0.028 s** (3×3840 getenv × 2.4 µs). **Combined wall: Zanol 48×80 base 3.08→2.99 s = ~2.8 %, 8/8 rounds faster.** Attribution sums exactly to the A/B delta. ReducedDataset pooling (the 4th candidate) NOT done: ceiling ~0.5 % and a stale-field correctness risk in a scoring-feeding path — not worth it (advisor concur). Driver `dev/profiling/drivers/sector-rss.R`. | -| T-S6d | **P0** | **HEADLINE — per-clip diagnostic getenv = ~22 % of TBR wall (cross-cutting, TBR-kernel-wide); TBR agent owns prod hoist** | — | [Optimise/Strategic] `std::getenv` is **2398 ns/call** on Windows/ucrt (linear env scan; `dev/profiling/microbench/bench_getenv.cpp`), and `tbr_search` calls `getenv("TS_REVERT_CHECK")` **per clip** (ts_tbr.cpp:1868, Phase-2 restore block) — a pure diagnostic that is off in production. **MISATTRIBUTION (the trap): VTune did NOT label this — getenv's env-scan time hid inside ucrtbase self-time** (`func@0x1800a20d4`/`func@0x180048b50`/`strcoll_l` entries), which is why my first pass wrongly concluded "no getenv hotspot." MEASURED via chrono setup/loop split + A/B hoist (cached `static const bool`): isolated-sectorial `tbr_search` LOOP **2.88 → 2.19 s**, wall **3.07 → 2.37 s** (just `TS_REVERT_CHECK`+`TS_IW_SCANCHK`); + `TS_PHYS_REROOT` hoist → **2.28 s**. **Total base 3.02 → 2.28 s = 24.5 % isolated-sectorial wall** (TBR getenv ~0.65 s + T-S6c ~0.085 s). Per-clip count ≈ 74 clips/call × 3888 calls × 2.4 µs ≈ 0.69 s. **This is a TBR-KERNEL cost (every `tbr_search` everywhere — main search, ratchet, sectorial), not sectorial-specific** ⇒ ~20 %+ mission-wide if `tbr_search` clip count dominates; the TBR agent flagged `TS_REVERT_CHECK` as "the main offender" and owns the production hoist. **MECHANISM (advisor/user Q "is getenv right?"): NO — getenv is wrong for any per-clip/per-candidate gate.** Fix = read-once: (a) **compile-out** the per-clip VALIDATION diagnostics (`TS_REVERT_CHECK`, `TS_IW_SCANCHK`) via a `#ifdef TS_VALIDATE` macro — they are undo-correctness invariant checks that should NEVER run in release (zero overhead + zero dead code in the hot loop); (b) for gates needing runtime toggle, a **centralized read-once debug-flags struct** populated at DLL init (plain bool loads, no per-access static-guard); (c) per-site `static const bool` (what was used here) is adequate but scatters statics. Validated from the sectorial side; 8 test files PASS with the hoist. **MISSION-WIDE A/B CONFIRMED (decisive): full `MaximizeParsimony` (ratchet+sectorial+TBR, NOT isolated), Zhu2013 thorough reps=3 × seed{1,2,3}: base 1.83/1.65/1.76 s → mod(hoist+T-S6c) 1.36/1.33/1.39 s = ~20–26 % wall, SCORE BYTE-IDENTICAL (625/624/625 both sides)** (`dev/profiling/drivers/mission-getenv-ab.R`). Of that, T-S6c sector levers ≈0.8 % mission ⇒ the getenv hoist ALONE ≈ ~19–25 % MISSION wall — one of the largest single wins in the program, and it was invisible to the full-search VTune (buried in ucrtbase); isolating sectorial's small-tree regime is what surfaced it. NOTE this does NOT contradict T-P5l/m "TBR kernel at-limit" — the scorer/combine/reduce ARE at-limit; the getenv is diagnostic overhead AROUND the kernel that the prior VTune mis-filed as ucrtbase. | +| T-S6c | P3 | **DONE (verified, byte-identical, ~2.8 %)** | T-S6a | [Optimize] Sectorial-specific byte-identical micro-levers (ts_sector.cpp) | THREE byte-identical levers banked (gate: score + n_sectors_searched/improved BIT-IDENTICAL base vs mod across {Zanol,Zhu,Wortley}×{wagner,tbr}×seed{1,2} = 12/12; 8 test files PASS). (1) **`search_sector` `ras_starts==1` fast path** (the default): the `best_left/right/parent` snapshot + post-loop restore + `build_postorder` are a provable no-op round-trip when there is one start — `rd.subtree` already holds the result and `reinsert_sector` never reads postorder. Skip them. **~0.05 s** (biggest of the three — advisor-flagged residual). (2) **`compute_from_above_for_sector` alloc hoist**: `new_from_above(tw)` was heap-allocated INSIDE the path loop (ts_sector.cpp:72); now allocated once + `std::swap` each step (padding words stay 0 in both ⇒ byte-identical). **~0.006 s**. (3) **getenv hoist** (see T-S6d for why): `TS_FREE_HTU_PROBE` was read 2–3×/pick + `TS_SECT_DEBUG` per accept → cached `static const bool`. **~0.028 s** (3×3840 getenv × 2.4 µs). **Combined wall: Zanol 48×80 base 3.08→2.99 s = ~2.8 %, 8/8 rounds faster.** Attribution sums exactly to the A/B delta. ReducedDataset pooling (the 4th candidate) NOT done: ceiling ~0.5 % and a stale-field correctness risk in a scoring-feeding path — not worth it (advisor concur). Driver `dev/profiling/drivers/sector-rss.R`. | +| T-S6d | **P0** | **HEADLINE — per-clip diagnostic getenv = ~22 % of TBR wall (cross-cutting, TBR-kernel-wide); TBR agent owns prod hoist** | — | [Optimize/Strategic] `std::getenv` is **2398 ns/call** on Windows/ucrt (linear env scan; `dev/profiling/microbench/bench_getenv.cpp`), and `tbr_search` calls `getenv("TS_REVERT_CHECK")` **per clip** (ts_tbr.cpp:1868, Phase-2 restore block) — a pure diagnostic that is off in production. **MISATTRIBUTION (the trap): VTune did NOT label this — getenv's env-scan time hid inside ucrtbase self-time** (`func@0x1800a20d4`/`func@0x180048b50`/`strcoll_l` entries), which is why my first pass wrongly concluded "no getenv hotspot." MEASURED via chrono setup/loop split + A/B hoist (cached `static const bool`): isolated-sectorial `tbr_search` LOOP **2.88 → 2.19 s**, wall **3.07 → 2.37 s** (just `TS_REVERT_CHECK`+`TS_IW_SCANCHK`); + `TS_PHYS_REROOT` hoist → **2.28 s**. **Total base 3.02 → 2.28 s = 24.5 % isolated-sectorial wall** (TBR getenv ~0.65 s + T-S6c ~0.085 s). Per-clip count ≈ 74 clips/call × 3888 calls × 2.4 µs ≈ 0.69 s. **This is a TBR-KERNEL cost (every `tbr_search` everywhere — main search, ratchet, sectorial), not sectorial-specific** ⇒ ~20 %+ mission-wide if `tbr_search` clip count dominates; the TBR agent flagged `TS_REVERT_CHECK` as "the main offender" and owns the production hoist. **MECHANISM (advisor/user Q "is getenv right?"): NO — getenv is wrong for any per-clip/per-candidate gate.** Fix = read-once: (a) **compile-out** the per-clip VALIDATION diagnostics (`TS_REVERT_CHECK`, `TS_IW_SCANCHK`) via a `#ifdef TS_VALIDATE` macro — they are undo-correctness invariant checks that should NEVER run in release (zero overhead + zero dead code in the hot loop); (b) for gates needing runtime toggle, a **centralized read-once debug-flags struct** populated at DLL init (plain bool loads, no per-access static-guard); (c) per-site `static const bool` (what was used here) is adequate but scatters statics. Validated from the sectorial side; 8 test files PASS with the hoist. **MISSION-WIDE A/B CONFIRMED (decisive): full `MaximizeParsimony` (ratchet+sectorial+TBR, NOT isolated), Zhu2013 thorough reps=3 × seed{1,2,3}: base 1.83/1.65/1.76 s → mod(hoist+T-S6c) 1.36/1.33/1.39 s = ~20–26 % wall, SCORE BYTE-IDENTICAL (625/624/625 both sides)** (`dev/profiling/drivers/mission-getenv-ab.R`). Of that, T-S6c sector levers ≈0.8 % mission ⇒ the getenv hoist ALONE ≈ ~19–25 % MISSION wall — one of the largest single wins in the program, and it was invisible to the full-search VTune (buried in ucrtbase); isolating sectorial's small-tree regime is what surfaced it. NOTE this does NOT contradict T-P5l/m "TBR kernel at-limit" — the scorer/combine/reduce ARE at-limit; the getenv is diagnostic overhead AROUND the kernel that the prior VTune mis-filed as ucrtbase. | | T-S6e | P2 | **EFFICIENCY AXIS PROBED — top ideas are quality-sensitive or marginal (no clean behaviour-neutral win)** | T-S6a | [Strategic/Recipe] gate-2 efficiency (work-to-target) probes, post fresh-ideas brainstorm (6 ideators→adversarial filter, `workflows/sectorial-efficiency-ideas`) | Brainstorm shortlist reduced to two instrument-only probes (Probe A counters in rss_search, TS_SECT_TIMING-gated). **Idea #1 (suppress the trailing global TBR in rss_search):** the driven pipeline runs XSS→RSS→CSS in SEQUENCE (ts_driven.cpp:246/265/288), each with its OWN trailing full-tree TBR; once one converges the tree the next re-searches it. MEASURED: rss trailing TBR improved score **0/36 (Zhu), 0/36 (Zanol), 0/39 (Wortley)** within rss_search, and its wall (g_t_gtbr **0.106 s ≈ 8 % of mission**, on Zhu) EXCEEDS the inner sector search (0.101 s). LOOKED like ~8%+ free. **BUT REFUTED as behaviour-neutral:** `TS_SKIP_RSS_GTBR` A/B (4 datasets × 3 seeds, full MaximizeParsimony) is ~20–30 % faster but **loses 1–2 steps on 3/12 seeds** (Zhu2013-s2 624→625, Zanol2014-s2 1261→1263, Wortley2006-s2 479→480). The 0/36 was a LOCAL artifact — the trailing TBR finds nothing *within* rss but leaves a TBR-converged tree that shapes the DOWNSTREAM ratchet/phase trajectory (the "locally-null phase is globally load-bearing" trap; component-isolation plan's "different optima for non-throughput reasons"). ⇒ NOT a safe lever; it's a speed/quality tradeoff (candidate "fast preset", not a default). **Smarter unexplored form:** consolidate the 3 sequential trailing TBRs into ONE after the xss→rss→css block (might keep the trajectory-shaping while dropping 2 redundant passes) — a RECIPE redesign (ts_driven.cpp) needing broad e2e + Hamilton validation; handed to the search-recipe owner, NOT done here. **Idea #2 (without-replacement sector picks):** within-constant-tree-window duplicate-pick rate = **0–4.6 % in the mission's ~5-pick regime** (the 69 % I first saw was a non-mission high-pick-converged artifact) ⇒ marginal (~1 % mission ceiling) + a stochastic-skip quality-leak risk. **Ideas #3–6 (collapse family / negative cache):** near-dead at the default `rss_rounds=1` (their prize rested on multi-round recurrence that doesn't exist); collapse (#3) is a zero-new-code probe (`collapse_target ∈ {0,8,15}` via the existing lever) left for a follow-up. **VERDICT: the efficiency axis offers no clean behaviour-neutral win at default config — sectorial is at-limit for safe gains; further sectorial speedups require accepting a quality tradeoff (recipe tuning, separate axis).** Probe A counters + TS_SKIP_RSS_GTBR are env-gated MEASUREMENT-ONLY. | ## T-S7 (2026-06-20) — FUSE/DRIFT gate-1 (AT-LIMIT) isolation, code-only, no builds (#52) diff --git a/dev/profiling/focus-areas.md b/dev/profiling/focus-areas.md index 71d8cd457..90ba00dc2 100644 --- a/dev/profiling/focus-areas.md +++ b/dev/profiling/focus-areas.md @@ -1,7 +1,7 @@ # Profiling focus areas Ranked by `(estimated wall-time share) × (remaining fixability)`. Areas at the -memory-bandwidth ceiling or already optimised drop to the bottom but remain +memory-bandwidth ceiling or already optimized drop to the bottom but remain visible so the rotation knows to skip them. Signals used to build this list: @@ -31,7 +31,7 @@ further wins — skip unless code changes), `SKIPPED` (out of rotation). | 11 | Per-candidate indirect scoring | `src/ts_driven.cpp`, `src/ts_fitch.cpp`, `src/ts_fitch_na_incr.h` | At memory-bandwidth ceiling (~23 ns / 75 tips). T-075 confirmed no further wins. | 23 ns / candidate (75 t) | 2026-03-18 | AT-LIMIT | | 12 | R-loop search engine (`MaxParsi`) | `R/MaximizeParsimony.R`, `R/TreeSearch.R` | < 0.5 % of wall time (Rprof, 2026-03-18). R is a passenger, not a bottleneck on hot path. | < 0.5 % | 2026-03-18 | AT-LIMIT | | 13 | **Standard-Fitch path (TNT-parity)** | `src/ts_tbr.cpp`, `src/ts_fitch.cpp`, `src/ts_simd.h`, `src/ts_tree.cpp` | The `-`→`?` path (has_na=FALSE) the TNT benchmark uses; ~20× faster/rep than NA. tbr_search self 25 %, SIMD 21 %, uppass 13 %, per-clip bookkeeping 18 % | Zhu2013 627 in 0.56 s/rep; total DLL 2.70 s/8rep | 2026-06-16 (r3) | PROFILED | -| 13a| → per-clip bookkeeping | `ts_tbr.cpp` (build_postorder, collect_*, compute_from_above), `ts_tree.cpp` | postorder rebuilt every clip+accept (5.2 %) + incremental down/uppass (6.4 %) — TNT minimises exactly this | — | 2026-06-16 | NEW (top code lever) | +| 13a| → per-clip bookkeeping | `ts_tbr.cpp` (build_postorder, collect_*, compute_from_above), `ts_tree.cpp` | postorder rebuilt every clip+accept (5.2 %) + incremental down/uppass (6.4 %) — TNT minimizes exactly this | — | 2026-06-16 | NEW (top code lever) | | 13b| → SIMD reduce / uppass arithmetic | `ts_simd.h`, `ts_fitch.cpp:54` | any_hit_reduce 21 % (compiler-optimal); uppass scalar loop 1.22× only | — | 2026-06-16 | AT-LIMIT | ## Notes on the ranking diff --git a/dev/profiling/kernel-speed-baseline.md b/dev/profiling/kernel-speed-baseline.md index 4eaf5f23b..3197eb943 100644 --- a/dev/profiling/kernel-speed-baseline.md +++ b/dev/profiling/kernel-speed-baseline.md @@ -29,7 +29,7 @@ full `edge_set[below]` gather + reduce over all `total_words = n_blocks × n_sta - The high-payoff targets are **(a) reducing words gathered/reduced per candidate** (incremental cross-candidate update so successive rerootings don't re-gather the full edge set; or a compacter edge_set footprint to cut cache misses) and **(b) the equal-length plateau processing** — an - equal-length candidate could perhaps be recognised/short-circuited without a full scan. + equal-length candidate could perhaps be recognized/short-circuited without a full scan. - ns/word grows with total_words (0.23 → 0.34 → 0.44 ns/word for 823 → 1227 → 1844 chars) = cache-miss pressure as the edge_set buffer outgrows cache. Footprint reduction cuts this directly. diff --git a/dev/profiling/log.md b/dev/profiling/log.md index 26a31d58c..71f4d7350 100644 --- a/dev/profiling/log.md +++ b/dev/profiling/log.md @@ -13,7 +13,7 @@ at the bottom of the file before saving. - Build: .vtune-lib mtime YYYY-MM-DD HH:MM (vs src/ HH:MM) - profvis: <2 % R overhead | top R line / [Port] finding> - VTune top 3: , , (module=TreeSearch.dll) -- Finding: [Port|Optimise|AT-LIMIT] short — verified Δ via micro-bench +- Finding: [Port|Optimize|AT-LIMIT] short — verified Δ via micro-bench - Filed: T-NNN row(s) in findings.md - Cleanup: result__ removed; .vtune-lib - Next reviewer: @@ -63,7 +63,7 @@ at the bottom of the file before saving. - ⚠ Caveat: 19.2 % is self-time only. fitch_na_score has SIMD callees (any_hit_reduce_avx2 0.309s, 9.6 %) that are shared with the incremental evaluation path — unknown fraction comes from full_rescore vs incremental. [Unknown source file] 2.076 s (39 %) includes inlined code from both paths. Full_rescore **inclusive time** is plausibly 22–30 %. The prior S-PROF round 7 estimate of 28 % was likely inclusive time and is not contradicted by the 19.2 % self-time measurement — they measure different things. - full_rescore at line 1138 (acceptance) >> line 563 (entry): ratchet-driven TBR accepts ~100–200 moves per call from perturbed trees vs 1 entry call, so ~99% of full_rescore time is the acceptance-path T-300 target - Source-line attribution for lines 1138/1283 not available via software sampling (inlined into [Unknown]). -- Finding: [Optimise] T-300 is confirmed: full_rescore after accepted move ≥ 19.2 % of DLL time (inclusive estimate 22–30 %). Incremental path (fitch_na_pass3_score + incr_uppass + incr_downpass = 12.2 % self) already costs less per call. T-300 (in-flight by parallel agent) is justified — predicted gain 15–30 % of DLL time. +- Finding: [Optimize] T-300 is confirmed: full_rescore after accepted move ≥ 19.2 % of DLL time (inclusive estimate 22–30 %). Incremental path (fitch_na_pass3_score + incr_uppass + incr_downpass = 12.2 % self) already costs less per call. T-300 (in-flight by parallel agent) is justified — predicted gain 15–30 % of DLL time. - Filed: T-300 row in findings.md (unverified — micro-bench pending T-300 implementation) - Cleanup: result_tbr-rescore_20260519/ removed; .vtune-lib-20260519061049 deleted - Next reviewer: After T-300 lands — re-run this driver to verify fitch_na_score drops from 18.2 % toward the incremental path baseline. Also look at ts::simd::any_hit_reduce_avx2 (9.6 %) as next T-300-independent target. @@ -107,7 +107,7 @@ path the TNT benchmark actually compares against. bit-identical (value+changed flag) but only **1.22×** at n_states=4, and the 4-wide path does NOT trigger for 2-state (binary) morph chars → ~1 % wall, not worth the incremental-uppass correctness risk. - [Optimise, modest] Per-clip/accept allocation churn (~3-4 %): compute_from_above, + [Optimize, modest] Per-clip/accept allocation churn (~3-4 %): compute_from_above, collect_main_edges/collect_subtree_edges, validate_topology heap-alloc std::vector scratch per call (_M_realloc_append 1.1 %). Extend existing prealloc pattern (work_stack/saved_postorder/clip_actives_buf). Low risk. @@ -269,7 +269,7 @@ last_focus: 13 Focus: the still-open T-P5c (ratchet internals). Lead: the ratchet perturbed-weight search runs on the scalar scorer path (ts_tbr.cpp:2129) because MIXED-mode upweight_mask disables use_flat+x4 — the one hot path the whole -flat/x4/edge-set/getenv program never optimised. thorough/intensive/large use +flat/x4/edge-set/getenv program never optimized. thorough/intensive/large use ratchetPerturbMode=2 (MIXED); .AutoStrategy routes the >=65t,>=100char roster there. Method: env-gated std::chrono (TS_RATCHET_TIMING) around ratchet_search's 3 diff --git a/dev/profiling/reeval/disc_nstates.R b/dev/profiling/reeval/disc_nstates.R index ec4887cab..e8d33311e 100644 --- a/dev/profiling/reeval/disc_nstates.R +++ b/dev/profiling/reeval/disc_nstates.R @@ -120,4 +120,4 @@ cat(sprintf(" ratio totns(nStates=%d)/totns(nStates=%d) = %.2fx (proporti max(states) / min(states))) saveRDS(res, file.path("dev/profiling/reeval", sprintf("disc_nstates_%dt.rds", nTip))) -cat("* SPR/9 col = totns normalised to n_states=9 (flat if ~equal across rows).\n") +cat("* SPR/9 col = totns normalized to n_states=9 (flat if ~equal across rows).\n") diff --git a/dev/profiling/s7-fastpath-sizing.md b/dev/profiling/s7-fastpath-sizing.md index f92a177fc..649ae5394 100644 --- a/dev/profiling/s7-fastpath-sizing.md +++ b/dev/profiling/s7-fastpath-sizing.md @@ -20,7 +20,7 @@ Two linked measurements: bookkeeping strip (category (iii): the flat scorer drops CharBlock deref + weight-multiply + `active_mask` check — byte-identical on unit-weight data) roughly triples the SPR saving to **~17% of the SPR scan ≈ 4.7% whole-search** (Zhu2013). The SPR scan is only ~19–23% of - candidates, and the dominant reroot path is *already* specialised (branch-free x4 kernel when + candidates, and the dominant reroot path is *already* specialized (branch-free x4 kernel when `all_weight_one`; a skip-predicate-hoisted scalar loop otherwise). Because branch/bookkeeping cost is **fixed** (data-/size-independent) while the scorer's gather grows with n_tips, this fraction *shrinks* at 5432 scale. **Not a per-move lever (~1.05×, not 10×).** The residual per-candidate @@ -73,7 +73,7 @@ a **per-clip**, platform-specific cost, not per-candidate orchestration. ### Experiment — controlled A/B, trajectory-identical by construction -Runtime toggle `TS_EW_STRIP` (uncommitted probe in `ts_tbr.cpp`, default OFF) selects a specialised +Runtime toggle `TS_EW_STRIP` (uncommitted probe in `ts_tbr.cpp`, default OFF) selects a specialized SPR loop keeping only the identity skip + **the same** `fitch_indirect_length_cached` scorer + accept, removing all (ii) branches. Because the removed branches never fire in plain EW, the candidate sequence and every strict-`<` accept are identical → **score + `candidates_evaluated` diff --git a/dev/profiling/s7-land.md b/dev/profiling/s7-land.md index 5f615341f..f631387a4 100644 --- a/dev/profiling/s7-land.md +++ b/dev/profiling/s7-land.md @@ -88,7 +88,7 @@ Wortley2006 / Zanol2014 / Zhu2013, gaps `-`→`?`, ≥2 seeds (EW) / 3 seeds (IW instantiation actually executed — `flat=450/450` (Wortley), `flat=803/803` (Zhu), `cached=1220/1220` (Zanol); i.e. Wortley2006 AND Zhu2013 are `all_weight_one` and drive `UseFlat=true`. A deliberately- wrong instantiation (data-dependent always-≥1 corruption) DIVERGES on all three (flat and cached), - proving the specialised path's output drives the search — not a no-op that passes the gate falsely + proving the specialized path's output drives the search — not a no-op that passes the gate falsely (the false-0% trap from the sizing doc). The wrong instantiations are emitted ONLY under a compile flag (`-DTS_EW_MONO_WRONG`), so a PRODUCTION binary has no env-var path to a corrupted scorer: verified that `TS_EW_MONO_WRONG=1` is INERT on the default build, and the divergence reproduces only diff --git a/dev/red-team/escalation-backlog.md b/dev/red-team/escalation-backlog.md index b4908a3a7..6c2911c10 100644 --- a/dev/red-team/escalation-backlog.md +++ b/dev/red-team/escalation-backlog.md @@ -243,8 +243,8 @@ missing-data correction, `R/tree_length.R:144-156`: `f = 1 + r·(nTaxa−obs)/ob called the kernel via `ts_fitch_score(..., min_steps, concavity)` — plain IW, no XPIWE args. Two different objectives by construction. The round further confirmed (i) `TreeLength(extended_iw = FALSE)` == kernel plain IW exactly, and (ii) kernel XPIWE == an independent `TreeLength()` XPIWE -rescore of a real `MaximizeParsimony(concavity = 10)` result, exactly — so production optimises -and reports the *same* objective in both modes. There is no optimise-vs-report mismatch. +rescore of a real `MaximizeParsimony(concavity = 10)` result, exactly — so production optimizes +and reports the *same* objective in both modes. There is no optimize-vs-report mismatch. **The arithmetic, re-confirmed independently 2026-07-28.** The 2026-07-27 re-queue treated the "non-rational" `0.16573` as its strongest evidence, reasoning that inverting `x/(x+10) = 0.16573` diff --git a/dev/red-team/findings-archive.md b/dev/red-team/findings-archive.md index a66717725..c3ed9bccc 100644 --- a/dev/red-team/findings-archive.md +++ b/dev/red-team/findings-archive.md @@ -79,7 +79,7 @@ moment of filing. This is expected for a directed pre-commit round; it is not st | T-339 | P2 | 6 (R↔C++ interface) | `nThreads = 0` silently ran serial instead of the documented "auto-detect cores" behaviour, on BOTH `MaximizeParsimony()` and `Resample()` | **fixed `b1482997`, then LOST IN A MERGE and reapplied by `ec3ce46e`** — ⚠ **both SHAs are ancestors of HEAD, so ancestry proves nothing here**: the merge `ef54f473` ("Rcpp crlf") produced a tree byte-identical to its first parent for every file the fix branch touched, silently reverting it, and `ec3ce46e` put it back. Confirmed by content at HEAD instead: all three dispatch gates now read `nThreads != 1` (`src/ts_rcpp.cpp:1886`, `:1911`, `:2399`) and auto-detect does `hardware_concurrency() - 1` floored at 1, matching the docs. **This row is the standing example of why archiving must verify content, not the cited SHA.** Regression tests assert `nThreads = 0` and `= 2` byte-identical via per-replicate seeding (not wall-clock) in `test-ts-parallel.R`/`test-ts-resample.R` | | T-340 | P2 | 6 (R↔C++ interface) | `concavity` accepted as a character value with no normalization → silently wrong IW scores (`"10"` entered IW mode with `min_steps` left all-zero, so homoplasy went uncorrected) or silent EW fallback (case-variant `"Profile"`/`"prof"`) | **fixed `f149cf2e`** (merge of `fix/t340-concavity-normalization`) — confirmed by content at HEAD: `MaximizeParsimony()` now normalizes once, up front (`R/MaximizeParsimony.R:983-992`), routing the profile test through the pre-existing lenient `.UseProfile()` and putting everything else through `as.numeric()` with an explicit `length != 1 \|\| is.na` guard that **errors** naming the three legal forms — the single-fix framing the opus verifier recommended, closing both symptoms at one site. Case-insensitive profile matching also live at `:1422` (`pmatch(tolower(concavity), "profile", -1L)`) | | T-341 | P3 | 6 (R↔C++ interface) | `maxReplicates < 1` unchecked in R → search ran zero replicates and silently returned the random starting tree tagged `score = -1` | **fixed `46c3806c`** — confirmed by content at HEAD: `R/MaximizeParsimony.R:788-789` now errors ("`maxReplicates` must be a single integer of at least 1"), with the reasoning preserved as a comment at `:783` | -| T-342 | P3 | 6 (R↔C++ interface) | Fractional-weight datasets got a ~1260×-inflated "recommended replicates" warning (cosmetic only) — `nChars` read the `.ScaleWeight()`-integerised weight | **fixed `46c3806c`** — confirmed by content at HEAD: `R/MaximizeParsimony.R:1176` now reads `sum(at$weight)`, the **pre-scaling** attribute, with `:1171-1174` documenting why. Note for future readers: the line still *looks* like the bug (`nChars <- sum(...$weight)`) — the fix is which `weight` it reads, so judge it by the `at$` prefix, not the shape | +| T-342 | P3 | 6 (R↔C++ interface) | Fractional-weight datasets got a ~1260×-inflated "recommended replicates" warning (cosmetic only) — `nChars` read the `.ScaleWeight()`-integerized weight | **fixed `46c3806c`** — confirmed by content at HEAD: `R/MaximizeParsimony.R:1176` now reads `sum(at$weight)`, the **pre-scaling** attribute, with `:1171-1174` documenting why. Note for future readers: the line still *looks* like the bug (`nChars <- sum(...$weight)`) — the fix is which `weight` it reads, so judge it by the `at$` prefix, not the shape | | T-343 | P3 | 6 (R↔C++ interface) | Four documented `SearchControl()` parameter ranges were unvalidated in R, unlike the four sibling parameters guarded immediately above them | **fixed `46c3806c`** — confirmed by content at HEAD: the `R/SearchControl.R` guard block now covers `enumTimeFraction` (`:396-397`, `[0,0.5]`), the `[0,1]` fractions (`:402`) and `sectorMinSize <= sectorMaxSize` (`:409`), all rejecting rather than clamping, matching the established pattern | | T-370 | P3 | 9 (Wagner) | `random_topology_tree` (and `random_constrained_tree`'s root-level wiring) double-weighted the merged root edge, so the documented "uniformly random binary tree" was not uniform | **fixed `547f6ab2`** — `edge_children` now seeded with a single entry for the root's one true unrooted edge in both functions (`src/ts_wagner.cpp:864-867`, `:1156`), instead of one entry per root child. Empirically verified with a temporary Rcpp harness (not committed): for `n_tip=4` over 30k reps, the root edge was selected ~49.6% of the time pre-fix (expected 1/2 under the bug) vs ~33.3% post-fix (uniform 1/3, χ² p=0.77) against the other two star edges. `resolve_randomly` correctly left untouched, per the finding's carve-out. Full Wagner/random-tree test suite green | @@ -95,7 +95,7 @@ Archive row template (one line, compressed from the findings.md row): | T-359 | P3 | 7 (Shiny) | Tree-space mapping-quality `future_promise()` has no session-end cancellation — the same lifecycle gap T-311 fixed for the search worker | fixed (see commit history) — archived 2026-08-04 at tracker migration; full original row: `git show findings-md-final:dev/red-team/findings.md` | | T-362 | P3 | 8 (Test suite health) | FIXED (this session) — awaiting `tidy` archive. A Tier-1 (CRAN-run) test file takes ~16.6 s against its documented < ~2 s budget, running four full `MaximizeParsimony()` searches on CRAN machines — T-351's fix moved the two worst blocks out but neve… | fixed (see commit history) — archived 2026-08-04 at tracker migration; full original row: `git show findings-md-final:dev/red-team/findings.md` | | T-364 | P1 | 9 (Wagner) | FIXED `7685bf07` + `796a29d3` — awaiting `tidy` archive. T-370 is this same defect filed twice; both are closed by these two commits, and the two earlier branch fixes (`bdc32fb2`, `355c4196`) are superseded, not merged. Enforcement now goes throug… | fixed `7685bf07` — archived 2026-08-04 at tracker migration; full original row: `git show findings-md-final:dev/red-team/findings.md` | -| T-384 | P2 | 9 (Wagner/constraints) | FIXED `4c66a554` — awaiting `tidy` archive. See the resolution at the end of this row, which also corrects three of this row's own reachability claims. `map_constraint_nodes()` is rooting-blind: it accepts a constraint only when the canonicalised sp… | fixed `4c66a554` — archived 2026-08-04 at tracker migration; full original row: `git show findings-md-final:dev/red-team/findings.md` | +| T-384 | P2 | 9 (Wagner/constraints) | FIXED `4c66a554` — awaiting `tidy` archive. See the resolution at the end of this row, which also corrects three of this row's own reachability claims. `map_constraint_nodes()` is rooting-blind: it accepts a constraint only when the canonicalized sp… | fixed `4c66a554` — archived 2026-08-04 at tracker migration; full original row: `git show findings-md-final:dev/red-team/findings.md` | | T-365 | P2 | 9 (Wagner/PolEscapa) | `LengthAdded()` / `PolEscapa()` with `concavity = "profile"` index the *pre*-`PrepareDataProfile` contrast and token space against the *post*-prepare phyDat — a documented option that errors out on any character that is not plain binary. | fixed (see commit history) — archived 2026-08-04 at tracker migration; full original row: `git show findings-md-final:dev/red-team/findings.md` | | T-366 | P3 | 3,9 (Prune-reinsert/Wagner) | FIXED `d94d76b0` — awaiting `tidy` archive. `expand_and_reinsert` seeded `prelim` in the NA (Brazeau) regime and then patched it in the standard-Fitch regime, so insertion edge sets were mixed-regime on inapplicable-bearing data. | fixed `d94d76b0` — archived 2026-08-04 at tracker migration; full original row: `git show findings-md-final:dev/red-team/findings.md` | | T-367 | P3 (dormant tripwire) | 9 (Wagner) | `wagner_incremental_rescore`'s Phase-2 uppass early termination is unsound, leaving `final_` stale — harmless today only because no caller reads `final_` before its next full rescore. | fixed (see commit history) — archived 2026-08-04 at tracker migration; full original row: `git show findings-md-final:dev/red-team/findings.md` | diff --git a/dev/red-team/heavy-tests/hsj-paper-oracle.R b/dev/red-team/heavy-tests/hsj-paper-oracle.R index be4ccf268..06667f9dc 100644 --- a/dev/red-team/heavy-tests/hsj-paper-oracle.R +++ b/dev/red-team/heavy-tests/hsj-paper-oracle.R @@ -39,7 +39,7 @@ # and the code does not implement it" -- was wrong, and cost a round of work. # `score_hierarchy_block()`'s a(n)/p(n) recurrences ARE Algorithm 1 lines 6-7, # term for term. The two-state DP was already there and was already -# rooting-invariant: its branch costs are symmetric and it minimises over the +# rooting-invariant: its branch costs are symmetric and it minimizes over the # root's own state. The defect was never in the DP; it was that `d(u, v)` was # read off `fitch_label_char()`'s directional resolution. # diff --git a/dev/red-team/log.md b/dev/red-team/log.md index 6d8901573..fefd49bf0 100644 --- a/dev/red-team/log.md +++ b/dev/red-team/log.md @@ -96,9 +96,9 @@ notes: Rotation `(13 mod 13)+1 = 1`. Legend reconciled at round start: no rung h **THE CHARTERED QUESTION WAS ANSWERED, AND THE ANSWER IS NEGATIVE — recorded as a genuine result, not an absence of one.** The finder built two independent reference implementations sharing no code with `src/ts_fitch*` — a from-scratch Sankoff DP for EW, and a from-scratch brute-force applicability-enumeration + region DP for BGS inapplicable scoring — and cross-checked the *absolute* output of the real `score_tree()` against them across ~3000 datasets (multi-state, ambiguous, `{-,X}`/`{-}`/`?`-dense inapplicable, degenerate, deep-pectinate and shallow-balanced). 0 mismatches wherever the criterion is tie-break-free; the one-sided never-below-minimum bound held 1000/1000 including 500 `{-,X}`-dense cases; rooting-invariance held on every edge across 150+1500+576 test configurations; block-boundary additivity held across nChar ∈ {1,5,63,64,65,70,130}; IW/profile C++-vs-R agreement held 0/900. **The shared-machinery correctness question for area 1 is now closed at this tier** — a future revisit needs either a genuinely new angle (the finder's suggestion: cross-check `vroot_cache`/`below_actives_cache` candidate-scoring values against an independent rescore of the applied move, rather than the reconstruction layer already covered here) or an escalation to fable if a *new* delta lands that this round's coverage doesn't reach. -**Incidental product of the chartered work: the BGS criterion this engine implements is now pinned down in writing for the first time in this repo.** `min` over applicability reconstructions that are Fitch-optimal for the binary applicability character, of `sum(within-region Fitch length) + (tip-bearing regions - 1)`, applicable-preferred tie-break on `{-,X}` tips. Two consequences that will look like bugs to a future reviewer and are not, now both encoded as assertions in the new `tests/testthat/test-ts-na-oracle.R`: (1) the engine can exceed the unconstrained minimum-cost reconstruction (17/500 definite-applicability cases, 30/500 with `{-,X}`) because Fitch minimises applicability *changes*, not *regions* — exactly what Goloboff et al. 2021 critique and what `inapplicable = "xform"` exists to address; (2) two applicable tips with the same state separated by inapplicable tips score 1, not 0. +**Incidental product of the chartered work: the BGS criterion this engine implements is now pinned down in writing for the first time in this repo.** `min` over applicability reconstructions that are Fitch-optimal for the binary applicability character, of `sum(within-region Fitch length) + (tip-bearing regions - 1)`, applicable-preferred tie-break on `{-,X}` tips. Two consequences that will look like bugs to a future reviewer and are not, now both encoded as assertions in the new `tests/testthat/test-ts-na-oracle.R`: (1) the engine can exceed the unconstrained minimum-cost reconstruction (17/500 definite-applicability cases, 30/500 with `{-,X}`) because Fitch minimizes applicability *changes*, not *regions* — exactly what Goloboff et al. 2021 critique and what `inapplicable = "xform"` exists to address; (2) two applicable tips with the same state separated by inapplicable tips score 1, not 0. -**Eight findings filed, none touching the shared-scoring correctness question itself — all either perf, latent-unreachable, or a documentation/invariant gap.** RT-A1-02 (#77, med): the EW dirty-buffer allocation was `static thread_local`-optimised, then reverted by the emutls/MinGW fix (`d6fa51293`) for correctness, and the cost was never recovered on the NA hot path (`exact_verify_sweep`, ~97.7% of native-NA wall per this project's Mission-B profiling) — same class exists in 4 sites across `ts_fitch.cpp`/`ts_fitch_na_dirty.h`/`ts_fitch_na_incr.h`. RT-A1-08 (#83, upgraded to med): `TreeLength()` defaults to extended IW (XPIWE) but `PrepareData()`/`EdgeListScore()` — the documented default `TreeScorer` for `TreeSearch()`/`Ratchet()`/`Jackknife()` — always scores plain IW with no way to opt into XPIWE, so the two nominally-identical "score this tree at concavity k" paths silently disagree by default (600/600 measured mismatches, resolved by explicit `extended_iw=FALSE`). RT-A1-01 (#76, low): a detached-tip OOB guard present in `fitch_na_score` (added after a real ASAN abort) is missing from both incremental NA sibling loops — latent, currently unreachable. RT-A1-03/04/05/06/07 (#78-#82, low): residual per-call allocations in two "allocation-free"-labelled functions; a negative-delta path in `precompute_profile_delta` whose bounded-scorer-bail soundness rests on an unstated, unasserted invariant; an unproven (7500+ fuzz cases, 0 counterexamples, no proof) equivalence between two different `ss_app` derivations for EW vs IW/profile NA scoring; a diagnostic-only export whose own internal sum doesn't reconcile because it omits `precomputed_steps`; and two harmless dead/unreachable code fragments. +**Eight findings filed, none touching the shared-scoring correctness question itself — all either perf, latent-unreachable, or a documentation/invariant gap.** RT-A1-02 (#77, med): the EW dirty-buffer allocation was `static thread_local`-optimized, then reverted by the emutls/MinGW fix (`d6fa51293`) for correctness, and the cost was never recovered on the NA hot path (`exact_verify_sweep`, ~97.7% of native-NA wall per this project's Mission-B profiling) — same class exists in 4 sites across `ts_fitch.cpp`/`ts_fitch_na_dirty.h`/`ts_fitch_na_incr.h`. RT-A1-08 (#83, upgraded to med): `TreeLength()` defaults to extended IW (XPIWE) but `PrepareData()`/`EdgeListScore()` — the documented default `TreeScorer` for `TreeSearch()`/`Ratchet()`/`Jackknife()` — always scores plain IW with no way to opt into XPIWE, so the two nominally-identical "score this tree at concavity k" paths silently disagree by default (600/600 measured mismatches, resolved by explicit `extended_iw=FALSE`). RT-A1-01 (#76, low): a detached-tip OOB guard present in `fitch_na_score` (added after a real ASAN abort) is missing from both incremental NA sibling loops — latent, currently unreachable. RT-A1-03/04/05/06/07 (#78-#82, low): residual per-call allocations in two "allocation-free"-labelled functions; a negative-delta path in `precompute_profile_delta` whose bounded-scorer-bail soundness rests on an unstated, unasserted invariant; an unproven (7500+ fuzz cases, 0 counterexamples, no proof) equivalence between two different `ss_app` derivations for EW vs IW/profile NA scoring; a diagnostic-only export whose own internal sum doesn't reconcile because it omits `precomputed_steps`; and two harmless dead/unreachable code fragments. **ONE HAIKU REFUTATION OVERTURNED ON RE-VERIFICATION — record the pattern, not just the outcome.** RT-A1-08 was initially routed to the haiku batch (all 8 candidates were low/med severity, per this skill's severity-matched-verification rule — no `sev:high` this round to trigger automatic peer-tier routing). The haiku verifier refuted it by arguing `extended_iw` is a legitimate, documented feature — true, and not the claim. The actual claim was a **default mismatch** between two commonly-used paths for the same nominal computation. Per [[redteam-reverify-flawed-refutes]] and this project's standing note to route library-fact verdicts above haiku, re-verified with a sonnet pass, which traced the full default chain (`R/tree_length.R` → `R/PrepareData.R` → `src/ts_rcpp.cpp`'s `xpiwe=false` C++ default) and confirmed REAL, overturning the refutation. **Generalisable point for this severity-routing rule: "not sev:high" does not mean "safe to haiku" when the claim is a library/default-semantics fact rather than a straightforward code-pattern check** — this round's median finding (allocation exists, guard missing) was well within haiku's competence, but the one finding requiring cross-file default-chain reasoning was not, independent of its filed severity. @@ -125,7 +125,7 @@ notes: Dispatched directly at the user's request ("sounds like we're ready for a **Ruled out this round, do not re-trace:** constraint-metadata resync on TBR / prune-reinsert / ratchet / sector *revert* paths (all correct — A13-09/A13-10 are specifically about narrower exit paths, not a general resync failure); per-worker `ConstraintData` copies in parallel mode (T-336 still holds); split canonicalisation agreement between `ts_splits.cpp::canonicalize_split` and `build_constraint` post-T-384 (now largely cosmetic for mapping, polarity handled symmetrically); `structurally_valid()` (T-333) remains a complete check as documented. -**Not yet examined, flagged as the obvious next seam:** `Resample.R`/`SuccessiveApproximations.R` also call `.PrepareConstraint` (`R/Resample.R:311`, `R/SuccessiveApproximations.R:71`) and route to `ts_resample_search`/`ts_parallel_resample`/`ts_successive_approx` — not traced this round for whether they carry the fuse/sector/pool gates at all. Also latent, not yet a bug: `impose_constraint`'s repair always targets making the *canonical* side the clade regardless of which side is cheaper to realise (`ts_constraint.cpp:741-753`), which combined with A13-07 probably explains part of the heuristic's weak convergence on nested/complex constraints. +**Not yet examined, flagged as the obvious next seam:** `Resample.R`/`SuccessiveApproximations.R` also call `.PrepareConstraint` (`R/Resample.R:311`, `R/SuccessiveApproximations.R:71`) and route to `ts_resample_search`/`ts_parallel_resample`/`ts_successive_approx` — not traced this round for whether they carry the fuse/sector/pool gates at all. Also latent, not yet a bug: `impose_constraint`'s repair always targets making the *canonical* side the clade regardless of which side is cheaper to realize (`ts_constraint.cpp:741-753`), which combined with A13-07 probably explains part of the heuristic's weak convergence on nested/complex constraints. **Doc fixes applied inline (not filed, comment-only):** `src/ts_constraint.h` — `regraft_violates_constraint`'s doc no longer references a nonexistent `above` parameter and now states explicitly it's screening-only (every caller must re-verify the applied move); `impose_constraint`'s doc replaced the false "after return, all constraint splits are displayed" contract with the actual heuristic contract (bail-out cap, per-move revert, return value doesn't distinguish repaired-from-gave-up). @@ -149,7 +149,7 @@ tier: sonnet (Sonnet 5) yield: **2 filed (#41 routing channel, #42 scope coverage) + 10 doc corrections applied inline + 1 false-and-hazardous source comment corrected**, from 8 candidates; 1 half-refuted (`area:N` label "inconsistency"); 1 **inverted** on verification (`rearrange.cpp`); and **1 finder ACTION reverted** notes: Rotation `(11 mod 13) + 1 = 12`. Legend reconciled at round start -- no rung has moved since 2026-07-27, so no version-bump re-eligibility fired and `escalation-backlog.md` reopened nothing. Last visit (2026-07-03) **yielded** -> stayed at sonnet with a fresh agent per the yielding-seam rule; the within-rung 4.6 -> 5 bump is incidental, not the reason. No `needs-escalation` label on any issue, area 12 included. Verification was **mostly orchestrator-mechanical rather than a verifier pass**, deliberately: almost every candidate reduced to "is file X in any scope row" or "is symbol Y registered", which a verifier would answer by re-running the same greps. A haiku batch took the three claims that rested on a file's *self-description*, which is exactly where a grep is not evidence. -**THE HEADLINE IS AN ACTION THE FINDER TOOK, NOT A CLAIM IT MADE -- and it is the round's most reusable result.** Following the skill's *Escalation tracking* rule ("promote it to a label the moment it gets filed as an issue"), the finder labelled #18 and #19 `needs-escalation` because `escalation-backlog.md` item 7 names them. **The orchestrator reverted both the same round** (verified zero issues now carry the label, restoring the pre-round state). The rule is written about the class where the ask is *more capability* -- a high-severity finding a finder could not pin down. Item 7 is the other class the file admits, a **cross-area-routed** residual, and its own text says *"The ask is a sequencing decision, not a review."* Two harms, the second the real one: (i) `needs-escalation` encodes only "dispatch at `opus`+", and area 13's `start_tier` is **already** `opus`, so it changed no decision; (ii) step 3's label branch reads "if non-empty, dispatch at opus regardless, **no further reading needed for this check**" -- so a label hit *suppresses* reading the backlog row that holds the actual ask. Labelling converted a sequencing question into a tier answer and then hid the question. **Generalised and filed as #41: the rotation's routing state is TIER-ONLY.** There is no channel for a non-tier verdict, and two distinct ones are now on record -- item 7's sequencing ask, and the *work-shape* verdicts of areas 10 and 13. Mis-encoding either into the tier channel is worse than leaving it in prose, because prose is at least read as prose. +**THE HEADLINE IS AN ACTION THE FINDER TOOK, NOT A CLAIM IT MADE -- and it is the round's most reusable result.** Following the skill's *Escalation tracking* rule ("promote it to a label the moment it gets filed as an issue"), the finder labelled #18 and #19 `needs-escalation` because `escalation-backlog.md` item 7 names them. **The orchestrator reverted both the same round** (verified zero issues now carry the label, restoring the pre-round state). The rule is written about the class where the ask is *more capability* -- a high-severity finding a finder could not pin down. Item 7 is the other class the file admits, a **cross-area-routed** residual, and its own text says *"The ask is a sequencing decision, not a review."* Two harms, the second the real one: (i) `needs-escalation` encodes only "dispatch at `opus`+", and area 13's `start_tier` is **already** `opus`, so it changed no decision; (ii) step 3's label branch reads "if non-empty, dispatch at opus regardless, **no further reading needed for this check**" -- so a label hit *suppresses* reading the backlog row that holds the actual ask. Labelling converted a sequencing question into a tier answer and then hid the question. **Generalized and filed as #41: the rotation's routing state is TIER-ONLY.** There is no channel for a non-tier verdict, and two distinct ones are now on record -- item 7's sequencing ask, and the *work-shape* verdicts of areas 10 and 13. Mis-encoding either into the tier channel is worse than leaving it in prose, because prose is at least read as prose. **AREA 13 IS NEXT IN ROTATION AND CARRIES A STANDING "NOT A FINDER" VERDICT.** `(12 mod 13) + 1 = 13`, and area 13's 2026-07-03 entry closed with *"NEXT VISIT: NOT another finder -- a BOUNDED EXHAUSTIVE HARNESS"* on the `topology_spr`/`build_postorder`-guard equivalence. *Normal run* step 4 **reads** the entry containing that verdict and step 5 then launches a finder unconditionally -- the gap is that nothing gates step 5 on what step 4 read, which is why #41 proposes a step-4.5 gate rather than a new `focus-areas.md` column (a column would add a second place to look with the same failure mode). Only project-local mitigation available was applied: area 13's maturity rationale was rewritten so the verdict is unmissable at step 1. **Whoever dispatches area 13 must also decide harness-first vs #18/#19-first and record why** -- the harness plan predates both findings. Area 10 carries the same shape of verdict (a wall-matched Hamilton-class A/B, 2026-08-03). @@ -186,7 +186,7 @@ notes: Rotation `(10 mod 13) + 1 = 11`, taken as written -- the 2026-08-03 area- **T-402 ALSO FORCED AN AMENDMENT TO T-324, WHICH MATTERS FOR ANY FIXER OF THIS CLASS.** T-324 said `nni_perturb`'s `impose_constraint` "can repair it only if `nni_perturb_per > 0` and the heuristic succeeds". In the score-better-than-constrained regime it can **never** repair: `best_tree = tree` snapshots the **violating** start at `ts_nni_perturb.cpp:80` *before* repair, and the repaired legal (therefore worse-scoring) tree is rejected at `:120` by `accept = tbr_result.best_score < best_score`. The illegal score is an unbeatable baseline, so **a verify-and-revert gate of the T-390/T-391 shape does not fix this class**; and gating the pool capture alone leaves the pool empty at `maxReplicates = 1`, whereupon `MaximizeParsimony.R:1682-1684` falls back to returning the user's violating start anyway. Both traps recorded in T-402. Second-order finding worth its own note: the illegal score **evicts** compliant trees, so an 8-replicate run returned the violating tree even though replicates 2-8 each reached the true constrained optimum -- and the bug is *masked* when the supplied tree is bad, so it bites precisely when the user supplies a **good** one. -**T-403 IS THE PRECISE LIMIT OF THE FINDER'S OWN CLEAN ANSWER, AND BOTH HALVES SHOULD BE KEPT.** Asked whether collapse can violate a constraint, the finder answered **no** with a sound derivation: contracting an edge replaces a resolution by a polytomy whose Fitch length is the min over resolutions, so no character's length can rise and a "constraint characters at minimum length" constraint survives any contraction. That is correct **in the min-length sense** and should not be re-derived. But in the **displayed-split** sense -- which is what `@param constraint` promises the user -- it does: the protection at `ts_rcpp.cpp:2244-2269` is a bit-identical match against `consSplitMatrix` rows and has no access to `consZero` (computed at `MaximizeParsimony.R:134`/`:143`/`:153`, then **dropped** from the returned list at `:191-198`), so a constraint split realised by any node that is not exactly the 1-group is unprotected. Measured through the public API: `collapse = TRUE` -> **20/20 returned trees violate the constraint**; `collapse = FALSE` -> **0/20**. +**T-403 IS THE PRECISE LIMIT OF THE FINDER'S OWN CLEAN ANSWER, AND BOTH HALVES SHOULD BE KEPT.** Asked whether collapse can violate a constraint, the finder answered **no** with a sound derivation: contracting an edge replaces a resolution by a polytomy whose Fitch length is the min over resolutions, so no character's length can rise and a "constraint characters at minimum length" constraint survives any contraction. That is correct **in the min-length sense** and should not be re-derived. But in the **displayed-split** sense -- which is what `@param constraint` promises the user -- it does: the protection at `ts_rcpp.cpp:2244-2269` is a bit-identical match against `consSplitMatrix` rows and has no access to `consZero` (computed at `MaximizeParsimony.R:134`/`:143`/`:153`, then **dropped** from the returned list at `:191-198`), so a constraint split realized by any node that is not exactly the 1-group is unprotected. Measured through the public API: `collapse = TRUE` -> **20/20 returned trees violate the constraint**; `collapse = FALSE` -> **0/20**. **TOOLING RESULT THAT CONTRADICTS A STANDING PROJECT NOTE, AND SHOULD CHANGE HOW THIS CLASS IS HUNTED.** [[feedback-memory-checking]] said all local memory checking is blocked on Windows/MinGW (Dr. Memory, gflags, local ASan) and to use GHA CI for everything. That remains true of *sanitizers*, but **`-D_GLIBCXX_ASSERTIONS` works fine locally** -- it is libstdc++ hardened mode, not a sanitizer, so it needs no runtime library -- and it caught T-400's three OOB write sites **in one run**, where static tracing had taken far longer and still understated the bug. Inject via `R_MAKEVARS_USER` (not a `src/Makevars.win`, which `AGENTS.md` forbids leaving behind) and put it in **`PKG_CPPFLAGS`**, per [[makevars-pkgcxxflags-clobber]]. Given how much of this project's bug class is container-index OOB (T-323/T-328/T-400/WGN-13-C), **reach for this before dispatching an ASan CI round.** Memory updated this round. @@ -205,7 +205,7 @@ notes: Rotation pointed at area 11, but the 2026-07-28 round closed with "**HIGH **THE FIX CHOICE IS A REAL TRADE-OFF, MEASURED BOTH WAYS -- do not let a future round assume the obvious gate is free.** The finder's `ScoringMode::EW` gate is correct and complete (PROFILE already excluded via `ds.concavity = 1.0`, `ts_data.cpp:478`, verified in source not assumed) but costs **2.7x/2.9x wall**. A **guard-only** variant (snapshot, restore-and-return-false when `actual >= s0`) removes the non-monotonicity at **wall parity** and is byte-identical for EW -- but buys **no reach** (p=1.0 / p=0.51). The gate's reach gain is real but **not wall-matched** (fixed `maxReplicates`, so its arm consumed ~2.8x the wall), so per [[completeness-secondary-to-wallclock]] and [[budget-regime-equal-wall-valid]] it is unproven, and the gain traces to exact root-edge enumeration -- i.e. to fixing T-377 at the root edge -- not to refusing the bad move. -**T-377 REOPENED RATHER THAN RE-FILED, AND THIS IS THE ROUND'S PROCESS RESULT.** Candidate A10-07 was a rediscovery of T-377 (hierarchy-blind TBR scan), a row marked "**MEASURED, CLOSED (no adopted fix)**" on 2026-07-29. I nearly filed it as a new ID; the duplication grep caught it. **The row's own closing caveat had named its reopening condition** -- "sampled on two small (7-8 tip) matrices only; a much larger/sparser hierarchy-heavy matrix ... would be the thread to reopen if this resurfaces" -- **and the verifier's measurement fires it exactly**: at **25 tips / 60 characters**, returned HSJ trees are not even **SPR-local optima** under the accept-time objective (3/5 seeds `total_words > 0`, 2/5 all-hierarchy) against an **EW control at 0/5 on the same matrix**, while the **8-tip enumeration oracle still reached the optimum 12/12**. So the 2026-07-29 close was **correct at the size it was measured** and is superseded at realistic size, not contradicted -- record it that way, because a future round that reads the old verdict as simply wrong will distrust a sound measurement. **Lesson: a closed row that documents its own reopening condition is worth far more than a deleted one; the condition is what let a later round recognise the fire.** Augmented in place; do not close again without a measurement at >=25 tips. +**T-377 REOPENED RATHER THAN RE-FILED, AND THIS IS THE ROUND'S PROCESS RESULT.** Candidate A10-07 was a rediscovery of T-377 (hierarchy-blind TBR scan), a row marked "**MEASURED, CLOSED (no adopted fix)**" on 2026-07-29. I nearly filed it as a new ID; the duplication grep caught it. **The row's own closing caveat had named its reopening condition** -- "sampled on two small (7-8 tip) matrices only; a much larger/sparser hierarchy-heavy matrix ... would be the thread to reopen if this resurfaces" -- **and the verifier's measurement fires it exactly**: at **25 tips / 60 characters**, returned HSJ trees are not even **SPR-local optima** under the accept-time objective (3/5 seeds `total_words > 0`, 2/5 all-hierarchy) against an **EW control at 0/5 on the same matrix**, while the **8-tip enumeration oracle still reached the optimum 12/12**. So the 2026-07-29 close was **correct at the size it was measured** and is superseded at realistic size, not contradicted -- record it that way, because a future round that reads the old verdict as simply wrong will distrust a sound measurement. **Lesson: a closed row that documents its own reopening condition is worth far more than a deleted one; the condition is what let a later round recognize the fire.** Augmented in place; do not close again without a measurement at >=25 tips. **FOUR MORE CONFIRMED, TWO OF THEM DEMONSTRATED IN PURE R BEFORE ANY BUILD.** T-393: `RecodeHierarchy` derives a secondary's state space from observed **token strings** (`R/recode_hierarchy.R:103-106`), so `"{01}"` becomes a novel third state at Hamming 1 from *both* `"0"` and `"1"` -- `TreeLength(xform)` = **6** where every concrete resolution gives **5** and HSJ gives **5**, on data `MatrixToPhyDat` accepts and `ValidateHierarchy` passes; four such cells trip an `82 states (> 32)` warning where the truth is 17. T-394: the same line can admit **no** levels, making the whole XFORM score `Inf` -- reachable by **taxon-subsetting valid input** (`ValidateHierarchy` runs *before* both subset sites), and the consequence was corrected upward during verification: `MaximizeParsimony` does not silently return `Inf`, it **aborts** at `R/MaximizeParsimony.R:1717` because `diff(range(c(Inf, Inf)))` is `NaN` -- which will fire for **any** all-`Inf` pool and is worth fixing in the same patch. T-395: `ValidateHierarchy` rejects **every** nested hierarchy, so the **documented roxygen example is unusable** while `R CMD check` stays **green** (the constructor never validates, so the example constructs and is never scored) -- and verification found an **independent second dead route**, `HierarchyFromNames`'s `intersect(subIdx, supIdx)` being provably empty, so fixing only the validator would leave nesting silently flattened. T-396: HSJ over-charges a `"-"` secondary by `alpha/m` wherever the primary is not *certainly* absent -- **but the present-primary half is test-frozen** at `tests/testthat/test-ts-hsj.R:857` (`expect_equal(..., c(0, 1, 1, 0))`, with `:837-838` rationalising it) in direct contradiction to the kernel's own comment at `ts_hsj.cpp:135-145`, so **the tree holds two contradictory specifications and any fix breaks a shipped test**. That is a maintainer adjudication, not a fixer's call, and is why it sits at P2. @@ -273,7 +273,7 @@ notes: Re-visit at the same rung after the 2026-06-16 opus-4.8 round yielded; th **T-364 WAS CONSIDERED FOR AN INLINE FIX AND DELIBERATELY LEFT FILED.** The tempting fix — detect `cn == root` at step 3 and reshuffle, mirroring `random_wagner_tree`/`biased_wagner_tree` (`:770-779`, `:809-822`) — does not transplant. Those own their shuffle, so they can redraw; `AdditionTree` assembles a **complete** order in R (user prefix plus `sample(unlisted)`, `R/AdditionTree.R:68`) and calls the kernel once, so C++ cannot tell a user-pinned prefix from a filled tail and has nothing safe to reshuffle. A retry therefore belongs in the R layer and brings real design decisions with it (retry count; what to do when the whole order is pinned; warn versus error; the added cost on every constrained `AdditionTree` call). Nor is base-triple permutation sufficient on its own: it fixes the "exactly two of three inside" arrangements, but not "all three inside", and with nested splits no permutation of a fixed triple satisfies every split simultaneously. Maintainer's call, so filed rather than half-fixed. -**THE MOST IMPORTANT INLINE FIX IS A COMMENT.** `ts_prune_reinsert.cpp:453` stated "`wagner_incremental_rescore` maintains both `prelim` and `final_`". The `final_` half is false (T-367), and this is precisely the comment that would walk the next author back into the wrong-cost bug that existed before the edge-set rewrite. Replaced with an explicit "do NOT extend this to `final_`" note explaining the Phase-1/Phase-2 mismatch. Equally: the `fitch_score` call at `ts_wagner.cpp:449` now carries a **DO-NOT-DELETE** note, because the obvious-looking cleanup (the variable it initialised was dead) would silently break the seeding of the `i == 3` insertion step. +**THE MOST IMPORTANT INLINE FIX IS A COMMENT.** `ts_prune_reinsert.cpp:453` stated "`wagner_incremental_rescore` maintains both `prelim` and `final_`". The `final_` half is false (T-367), and this is precisely the comment that would walk the next author back into the wrong-cost bug that existed before the edge-set rewrite. Replaced with an explicit "do NOT extend this to `final_`" note explaining the Phase-1/Phase-2 mismatch. Equally: the `fitch_score` call at `ts_wagner.cpp:449` now carries a **DO-NOT-DELETE** note, because the obvious-looking cleanup (the variable it initialized was dead) would silently break the seeding of the `i == 3` insertion step. **VERIFICATION CHANGED THE RECORD ON THREE OF TEN — the value here was precision, not filtering.** Nothing was refuted outright, but: (1) **T-365 down, P1 to P2** — every demonstrated `concavity = "profile"` failure is a loud `Rcpp::exception`, and the verifier actively searched for a silent-wrong-numbers path and could not construct one, so the finder's implication of corrupted output is unsupported. The verifier also found a **third** failing variant the finder missed. (2) **T-364 up** — the finder scoped it as a T-324 mechanism note; the verifier found it silently breaks the public `AdditionTree()` entry point and flunks an existing test. (3) **T-366 down, med to P3** — the mechanism is real, but `pruneReinsertCycles = 0L` in both defaults with no preset override, so the path is opt-in and off everywhere shipped. **T-367's most quotable claim was REFUTED in magnitude and must not be recorded:** "stale `final_` made every edge below the break point cost the full `active_mask`" is wrong, because `simd::any_hit_reduce3` *unions* the two endpoint finals, so one stale-zero endpoint degrades the state set without zeroing it. Stale `final_` is a plausible contributor to the historical "+30% Wagner trees", not the explanation — an attribution error that would have entered the project's record unchallenged. The verifier also narrowed the finder's four "false comment" citations to the two that genuinely assert the invariant, one of them `#ifdef`-gated. @@ -885,8 +885,8 @@ kernel `ts_fitch_score` plain IW, **exactly** (8.566683 == 8.566683). (2) `attr(.,'score')` (XPIWE, since production sets `useXpiwe <- isTRUE(extended_iw) && is.finite(concavity) && !useProfile`, `R/MaximizeParsimony.R:813-839`) == an independent `TreeLength()` XPIWE rescore of the returned tree, **exactly** (1.521827). -So production **optimises and reports the same objective** in BOTH plain-IW and XPIWE -modes — no optimise-vs-report mismatch. (Numbers are tree-dependent; the three +So production **optimizes and reports the same objective** in BOTH plain-IW and XPIWE +modes — no optimize-vs-report mismatch. (Numbers are tree-dependent; the three equalities are not.) **Conclusion: NOT A BUG.** The signal raised in the area-9 entry below is resolved; do not carry it forward. **last_focus deliberately left at 9** — this was a targeted signal check, not a rotation round, so area 10 still earns a proper @@ -939,7 +939,7 @@ AdditionTree kernel's NA+IW score on Vinther2008 matches the documented IW refer divergence is purely IW per-character minimum on ~7 inapplicable chars (e.g. char 23 cs=3 ml=1: ref `2/12=0.16667`, `TreeLength=0.16573`). On Lobo (also NA) they agree → inapplicable-pattern dependent. The kernel is correct *per the package's stated contract*, so Wagner needs no change, -but if `TreeLength` is user-facing ground truth then `MaximizeParsimony` NA+IW optimises/reports +but if `TreeLength` is user-facing ground truth then `MaximizeParsimony` NA+IW optimizes/reports a subtly different objective than `TreeLength` for affected datasets. Root cause is in `TreeLength`/`MinimumLength`/`CharacterLength` (R-layer NA-IW scoring) — **not** area-9 files. `test-iw-scoring.R` asserts `TreeLength==reference` but only on Lobo, so the divergent case is diff --git a/dev/red-team/migration-map.tsv b/dev/red-team/migration-map.tsv index 35a5d0dba..b1209c212 100644 --- a/dev/red-team/migration-map.tsv +++ b/dev/red-team/migration-map.tsv @@ -11,7 +11,7 @@ T-359 archived dev/red-team/findings-archive.md Tree-space mapping-quality `futu T-362 archived dev/red-team/findings-archive.md FIXED (this session) — awaiting `tidy` archive. A Tier-1 (CRAN-run) test file takes ~16.6 s against its documented < ~2 s budget, running four full `MaximizeParsimony()` searches on CRAN machines — T-351's fix moved the two worst blocks out but neve… T-363 issue agent-issues/TreeSearch#4 Three R-level-API test files are `skip_on_cran()`-gated as "Tier 2", a tier the strategy doc has no slot for them in — so they silently never run on CRAN and cannot serve the dependency-regression purpose Tier 1 exists for T-364 archived dev/red-team/findings-archive.md FIXED `7685bf07` + `796a29d3` — awaiting `tidy` archive. T-370 is this same defect filed twice; both are closed by these two commits, and the two earlier branch fixes (`bdc32fb2`, `355c4196`) are superseded, not merged. Enforcement now goes throug… -T-384 archived dev/red-team/findings-archive.md FIXED `4c66a554` — awaiting `tidy` archive. See the resolution at the end of this row, which also corrects three of this row's own reachability claims. `map_constraint_nodes()` is rooting-blind: it accepts a constraint only when the canonicalised sp… +T-384 archived dev/red-team/findings-archive.md FIXED `4c66a554` — awaiting `tidy` archive. See the resolution at the end of this row, which also corrects three of this row's own reachability claims. `map_constraint_nodes()` is rooting-blind: it accepts a constraint only when the canonicalized sp… T-365 archived dev/red-team/findings-archive.md `LengthAdded()` / `PolEscapa()` with `concavity = "profile"` index the *pre*-`PrepareDataProfile` contrast and token space against the *post*-prepare phyDat — a documented option that errors out on any character that is not plain binary. T-366 archived dev/red-team/findings-archive.md FIXED `d94d76b0` — awaiting `tidy` archive. `expand_and_reinsert` seeded `prelim` in the NA (Brazeau) regime and then patched it in the standard-Fitch regime, so insertion edge sets were mixed-regime on inapplicable-bearing data. T-367 archived dev/red-team/findings-archive.md `wagner_incremental_rescore`'s Phase-2 uppass early termination is unsound, leaving `final_` stale — harmless today only because no caller reads `final_` before its next full rescore. diff --git a/dev/red-team/union-of-finals-bound-proof.md b/dev/red-team/union-of-finals-bound-proof.md index 04483284e..959bf1ed2 100644 --- a/dev/red-team/union-of-finals-bound-proof.md +++ b/dev/red-team/union-of-finals-bound-proof.md @@ -288,7 +288,7 @@ bound.** ("skip candidates whose bound $\ge$ current best"): being an upper bound, it can discard genuinely improving moves. Its only safe use is the current one — a heuristic *ranker* whose chosen move is **re-scored exactly** afterwards (the `temper` caller); even there it systematically - over-penalises attachment points where the simplified final dropped an MPR state. + over-penalizes attachment points where the simplified final dropped an MPR state. - Not exact on binary/unambiguous data (S4): explicit all-singleton, two-state, fully-resolved counterexample; the gate independently found over-counts on 34.8% and true-finals under-counts on 27.6% of binary candidates. diff --git a/dev/strategy.md b/dev/strategy.md index a972e1a85..39bb32326 100644 --- a/dev/strategy.md +++ b/dev/strategy.md @@ -400,7 +400,7 @@ confirmed zero benefit on score, MPT count, or topological diversity, with **GHA fixes committed to cpp-search:** - Spelling wordlist: added LCM, TREE's, speedup; removed 28 stale entries -- PrepareDataProfile/StepInformation codoc: `n_mc` 5000→100000 (stale Rd +- PrepareDataProfile/StepInformation codoc: `mcSamples` 5000→100000 (stale Rd from devtools::check_man() loading old installed version) - test-ts-parallel.R:85 flaky timeout: Vinther→Agnarsson (fast ARM64 completed 23-tip replicates before 1s timeout) diff --git a/inst/REFERENCES.bib b/inst/REFERENCES.bib index 8fd1e150c..dbfe661d5 100644 --- a/inst/REFERENCES.bib +++ b/inst/REFERENCES.bib @@ -355,7 +355,8 @@ @incollection{Goloboff2024 volume = {2}, pages = {119--183}, publisher = {CRC Press}, - doi = {10.1201/9780367823412-8}, + doi = {10.1201/9780367823412}, + comment = {Chapter's doi of 10.1201/9780367823412-8 is not registered with CrossRef}, volumes = {2} } @@ -496,6 +497,28 @@ @article{Mir2013 number = {1} } +@article{Bremer1988, + title = {The limits of amino acid sequence data in angiosperm phylogenetic reconstruction}, + author = {Bremer, Kare}, + year = {1988}, + volume = {42}, + pages = {795--803}, + doi = {10.1111/j.1558-5646.1988.tb02497.x}, + journal = {Evolution}, + number = {4} +} + +@article{Bremer1994, + title = {Branch support and tree stability}, + author = {Bremer, Kare}, + year = {1994}, + volume = {10}, + pages = {295--304}, + doi = {10.1111/j.1096-0031.1994.tb00179.x}, + journal = {Cladistics}, + number = {3} +} + @article{Muller2005, title = {The efficiency of different search strategies in estimating parsimony jackknife, bootstrap, and {{Bremer}} support}, todo_author = {M{\"u}ller, Kai F.}, diff --git a/inst/WORDLIST b/inst/WORDLIST index 733e626c7..ba0619d68 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -200,12 +200,15 @@ ac aculiferan archaeopriapulid autapomorphic +bgs +bipartitions bristleworm cdef cdot cf characterwise cla +clade's cleanup codecov colourblind @@ -283,6 +286,7 @@ rescoring reweighting rightarrow run's +search's secondaries segfaulted sim diff --git a/inst/datasets/Wills2012.nex b/inst/datasets/Wills2012.nex index 048127d01..b51864a03 100644 --- a/inst/datasets/Wills2012.nex +++ b/inst/datasets/Wills2012.nex @@ -577,7 +577,7 @@ BEGIN NOTES; TEXT CHARACTER=68 TEXT="Lemburg (1999) recognized this as a synapomorphy of lorciferans and extant priapulids."; TEXT CHARACTER=69 TEXT=" The presence of a flagellum in spermatozoa is a metazoan symplesiomorphy, but a flagellum is lacking from the spermatozoa of nematodes and nematomorphs (Schmidt-Rhaesa 1998)."; TEXT CHARACTER=64 TEXT="Lemburg (1999) recognizes the presence of this character as a synapomorphy of Megaintroverta (Priapulopsis + Acanthopriapulus + Priapulus)."; - TEXT CHARACTER=63 TEXT="Lemburg (1999) recognised the presence of this character as a synapomorphy of (extant) Priapulida. However, it has since been demonstrated that the larvae of nematomorphs also possess six pharyngeal retractor muscles (Kristensen, 2003; Muller et al., 2004). "; + TEXT CHARACTER=63 TEXT="Lemburg (1999) recognized the presence of this character as a synapomorphy of (extant) Priapulida. However, it has since been demonstrated that the larvae of nematomorphs also possess six pharyngeal retractor muscles (Kristensen, 2003; Muller et al., 2004). "; TEXT CHARACTER=56 TEXT="Lemburg (1999) recognized this as a synapomorphy of Introverta (Nematoda + Nematomorpha + Kinorhyncha + Loricifera + Priapulida)."; TEXT CHARACTER=58 TEXT="Lemburg (1999) recognizes the presence of this character as a synapomorphy of (extant) Priapulida."; diff --git a/man/AdditionTree.Rd b/man/AdditionTree.Rd index dcc28b8ad..1205fdbcb 100644 --- a/man/AdditionTree.Rd +++ b/man/AdditionTree.Rd @@ -61,7 +61,7 @@ order in which taxa were added and is not necessarily \code{sequence[1]}; parsimony scores are unaffected by rooting, so root the result yourself with \href{https://ms609.github.io/TreeTools/reference/RootTree}{\code{TreeTools::RootTree()}} if the position of the root matters to you. With fewer than four -taxa there is nothing to optimise, and a pectinate tree of the dataset's taxa +taxa there is nothing to optimize, and a pectinate tree of the dataset's taxa is returned without consulting \code{sequence} or \code{constraint}. } \description{ diff --git a/man/Bremer.Rd b/man/Bremer.Rd new file mode 100644 index 000000000..e9974b0d9 --- /dev/null +++ b/man/Bremer.Rd @@ -0,0 +1,142 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/Bremer.R +\name{Bremer} +\alias{Bremer} +\title{Bremer support (decay index)} +\usage{ +Bremer( + tree, + dataset, + method = c("constraint", "pool"), + maxBremer = Inf, + optimalScore = NULL, + format = "numeric", + concavity = Inf, + extended_iw = TRUE, + xpiwe_r = 0.5, + xpiwe_max_f = 5, + hierarchy = NULL, + inapplicable = "bgs", + hsj_alpha = 1, + ... +) +} +\arguments{ +\item{tree}{A tree of class \code{phylo} whose clades are to be evaluated. If a +\code{multiPhylo} object is provided, its strict consensus will be evaluated.} + +\item{dataset}{A phylogenetic data matrix of \pkg{phangorn} class +\code{phyDat}, whose names correspond to the labels of any accompanying tree.} + +\item{method}{Character: \code{"constraint"} for negative-constraint searches, +or \code{"pool"} for approximation via the suboptimal pool.} + +\item{maxBremer}{Numeric specifying the largest decay value to resolve when +\code{method = "pool"}. Ignored under \code{method = "constraint"}.} + +\item{optimalScore}{Optional numeric: the score of the optimal tree, +\eqn{L^*}{L*}. If \code{NULL}, it is read from \code{attr(tree, "score")}, or computed +by search.} + +\item{format}{Character specifying return format. +\code{"character"} returns a character string suitable to add to the \code{node.labels} +attribute of a tree; +"numeric" returns numeric values suitable for further analysis.} + +\item{concavity, extended_iw, xpiwe_r, xpiwe_max_f, hierarchy, hsj_alpha, inapplicable}{Tree +scoring parameters passed to \code{MaximizeParsimony()}. +These will usually match the parameters used to obtain \code{tree}.} + +\item{\dots}{Further arguments passed to \code{\link[=MaximizeParsimony]{MaximizeParsimony()}} / +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, e.g. \code{maxReplicates}, \code{maxSeconds}, \code{effort}, +\code{nThreads}, \code{verbosity}.} +} +\value{ +\code{Bremer()} returns a numeric vector (or, if \code{format = "character"}, a +\code{phylo$node.label}-shaped character vector) giving the Bremer support of each +resolved clade in \code{tree}, named by node number (the row names of +\code{\link[TreeTools:as.Splits]{TreeTools::as.Splits()}}). +Annotate a plot with \code{\link[TreeTools:LabelSplits]{TreeTools::LabelSplits()}}, or assign to +\code{tree$node.label}. +Under \code{method = "pool"} the numeric result carries a logical \code{censored} +attribute marking clades whose support exceeds \code{maxBremer}. +} +\description{ +\code{Bremer()} calculates the Bremer support (decay index) +\insertCite{Bremer1988,Bremer1994}{TreeSearch} of each clade in a reference +tree: the number of extra steps required before the clade is no longer +present in an optimal tree. Formally, for a clade \emph{C}, +\deqn{\textrm{Bremer}(C) = L(\neg C) - L^\star}{Bremer(C) = L(not C) - L\*} +where \eqn{L(\neg C)}{L(not C)} is the score of the shortest tree that does +not contain \emph{C} and \eqn{L^\star}{L\*} is the score of the most-parsimonious +tree. Larger values indicate better-supported clades. +} +\details{ +Two engines are available: + +\describe{ +\item{\code{method = "constraint"} (default, rigorous)}{For each clade, runs +a tree search under the constraint that that clade is absent, taking +the score of the shortest resulting tree.} +\item{\code{method = "pool"} (fast, loose)}{Runs a single search and collects +a pool of suboptimal trees with \code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, then for each +clade, take the best score found in a tree that lacks that clade. +Provides an upper bound on the support: better trees that lack the clade +may exist outside the sampled trees. Useful for exploration, but not +recommended for publication values.} +} + +Ensure that search options (\code{concavity}, \code{inapplicable}, ...) match those +used to find \code{tree}. +} +\examples{ +data("inapplicable.phyData", package = "TreeSearch") +dataset <- inapplicable.phyData[["Vinther2008"]] +\donttest{ +# `set.seed()` with `nThreads = 1` makes the heuristic search reproducible +set.seed(0) +trees <- MaximizeParsimony(dataset, maxReplicates = 8, nThreads = 1, + verbosity = 0) + +# Fast, approximate decay index for the strict consensus topology, +# from the suboptimal-tree pool. +decay <- Bremer(trees, dataset, method = "pool", maxBremer = 4, + maxSeconds = 20, nThreads = 1, verbosity = 1) +decay + +# Annotate the reference tree +reference <- TreeTools::Consensus(trees, p = 1) +plot(reference) +TreeTools::LabelSplits(reference, decay, adj = c(0, -0.5)) + +# `method = "constraint"` is rigorous but may be slower. +# Bounding each search with `maxSeconds` increases both speed, and the chance +# of over-estimating support through missing a better score. +slower <- Bremer(trees, dataset, maxReplicates = 8, maxSeconds = 2, + verbosity = 0) +TreeTools::LabelSplits(reference, slower, bg = "gold", adj = c(0, 1.5)) +} +} +\references{ +\insertAllCited{} +} +\seealso{ +Other clade support measures: \code{\link[=JackLabels]{JackLabels()}}, \code{\link{SiteConcordance}}; +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} collects the pool used by \code{method = "pool"}. + +Other split support functions: +\code{\link[=ConcordanceTable]{ConcordanceTable()}}, +\code{\link[=JackLabels]{JackLabels()}}, +\code{\link[=Jackknife]{Jackknife()}}, +\code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, +\code{\link[=PaintCharacters]{PaintCharacters()}}, +\code{\link[=PresCont]{PresCont()}}, +\code{\link[=Resample]{Resample()}}, +\code{\link{SiteConcordance}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} +} +\author{ +\href{https://smithlabdurham.github.io/}{Martin R. Smith} +(\href{mailto:martin.smith@durham.ac.uk}{martin.smith@durham.ac.uk}) +} +\concept{split support functions} diff --git a/man/Carter1.Rd b/man/Carter1.Rd index 1385be5ee..4ac234ff0 100644 --- a/man/Carter1.Rd +++ b/man/Carter1.Rd @@ -7,7 +7,7 @@ \alias{LogCarter1} \title{Number of trees with \emph{m} steps} \usage{ -MaddisonSlatkin(steps, states) +MaddisonSlatkin(steps, states, maxSeconds = 2) Carter1(m, a, b) @@ -23,6 +23,9 @@ entry 3 = ambiguous state \verb{\{1,2\}} (binary \code{011}), and so on. Only observed singleton states need non-zero counts; polymorphic entries are typically zero.} +\item{maxSeconds}{Numeric giving maximum computation time in seconds. +\code{NA} is returned when the budget is exceeded.} + \item{m, steps}{Number of steps.} \item{a, b}{Number of leaves labelled \code{0} and \code{1}.} @@ -47,7 +50,7 @@ implement theorem 1 of \insertCite{Carter1990;textual}{TreeTools} for \strong{binary} characters, where \emph{a} leaves bear one state and \emph{b} bear the other. -\code{MaddisonSlatkin()} generalises this result to characters with multiple +\code{MaddisonSlatkin()} generalizes this result to characters with multiple states using the recursive approach of \insertCite{Maddison1991;textual}{TreeSearch}. It returns the \strong{log-probability} (i.e. log of the fraction of unrooted diff --git a/man/CharacterHierarchy.Rd b/man/CharacterHierarchy.Rd index e24157dba..949175e7f 100644 --- a/man/CharacterHierarchy.Rd +++ b/man/CharacterHierarchy.Rd @@ -56,6 +56,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/CharacterLength.Rd b/man/CharacterLength.Rd index 2c4e07d0e..6c1753a7e 100644 --- a/man/CharacterLength.Rd +++ b/man/CharacterLength.Rd @@ -54,6 +54,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/ConcordanceTable.Rd b/man/ConcordanceTable.Rd index 61b010f79..f796fb4fc 100644 --- a/man/ConcordanceTable.Rd +++ b/man/ConcordanceTable.Rd @@ -126,12 +126,14 @@ image(t(`mode<-`(PhyDatToMatrix(dataset), "numeric")), axes = FALSE, } Other split support functions: +\code{\link[=Bremer]{Bremer()}}, \code{\link[=JackLabels]{JackLabels()}}, \code{\link[=Jackknife]{Jackknife()}}, \code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, \code{\link[=PaintCharacters]{PaintCharacters()}}, \code{\link[=PresCont]{PresCont()}}, \code{\link[=Resample]{Resample()}}, -\code{\link{SiteConcordance}} +\code{\link{SiteConcordance}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} } \concept{split support functions} diff --git a/man/ExpectedLength.Rd b/man/ExpectedLength.Rd index 02b7c5201..b7e5a2d88 100644 --- a/man/ExpectedLength.Rd +++ b/man/ExpectedLength.Rd @@ -46,6 +46,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/HierarchyFromNames.Rd b/man/HierarchyFromNames.Rd index 436e0f622..547baa762 100644 --- a/man/HierarchyFromNames.Rd +++ b/man/HierarchyFromNames.Rd @@ -39,6 +39,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/JackLabels.Rd b/man/JackLabels.Rd index dc7475f5a..d40a774ca 100644 --- a/man/JackLabels.Rd +++ b/man/JackLabels.Rd @@ -85,13 +85,15 @@ Generate trees by jackknife resampling using \code{\link[=Resample]{Resample()}} parsimony searches, or \code{\link[=Jackknife]{Jackknife()}} for custom search criteria. Other split support functions: +\code{\link[=Bremer]{Bremer()}}, \code{\link[=ConcordanceTable]{ConcordanceTable()}}, \code{\link[=Jackknife]{Jackknife()}}, \code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, \code{\link[=PaintCharacters]{PaintCharacters()}}, \code{\link[=PresCont]{PresCont()}}, \code{\link[=Resample]{Resample()}}, -\code{\link{SiteConcordance}} +\code{\link{SiteConcordance}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} } \author{ \href{https://smithlabdurham.github.io/}{Martin R. Smith} diff --git a/man/Jackknife.Rd b/man/Jackknife.Rd index c91a958ee..ca59ec122 100644 --- a/man/Jackknife.Rd +++ b/man/Jackknife.Rd @@ -140,13 +140,15 @@ engine. } Other split support functions: +\code{\link[=Bremer]{Bremer()}}, \code{\link[=ConcordanceTable]{ConcordanceTable()}}, \code{\link[=JackLabels]{JackLabels()}}, \code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, \code{\link[=PaintCharacters]{PaintCharacters()}}, \code{\link[=PresCont]{PresCont()}}, \code{\link[=Resample]{Resample()}}, -\code{\link{SiteConcordance}} +\code{\link{SiteConcordance}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} Other custom search functions: \code{\link[=BootstrapTree]{BootstrapTree()}}, diff --git a/man/LeastSquaresFit.Rd b/man/LeastSquaresFit.Rd index 876170dc4..5baec79eb 100644 --- a/man/LeastSquaresFit.Rd +++ b/man/LeastSquaresFit.Rd @@ -28,7 +28,7 @@ fitted branch lengths and an attribute \code{"RSS"} giving the residual sum of squares. } \description{ -Fits branch lengths on a fixed tree topology that minimise the (optionally +Fits branch lengths on a fixed tree topology that minimize the (optionally weighted) least-squares discrepancy between the tree's patristic distances and a target distance matrix, using the package's C++ kernel. This is the fixed-topology counterpart of \code{\link[=LeastSquaresTree]{LeastSquaresTree()}}, and the direct analogue diff --git a/man/LeastSquaresTree.Rd b/man/LeastSquaresTree.Rd index a86dcf033..8bfd5cff8 100644 --- a/man/LeastSquaresTree.Rd +++ b/man/LeastSquaresTree.Rd @@ -45,7 +45,7 @@ The best-fitting tree found, returned \strong{unrooted}, with fitted Searches tree topologies for the one whose patristic distances best fit a target distance matrix under a least-squares criterion, fitting branch lengths on each candidate and minimising the residual sum of squares. The -heuristic uses the package's optimised C++ kernel, alternating \acronym{NNI} +heuristic uses the package's optimized C++ kernel, alternating \acronym{NNI} and \acronym{SPR} rearrangements, exactly as the parsimony search does — but driven by the least-squares score rather than tree length. } diff --git a/man/LengthAdded.Rd b/man/LengthAdded.Rd index 8da377732..ab31f7fab 100644 --- a/man/LengthAdded.Rd +++ b/man/LengthAdded.Rd @@ -76,6 +76,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/MaximizeParsimony.Rd b/man/MaximizeParsimony.Rd index d8a8a8fde..3bdf3434a 100644 --- a/man/MaximizeParsimony.Rd +++ b/man/MaximizeParsimony.Rd @@ -16,6 +16,7 @@ MaximizeParsimony( inapplicable = "bgs", hsj_alpha = 1, constraint, + .negativeConstraint = NULL, effort = 0L, maxReplicates = 96L, targetHits = NULL, @@ -126,7 +127,7 @@ than the number of secondaries it brings into existence, against 1 to lose it -- so a tree's length depends on where its root sits, whereas parsimony under the other methods does not. Lengths are therefore reported at a canonical rooting, on the first taxon of \code{dataset}, which -is the rooting the returned trees carry; \code{TreeLength()} canonicalises +is the rooting the returned trees carry; \code{TreeLength()} canonicalizes identically, so it reproduces the reported score and one topology has one length. That value is an upper bound on the rooting-free minimum, exceeding it by at most the total number of secondary characters across @@ -157,6 +158,10 @@ Each constraint character is enforced as a single split, so one with more than two states is read as the split between its first and last state alone: taxa in an intermediate state are left unconstrained, with a warning.} +\item{.negativeConstraint}{Internal. A splits object (or \code{NULL}) naming +clades that returned trees must \emph{not} display; used by \code{\link[=Bremer]{Bremer()}} to run +converse-constraint searches. Not intended for direct use.} + \item{effort}{Integer: how much search effort to spend, \strong{relative to the amount chosen automatically} for this dataset. \code{0} (the default) accepts the automatic choice; \code{1} asks for one notch more, \code{-1} one notch less. @@ -374,6 +379,16 @@ A \code{multiPhylo} object containing the best tree(s) found, with attributes: \describe{ \item{\code{score}}{Best parsimony score.} +\item{\code{scoring}}{A list recording the scoring conditions the score is +optimal under (\code{concavity}, \code{inapplicable}, ...), so a saved score +remains interpretable. \code{\link[=Bremer]{Bremer()}} checks it against its own +scoring arguments and warns on a mismatch.} +\item{\code{scores}}{Present only when \code{collapse = FALSE}: a numeric vector of +the parsimony score of each returned tree, aligned with the returned +\code{multiPhylo} (the same values are attached as a \code{score} attribute on +each individual tree). With \code{poolSuboptimal > 0} this exposes the +retained suboptimal pool for landscape analysis (see +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=Suboptimality]{Suboptimality()}}).} \item{\code{replicates}}{Number of replicates completed.} \item{\code{hits_to_best}}{Number of independent discoveries of the best score.} @@ -545,6 +560,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/MinimumLength.Rd b/man/MinimumLength.Rd index ebfdce34d..fe849fd35 100644 --- a/man/MinimumLength.Rd +++ b/man/MinimumLength.Rd @@ -90,6 +90,7 @@ Other tree scoring: \code{\link[=MaximizeParsimony]{MaximizeParsimony()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/MostContradictedFreq.Rd b/man/MostContradictedFreq.Rd index 0ccbe0d70..e35ae1023 100644 --- a/man/MostContradictedFreq.Rd +++ b/man/MostContradictedFreq.Rd @@ -36,12 +36,14 @@ contradicted" score. \code{PresCont()} calculates the "groups present / contradicted" score. Other split support functions: +\code{\link[=Bremer]{Bremer()}}, \code{\link[=ConcordanceTable]{ConcordanceTable()}}, \code{\link[=JackLabels]{JackLabels()}}, \code{\link[=Jackknife]{Jackknife()}}, \code{\link[=PaintCharacters]{PaintCharacters()}}, \code{\link[=PresCont]{PresCont()}}, \code{\link[=Resample]{Resample()}}, -\code{\link{SiteConcordance}} +\code{\link{SiteConcordance}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} } \concept{split support functions} diff --git a/man/PaintCharacters.Rd b/man/PaintCharacters.Rd index 49500254f..d21856b78 100644 --- a/man/PaintCharacters.Rd +++ b/man/PaintCharacters.Rd @@ -65,12 +65,14 @@ plot(tree, edge.color = paint$edgeCol, edge.width = 2) \code{\link[TreeTools:PaintTree]{TreeTools::PaintTree()}}, \code{\link[=ConcordanceTable]{ConcordanceTable()}} Other split support functions: +\code{\link[=Bremer]{Bremer()}}, \code{\link[=ConcordanceTable]{ConcordanceTable()}}, \code{\link[=JackLabels]{JackLabels()}}, \code{\link[=Jackknife]{Jackknife()}}, \code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, \code{\link[=PresCont]{PresCont()}}, \code{\link[=Resample]{Resample()}}, -\code{\link{SiteConcordance}} +\code{\link{SiteConcordance}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} } \concept{split support functions} diff --git a/man/ParsSim.Rd b/man/ParsSim.Rd index 6c8fc8d27..82f365d2a 100644 --- a/man/ParsSim.Rd +++ b/man/ParsSim.Rd @@ -124,6 +124,7 @@ Other tree scoring: \code{\link[=MaximizeParsimony]{MaximizeParsimony()}}, \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/PrepareDataProfile.Rd b/man/PrepareDataProfile.Rd index 8fbb533bf..78cfbcaa8 100644 --- a/man/PrepareDataProfile.Rd +++ b/man/PrepareDataProfile.Rd @@ -6,23 +6,33 @@ \alias{PrepareDataIW} \title{Prepare data for Profile Parsimony} \usage{ -PrepareDataProfile(dataset, approx = "auto", n_mc = 100000L) +PrepareDataProfile( + dataset, + approx = "auto", + maxSeconds = 2, + mcSamples = 100000L +) PrepareDataIW(dataset) } \arguments{ \item{dataset}{dataset of class \code{phyDat}} -\item{approx}{Character string controlling how profile information amounts -are computed for multi-state characters with many tips. +\item{approx}{Character string controlling how profile information +amounts are computed for multi-state characters with many tips. \code{"auto"} (default) uses the exact Maddison & Slatkin calculation when feasible, falling back to a Monte Carlo approximation for large or complex characters. \code{"mc"} always uses the Monte Carlo approximation; -\code{"exact"} always uses the exact calculation (may be very slow).} +\code{"exact"} always uses the exact calculation regardless of the feasibility +gate.} -\item{n_mc}{Integer; number of Monte Carlo samples for the MC -approximation. Default 100 000.} +\item{maxSeconds}{Non-negative numeric giving the time budget, in seconds, +allowed to the exact solver (under \code{"auto"} or \code{"exact"}) before falling +back to the Monte Carlo approximation.} + +\item{mcSamples}{Integer specifying number of Monte Carlo samples for the MC +approximation.} } \value{ An object of class \code{phyDat}, with additional attributes. diff --git a/man/PresCont.Rd b/man/PresCont.Rd index d1a1be7a3..e4ae9a180 100644 --- a/man/PresCont.Rd +++ b/man/PresCont.Rd @@ -103,13 +103,15 @@ of trees that contain the split, and the frequency of the most contradicted split, respectively. Other split support functions: +\code{\link[=Bremer]{Bremer()}}, \code{\link[=ConcordanceTable]{ConcordanceTable()}}, \code{\link[=JackLabels]{JackLabels()}}, \code{\link[=Jackknife]{Jackknife()}}, \code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, \code{\link[=PaintCharacters]{PaintCharacters()}}, \code{\link[=Resample]{Resample()}}, -\code{\link{SiteConcordance}} +\code{\link{SiteConcordance}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} } \author{ \href{https://smithlabdurham.github.io/}{Martin R. Smith} diff --git a/man/Ratchet.Rd b/man/Ratchet.Rd index 0ef9fecca..c831d0a3e 100644 --- a/man/Ratchet.Rd +++ b/man/Ratchet.Rd @@ -184,7 +184,8 @@ near-optimal, within \code{suboptimal}) trees encountered, each carrying its own \code{MultiRatchet()} returns a list of optimal trees produced by \code{nSearch} \code{Ratchet()} searches, from which a consensus tree can -be generated using \code{\link[ape:consensus]{ape::consensus()}} or \code{\link[TreeTools:ConsensusWithout]{TreeTools::ConsensusWithout()}}. +be generated using \code{\link[TreeTools:Consensus]{TreeTools::Consensus()}} or +\code{\link[TreeTools:ConsensusWithout]{TreeTools::ConsensusWithout()}}. } \description{ \code{Ratchet()} uses the parsimony ratchet \insertCite{Nixon1999}{TreeSearch} diff --git a/man/RecodeHierarchy.Rd b/man/RecodeHierarchy.Rd index 91e24135a..b52f1af23 100644 --- a/man/RecodeHierarchy.Rd +++ b/man/RecodeHierarchy.Rd @@ -101,6 +101,7 @@ Other tree scoring: \code{\link[=MaximizeParsimony]{MaximizeParsimony()}}, \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/Resample.Rd b/man/Resample.Rd index 98a744665..1668dfe5e 100644 --- a/man/Resample.Rd +++ b/man/Resample.Rd @@ -141,7 +141,7 @@ than the number of secondaries it brings into existence, against 1 to lose it -- so a tree's length depends on where its root sits, whereas parsimony under the other methods does not. Lengths are therefore reported at a canonical rooting, on the first taxon of \code{dataset}, which -is the rooting the returned trees carry; \code{TreeLength()} canonicalises +is the rooting the returned trees carry; \code{TreeLength()} canonicalizes identically, so it reproduces the reported score and one topology has one length. That value is an upper bound on the rooting-free minimum, exceeding it by at most the total number of secondary characters across @@ -219,12 +219,14 @@ JackLabels(result[[1]], jackTrees) } \seealso{ Other split support functions: +\code{\link[=Bremer]{Bremer()}}, \code{\link[=ConcordanceTable]{ConcordanceTable()}}, \code{\link[=JackLabels]{JackLabels()}}, \code{\link[=Jackknife]{Jackknife()}}, \code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, \code{\link[=PaintCharacters]{PaintCharacters()}}, \code{\link[=PresCont]{PresCont()}}, -\code{\link{SiteConcordance}} +\code{\link{SiteConcordance}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} } \concept{split support functions} diff --git a/man/SPR.Rd b/man/SPR.Rd index d92395409..823b8d0f1 100644 --- a/man/SPR.Rd +++ b/man/SPR.Rd @@ -80,8 +80,7 @@ then this function will occasionally return a functionally equivalent topology. \code{RootIrrelevantSPR} will search tree space more efficiently in these cases. Branch lengths are not (yet) supported. -All nodes in a tree must be bifurcating; \link[ape:collapse.singles]{ape::collapse.singles} and -\link[ape:multi2di]{ape::multi2di} may help. +All nodes in a tree must be bifurcating; \link[TreeTools:MakeTreeBinary]{TreeTools::MakeTreeBinary} may help. } \section{Functions}{ \itemize{ diff --git a/man/SearchControl.Rd b/man/SearchControl.Rd index 9666063ba..377107166 100644 --- a/man/SearchControl.Rd +++ b/man/SearchControl.Rd @@ -223,7 +223,7 @@ sequential sector replacements before the next global cleanup, matching the tips) above which a sector is solved by tree-drifting (\code{sectorGoDrift}) or by combined analysis (\code{sectorGoComb}) instead of plain \acronym{RAS}+\acronym{TBR}, following \acronym{TNT}'s \code{sectsch} -\code{godrift}/\code{gocomb}. Small sectors are cheap to optimise exhaustively; large +\code{godrift}/\code{gocomb}. Small sectors are cheap to optimize exhaustively; large sectors have more reach but need drift to escape their own local optima. \code{sectorGoComb} solves the sector with \code{sectorCombStarts} \acronym{RAS}+drift starts and then \emph{fuses} them (recombines shared clades across the starts over @@ -346,7 +346,7 @@ falls back to random otherwise. characters are preferentially dropped. High-missingness taxa are hardest to score correctly and most likely to be trapped in suboptimal positions. -\item \code{3} = combined: weight = instability × (1 + normalised missingness). +\item \code{3} = combined: weight = instability × (1 + normalized missingness). Targets taxa that are both unstably placed and data-poor. }} diff --git a/man/SiteConcordance.Rd b/man/SiteConcordance.Rd index c29a9880c..edf80551b 100644 --- a/man/SiteConcordance.Rd +++ b/man/SiteConcordance.Rd @@ -273,13 +273,15 @@ ClusteringConcordance(TreeTools::NJTree(myMatrix), myMatrix) } Other split support functions: +\code{\link[=Bremer]{Bremer()}}, \code{\link[=ConcordanceTable]{ConcordanceTable()}}, \code{\link[=JackLabels]{JackLabels()}}, \code{\link[=Jackknife]{Jackknife()}}, \code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, \code{\link[=PaintCharacters]{PaintCharacters()}}, \code{\link[=PresCont]{PresCont()}}, -\code{\link[=Resample]{Resample()}} +\code{\link[=Resample]{Resample()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}} } \author{ \href{https://smithlabdurham.github.io/}{Martin R. Smith} diff --git a/man/StepInformation.Rd b/man/StepInformation.Rd index 14235d586..702cc4dc9 100644 --- a/man/StepInformation.Rd +++ b/man/StepInformation.Rd @@ -7,8 +7,9 @@ StepInformation( char, ambiguousTokens = c("-", "?"), - approx = "auto", - n_mc = 100000L + approx = c("auto", "mc", "exact"), + maxSeconds = 2, + mcSamples = 100000L ) } \arguments{ @@ -17,14 +18,17 @@ StepInformation( \item{ambiguousTokens}{Vector specifying which tokens, if any, correspond to the ambiguous token (\verb{?}).} -\item{approx}{Character string controlling the computation method: -\code{"auto"} (default) uses exact computation when feasible, falling back to -Monte Carlo for large or complex characters (see Details); +\item{approx}{Character string specifying the computation method: \code{"mc"} always uses the Monte Carlo approximation; -\code{"exact"} forces exact computation regardless of cost (may be very slow -for large or complex characters).} +\code{"exact"} forces exact computation regardless of the feasibility gate; +\code{"auto"} (default) uses exact computation when feasible, falling back to +Monte Carlo for large or complex characters (see Details).} + +\item{maxSeconds}{Non-negative numeric giving the time budget, in seconds, +allowed to the exact solver (under \code{"auto"} or \code{"exact"}) before falling +back to the Monte Carlo approximation.} -\item{n_mc}{Integer. Number of random trees used by the MC approximation. +\item{mcSamples}{Integer. Number of random trees used by the MC approximation. Larger values improve accuracy but increase computation time. Default: 100 000.} } @@ -55,7 +59,9 @@ back to a Monte Carlo approximation for large or complex characters. When the Maddison & Slatkin computation would be infeasible (exponential in the number of tips for a given number of tokens), behaviour depends on the \code{approx} argument. With \code{"auto"} (default), the exact solver is used -where feasible and the Monte Carlo approximation is used otherwise. +where feasible and the Monte Carlo approximation is used otherwise; the +exact solver is allowed a time budget of \code{maxSeconds}, past which the +calculation falls back to the Monte Carlo approximation. With \code{"mc"}, the Monte Carlo approximation is always used. The MC approximation computes the exact minimum-steps probability analytically, uses random trees for the @@ -65,7 +71,8 @@ The exact feasibility threshold depends on the partition shape beyond ~27 tips, 4-state beyond ~13 tips, and 5-state beyond ~9 tips trigger the approximation. With \code{"exact"}, the full Maddison & Slatkin recursion is forced regardless -of cost (may be very slow for large or complex characters). +of the feasibility gate (may be very slow for large or complex +characters), still subject to \code{maxSeconds}. } \examples{ character <- rep(c(0:3, "?", "-"), c(8, 5, 1, 1, 2, 2)) diff --git a/man/SuboptimalTrees.Rd b/man/SuboptimalTrees.Rd new file mode 100644 index 000000000..808d355d4 --- /dev/null +++ b/man/SuboptimalTrees.Rd @@ -0,0 +1,92 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/SuboptimalTrees.R +\name{SuboptimalTrees} +\alias{SuboptimalTrees} +\title{Collect suboptimal trees for landscape analysis} +\usage{ +SuboptimalTrees(dataset, tree = NULL, maxSuboptimal = 5, maxPool = 1000L, ...) +} +\arguments{ +\item{dataset}{A phylogenetic data matrix of \pkg{phangorn} class +\code{phyDat}, whose names correspond to the labels of any accompanying tree.} + +\item{tree}{Optional starting tree (of class \code{phylo}) or \code{multiPhylo}; if +\code{NULL}, the search begins from random addition sequence trees.} + +\item{maxSuboptimal}{Numeric: only trees whose score is within +\code{maxSuboptimal} of the best score found will be retained. +Corresponds to \code{poolSuboptimal} in \code{\link[=SearchControl]{SearchControl()}}.} + +\item{maxPool}{Integer: maximum number of trees to retain in the pool. +Corresponds to \code{poolMaxSize} in \code{\link[=SearchControl]{SearchControl()}}.} + +\item{\dots}{Further arguments passed to \code{\link[=MaximizeParsimony]{MaximizeParsimony()}}, including +scoring options (\code{concavity}, \code{inapplicable}, ...), search effort +(\code{maxReplicates}, \code{maxSeconds}, \code{effort}, \code{nThreads}, \code{verbosity}), +and \code{\link[=SearchControl]{SearchControl()}} fields.} +} +\value{ +\code{SuboptimalTrees()} returns a \code{multiPhylo} object listing the +retained trees, each supplied with a \code{score} attribute of each tree that +records the score it attained. The global \code{scores} attribute compiles these +scores in a numeric vector. +\code{\link[=Suboptimality]{Suboptimality()}} reports each tree's excess over the optimum. +} +\description{ +\code{SuboptimalTrees()} performs a parsimony search with \code{\link[=MaximizeParsimony]{MaximizeParsimony()}} +and returns every retained tree within a specified number of steps of the +optimum, each annotated with its parsimony score. +} +\details{ +The retained trees represent the search engine's internal tree pool, whose +size is bounded by \code{maxPool}. Once the pool is filled, trees are chosen for +eviction so as to preserve topological diversity. +} +\examples{ +data("inapplicable.phyData", package = "TreeSearch") +dataset <- inapplicable.phyData[["Vinther2008"]] +\donttest{ +set.seed(0) +trees <- SuboptimalTrees(dataset, maxSuboptimal = 3, maxReplicates = 5, + nThreads = 1, maxPool = 100L, verbosity = 0) +table(attr(trees, "scores")) # Parsimony lengths of retained trees +table(Suboptimality(trees)) # Excess over the optimum +} +} +\seealso{ +\code{\link[=MaximizeParsimony]{MaximizeParsimony()}} performs the underlying search; +\code{\link[=Bremer]{Bremer()}} uses this pool for approximate decay indices; +\code{\link[=Suboptimality]{Suboptimality()}} summarizes the excess length of each tree. + +Other split support functions: +\code{\link[=Bremer]{Bremer()}}, +\code{\link[=ConcordanceTable]{ConcordanceTable()}}, +\code{\link[=JackLabels]{JackLabels()}}, +\code{\link[=Jackknife]{Jackknife()}}, +\code{\link[=MostContradictedFreq]{MostContradictedFreq()}}, +\code{\link[=PaintCharacters]{PaintCharacters()}}, +\code{\link[=PresCont]{PresCont()}}, +\code{\link[=Resample]{Resample()}}, +\code{\link{SiteConcordance}} + +Other tree scoring: +\code{\link[=CharacterHierarchy]{CharacterHierarchy()}}, +\code{\link[=CharacterLength]{CharacterLength()}}, +\code{\link[=ExpectedLength]{ExpectedLength()}}, +\code{\link[=HierarchyFromNames]{HierarchyFromNames()}}, +\code{\link[=IWScore]{IWScore()}}, +\code{\link[=LengthAdded]{LengthAdded()}}, +\code{\link[=MaximizeParsimony]{MaximizeParsimony()}}, +\code{\link[=MinimumLength]{MinimumLength()}}, +\code{\link[=ParsSim]{ParsSim()}}, +\code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=TaxonInfluence]{TaxonInfluence()}}, +\code{\link[=TreeScore]{TreeScore()}}, +\code{\link[=WideSample]{WideSample()}} +} +\author{ +\href{https://smithlabdurham.github.io/}{Martin R. Smith} +(\href{mailto:martin.smith@durham.ac.uk}{martin.smith@durham.ac.uk}) +} +\concept{split support functions} +\concept{tree scoring} diff --git a/man/Suboptimality.Rd b/man/Suboptimality.Rd index 4cf0f778b..20fd1934f 100644 --- a/man/Suboptimality.Rd +++ b/man/Suboptimality.Rd @@ -1,22 +1,22 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/SuccessiveApproximations.R +% Please edit documentation in R/SuboptimalTrees.R \name{Suboptimality} \alias{Suboptimality} -\title{Tree suboptimality} +\title{Compare scores of suboptimal trees} \usage{ -Suboptimality(trees, proportional = FALSE) +Suboptimality(trees, normalize = FALSE) } \arguments{ -\item{trees}{list of trees, to include an optimal tree} +\item{trees}{List of trees, perhaps generated through \code{SuboptimalTrees()}, +each of which must bear a numeric attribute \code{score} giving its score.} -\item{proportional}{logical stating whether to normalise results to lowest -score} +\item{normalize}{Logical stating whether to normalize results to lowest +score.} } \value{ -\code{Suboptimality()} returns a vector listing, for each tree, how much -its score differs from the optimal (lowest) score. +\code{Suboptimality()} returns a numeric vector listing, for each tree, +its difference in score from the optimal (lowest) within \code{trees}. } \description{ -How suboptimal is a tree? +Compare scores of suboptimal trees } -\keyword{internal} diff --git a/man/TBR.Rd b/man/TBR.Rd index 2df173d4b..635b96930 100644 --- a/man/TBR.Rd +++ b/man/TBR.Rd @@ -77,8 +77,7 @@ to the rearranged \code{parent} and \code{child} parameters. \details{ Branch lengths are not (yet) supported. -All nodes in a tree must be bifurcating; \link[ape:collapse.singles]{ape::collapse.singles} and -\link[ape:multi2di]{ape::multi2di} may help. +All nodes in a tree must be bifurcating; \link[TreeTools:MakeTreeBinary]{TreeTools::MakeTreeBinary} may help. } \section{Functions}{ \itemize{ diff --git a/man/TaxonInfluence.Rd b/man/TaxonInfluence.Rd index 1cb91151f..daa68d929 100644 --- a/man/TaxonInfluence.Rd +++ b/man/TaxonInfluence.Rd @@ -81,7 +81,7 @@ but are biased towards areas of uncertainty. It is possible that a set of trees contains all possible resolutions of a particular clade, and a single other topology in which that clade does not exist – essentially two distinct solutions, one (\emph{a}) which could be -summarised with a summary tree that contains a polytomy, and another (\emph{b}) +summarized with a summary tree that contains a polytomy, and another (\emph{b}) which could be summarized by a perfectly resolved tree. Neither of these scenarios is preferable under the principles of parsimony; but summary statistics (e.g. mean, median) will be strongly influenced by the @@ -149,6 +149,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} } diff --git a/man/TreeLength.Rd b/man/TreeLength.Rd index 767c11c9c..bc6b37352 100644 --- a/man/TreeLength.Rd +++ b/man/TreeLength.Rd @@ -150,7 +150,7 @@ than the number of secondaries it brings into existence, against 1 to lose it -- so a tree's length depends on where its root sits, whereas parsimony under the other methods does not. Lengths are therefore reported at a canonical rooting, on the first taxon of \code{dataset}, which -is the rooting the returned trees carry; \code{TreeLength()} canonicalises +is the rooting the returned trees carry; \code{TreeLength()} canonicalizes identically, so it reproduces the reported score and one topology has one length. That value is an upper bound on the rooting-free minimum, exceeding it by at most the total number of secondary characters across @@ -214,6 +214,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}}, \code{\link[=WideSample]{WideSample()}} diff --git a/man/TreeScore.Rd b/man/TreeScore.Rd index 8cb083f8c..02298d1fc 100644 --- a/man/TreeScore.Rd +++ b/man/TreeScore.Rd @@ -66,6 +66,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=WideSample]{WideSample()}} } diff --git a/man/WideSample.Rd b/man/WideSample.Rd index 4d7cd3114..3fdb86d54 100644 --- a/man/WideSample.Rd +++ b/man/WideSample.Rd @@ -168,6 +168,7 @@ Other tree scoring: \code{\link[=MinimumLength]{MinimumLength()}}, \code{\link[=ParsSim]{ParsSim()}}, \code{\link[=RecodeHierarchy]{RecodeHierarchy()}}, +\code{\link[=SuboptimalTrees]{SuboptimalTrees()}}, \code{\link[=TaxonInfluence]{TaxonInfluence()}}, \code{\link[=TreeScore]{TreeScore()}} } diff --git a/man/mc_fitch_scores.Rd b/man/mc_fitch_scores.Rd index 8d2db7971..9624a5402 100644 --- a/man/mc_fitch_scores.Rd +++ b/man/mc_fitch_scores.Rd @@ -4,7 +4,7 @@ \alias{mc_fitch_scores} \title{Monte Carlo Fitch scores for a single character} \usage{ -mc_fitch_scores(state_counts, n_mc) +mc_fitch_scores(state_counts, mcSamples) } \arguments{ \item{state_counts}{Integer vector giving the number of tips in each @@ -12,14 +12,14 @@ state. Length determines the number of states (k); sum determines the number of tips (n). For example, \code{c(13, 13, 12)} defines a 3-state character with 38 tips.} -\item{n_mc}{Number of random trees to generate and score.} +\item{mcSamples}{Number of random trees to generate and score.} } \value{ -Integer vector of length \code{n_mc} containing the Fitch parsimony +Integer vector of length \code{mcSamples} containing the Fitch parsimony score (number of state changes) for each random tree. } \description{ -Generates \code{n_mc} random trees and scores each with a Fitch parsimony +Generates \code{mcSamples} random trees and scores each with a Fitch parsimony downpass for a single character defined by \code{state_counts}. Tree generation and scoring are done entirely in C with no R object allocation per tree, making this very fast (~0.01 ms per tree). diff --git a/src/MaddisonSlatkin.cpp b/src/MaddisonSlatkin.cpp index 753448e89..22318c3db 100644 --- a/src/MaddisonSlatkin.cpp +++ b/src/MaddisonSlatkin.cpp @@ -917,48 +917,8 @@ class SolverT { // Global s_max for this solver instance (set at first run() call) int s_max_global = 0; - // Time budget: abort if computation exceeds this many seconds. - // - // This is a latency promise, and it is deliberately the binding one. It - // caps what a caller waits per character; `.MS_SC_THRESHOLD` only skips work - // that is hopeless enough to be worth not starting. The two could be - // arranged the other way round -- gate on the character's shape, which is a - // machine-independent quantity, and let the clock recede to a backstop -- - // and that would buy exactness that reproduces across machines. It is not - // worth its price. A dataset is hundreds of characters; a budget generous - // enough for the slowest one the gate admits (k=3 (9,9,9), sc=75, measured - // at 12.7 s on a 2021 desktop) is an hour of unresponsiveness in the bad - // case, for a caller who mostly wants a number back. Exactness here is a - // refinement over an already-documented approximation, so it yields. - // - // The consequence, accepted knowingly: whether a given character is scored - // exactly or by Monte Carlo depends on how fast the machine is. The - // fallback is a sampling estimate in any case, so its value was never - // machine-invariant either. `approx = "mc"` is the escape hatch that is - // stable by construction; note that `approx = "exact"` is not one, since it - // waives the gate but is still stopped by this budget. - // - // An instrumented build runs one to two orders of magnitude slower, so the - // budget would fire there on anything at all -- leaving the sanitizer - // checking the bailout path instead of the algorithm it was pointed at. - // Scale by that slowdown rather than disabling, so a genuine blowup is still - // bounded; there is no responsiveness to protect in a sanitizer run, which - // is a nightly check and not a user sitting at a prompt. - // GCC announces ASan through __SANITIZE_ADDRESS__ and clang through - // __has_feature; TS_SANITIZER_BUILD is the manual escape hatch for the - // instrumented builds that announce themselves through neither (valgrind). -#if defined(__SANITIZE_ADDRESS__) || defined(TS_SANITIZER_BUILD) -# define TS_MS_SLOW_BUILD 1 -#elif defined(__has_feature) -# if __has_feature(address_sanitizer) || __has_feature(memory_sanitizer) -# define TS_MS_SLOW_BUILD 1 -# endif -#endif -#ifdef TS_MS_SLOW_BUILD - static constexpr double TIME_BUDGET_S = 200.0; -#else - static constexpr double TIME_BUDGET_S = 2.0; -#endif + // Per-call time budget, supplied by StepInformation(). + double maxSeconds; std::chrono::steady_clock::time_point start_time; bool budget_exceeded = false; // Set when we bail because a memo table reached its reserved capacity (as @@ -1075,7 +1035,7 @@ class SolverT { if (budget_exceeded) return NEG_INF; { auto now = std::chrono::steady_clock::now(); - if (std::chrono::duration(now - start_time).count() > TIME_BUDGET_S) { + if (std::chrono::duration(now - start_time).count() > maxSeconds) { budget_exceeded = true; return NEG_INF; } @@ -1185,7 +1145,7 @@ class SolverT { if (budget_exceeded) return bailout_vec; { auto now = std::chrono::steady_clock::now(); - if (std::chrono::duration(now - start_time).count() > TIME_BUDGET_S) { + if (std::chrono::duration(now - start_time).count() > maxSeconds) { budget_exceeded = true; return bailout_vec; } @@ -1338,7 +1298,7 @@ class SolverT { } } - // Subtract denom (LogB normaliser) + // Subtract denom (LogB normalizer) for (int s = 0; s < c_size; ++s) { if ((outerVec[s] > NEG_INF)) outerVec[s] -= denom; } @@ -1351,8 +1311,9 @@ class SolverT { public: SolverT(const Downpass& D_, const TokenPairs& p, int presentBits_, - LnRootedCache& lnr) - : D(D_), pairs(p), presentBits(presentBits_), lnRooted(lnr) { + LnRootedCache& lnr, double maxSeconds_) + : D(D_), pairs(p), presentBits(presentBits_), maxSeconds(maxSeconds_), + lnRooted(lnr) { // Sized against the worst character `.MS_SC_THRESHOLD` admits to this // solver, measured at the entry high-water mark rather than guessed: @@ -1400,9 +1361,9 @@ class SolverT { "capacity; results will be NA (the caller falls back to " "the Monte Carlo approximation)."); } else { - Rcpp::warning("MaddisonSlatkin: computation exceeded %.0f s time budget; " + Rcpp::warning("MaddisonSlatkin: computation exceeded %.3g s time budget; " "results will be NA. Consider reducing to binary.", - TIME_BUDGET_S); + maxSeconds); } for (int i = 0; i < (int)steps_vec.size(); ++i) out[i] = NA_REAL; return; @@ -1427,9 +1388,9 @@ class SolverT { "capacity; results will be NA (the caller falls back to " "the Monte Carlo approximation)."); } else { - Rcpp::warning("MaddisonSlatkin: computation exceeded %.0f s time budget; " + Rcpp::warning("MaddisonSlatkin: computation exceeded %.3g s time budget; " "results will be NA. Consider reducing to binary.", - TIME_BUDGET_S); + maxSeconds); } for (int i = 0; i < (int)steps_vec.size(); ++i) out[i] = NA_REAL; return; @@ -1694,7 +1655,11 @@ static double logCarter1_cpp(int m, int a, int b) { //' //' @export // [[Rcpp::export]] -NumericVector MaddisonSlatkin(IntegerVector steps, IntegerVector states) { +NumericVector MaddisonSlatkin(IntegerVector steps, IntegerVector states, + double maxSeconds = 2.0) { + if (!std::isfinite(maxSeconds) || maxSeconds < 0.0) { + stop("`maxSeconds` must be a non-negative finite number."); + } int len = states.size(); if (len <= 0) stop("`states` must have positive length."); @@ -1800,19 +1765,19 @@ NumericVector MaddisonSlatkin(IntegerVector steps, IntegerVector states) { // DISPATCH based on nTokens — call runAll() once for all step counts if (nTokens == 2) { StateKeyT<2> rootKey(leavesVec); - SolverT<2> solver(D, pairs, presentBits, lnRooted); + SolverT<2> solver(D, pairs, presentBits, lnRooted, maxSeconds); solver.runAll(valid_steps, rootKey, results.data()); } else if (nTokens == 3) { StateKeyT<3> rootKey(leavesVec); - SolverT<3> solver(D, pairs, presentBits, lnRooted); + SolverT<3> solver(D, pairs, presentBits, lnRooted, maxSeconds); solver.runAll(valid_steps, rootKey, results.data()); } else if (nTokens == 4) { StateKeyT<4> rootKey(leavesVec); - SolverT<4> solver(D, pairs, presentBits, lnRooted); + SolverT<4> solver(D, pairs, presentBits, lnRooted, maxSeconds); solver.runAll(valid_steps, rootKey, results.data()); } else { // nTokens == 5 StateKeyT<5> rootKey(leavesVec); - SolverT<5> solver(D, pairs, presentBits, lnRooted); + SolverT<5> solver(D, pairs, presentBits, lnRooted, maxSeconds); solver.runAll(valid_steps, rootKey, results.data()); } diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index 3d1f0ca32..c9d63a08e 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -11,14 +11,15 @@ Rcpp::Rostream& Rcpp::Rcerr = Rcpp::Rcpp_cerr_get(); #endif // MaddisonSlatkin -NumericVector MaddisonSlatkin(IntegerVector steps, IntegerVector states); -RcppExport SEXP _TreeSearch_MaddisonSlatkin(SEXP stepsSEXP, SEXP statesSEXP) { +NumericVector MaddisonSlatkin(IntegerVector steps, IntegerVector states, double maxSeconds); +RcppExport SEXP _TreeSearch_MaddisonSlatkin(SEXP stepsSEXP, SEXP statesSEXP, SEXP maxSecondsSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; Rcpp::traits::input_parameter< IntegerVector >::type steps(stepsSEXP); Rcpp::traits::input_parameter< IntegerVector >::type states(statesSEXP); - rcpp_result_gen = Rcpp::wrap(MaddisonSlatkin(steps, states)); + Rcpp::traits::input_parameter< double >::type maxSeconds(maxSecondsSEXP); + rcpp_result_gen = Rcpp::wrap(MaddisonSlatkin(steps, states, maxSeconds)); return rcpp_result_gen; END_RCPP } @@ -164,14 +165,14 @@ BEGIN_RCPP END_RCPP } // mc_fitch_scores -IntegerVector mc_fitch_scores(IntegerVector state_counts, int n_mc); -RcppExport SEXP _TreeSearch_mc_fitch_scores(SEXP state_countsSEXP, SEXP n_mcSEXP) { +IntegerVector mc_fitch_scores(IntegerVector state_counts, int mcSamples); +RcppExport SEXP _TreeSearch_mc_fitch_scores(SEXP state_countsSEXP, SEXP mcSamplesSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; Rcpp::traits::input_parameter< IntegerVector >::type state_counts(state_countsSEXP); - Rcpp::traits::input_parameter< int >::type n_mc(n_mcSEXP); - rcpp_result_gen = Rcpp::wrap(mc_fitch_scores(state_counts, n_mc)); + Rcpp::traits::input_parameter< int >::type mcSamples(mcSamplesSEXP); + rcpp_result_gen = Rcpp::wrap(mc_fitch_scores(state_counts, mcSamples)); return rcpp_result_gen; END_RCPP } diff --git a/src/TreeSearch-init.c b/src/TreeSearch-init.c index c7ecc6e12..fd5734d23 100644 --- a/src/TreeSearch-init.c +++ b/src/TreeSearch-init.c @@ -49,7 +49,7 @@ extern SEXP _TreeSearch_ts_parallel_resample(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, extern SEXP _TreeSearch_ts_successive_approx(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP); extern SEXP _TreeSearch_ts_bench_tbr_phases(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP); extern SEXP _TreeSearch_ts_simplify_diag(SEXP, SEXP, SEXP, SEXP); -extern SEXP _TreeSearch_MaddisonSlatkin(SEXP, SEXP); +extern SEXP _TreeSearch_MaddisonSlatkin(SEXP, SEXP, SEXP); extern SEXP _TreeSearch_MaddisonSlatkin_clear_cache(); extern SEXP _TreeSearch_ts_hsj_score(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP); extern SEXP _TreeSearch_ts_sankoff_test(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP, SEXP); @@ -106,7 +106,7 @@ static const R_CallMethodDef callMethods[] = { {"_TreeSearch_ts_successive_approx", (DL_FUNC) &_TreeSearch_ts_successive_approx, 25}, {"_TreeSearch_ts_bench_tbr_phases", (DL_FUNC) &_TreeSearch_ts_bench_tbr_phases, 7}, {"_TreeSearch_ts_simplify_diag", (DL_FUNC) &_TreeSearch_ts_simplify_diag, 4}, - {"_TreeSearch_MaddisonSlatkin", (DL_FUNC) &_TreeSearch_MaddisonSlatkin, 2}, + {"_TreeSearch_MaddisonSlatkin", (DL_FUNC) &_TreeSearch_MaddisonSlatkin, 3}, {"_TreeSearch_MaddisonSlatkin_clear_cache", (DL_FUNC) &_TreeSearch_MaddisonSlatkin_clear_cache, 0}, {"_TreeSearch_ts_hsj_score", (DL_FUNC) &_TreeSearch_ts_hsj_score, 9}, {"_TreeSearch_ts_sankoff_test", (DL_FUNC) &_TreeSearch_ts_sankoff_test, 7}, diff --git a/src/ts_constraint.cpp b/src/ts_constraint.cpp index 881e7e492..85f6be76e 100644 --- a/src/ts_constraint.cpp +++ b/src/ts_constraint.cpp @@ -169,6 +169,91 @@ std::vector compute_node_tips(const TreeState& tree, int n_words) return node_tips; } +// ========================================================================= +// Negative (converse) constraints +// ========================================================================= + +void add_negative_constraint( + ConstraintData& cd, const int* neg_matrix, int n_neg, int n_tips) +{ + if (n_neg <= 0) return; + + int n_words = (n_tips + 63) / 64; + if (!cd.active) { + // Initialize an otherwise-empty ConstraintData so the search's + // `constrained` path runs (it will be a no-op over zero positive splits). + cd.active = true; + cd.n_words = n_words; + int n_node = 2 * n_tips - 1; + cd.dfs_entry.assign(n_node, 0); + cd.dfs_exit.assign(n_node, 0); + cd.clip_tip_mask.resize(n_words, 0ULL); + } + + cd.neg_active = true; + cd.n_neg_splits = n_neg; + cd.neg_split_tips.assign( + static_cast(n_neg) * cd.n_words, 0ULL); + + // Pack rows into bitmasks (column-major from R: [s, t] at s + n_neg * t), + // canonicalizing so tip 0 is always outside (bit 0 = 0), exactly as + // build_constraint() does for positive splits. + for (int s = 0; s < n_neg; ++s) { + uint64_t* mask = &cd.neg_split_tips[static_cast(s) * cd.n_words]; + for (int t = 0; t < n_tips; ++t) { + if (neg_matrix[s + n_neg * t]) { + mask[t / 64] |= (1ULL << (t % 64)); + } + } + if (mask[0] & 1ULL) { + for (int w = 0; w < cd.n_words; ++w) { + mask[w] = ~mask[w]; + } + int remainder = n_tips % 64; + if (remainder > 0) { + mask[cd.n_words - 1] &= (1ULL << remainder) - 1; + } + } + } +} + +bool displays_forbidden_clade(const TreeState& tree, const ConstraintData& cd) +{ + if (!cd.neg_active || cd.n_neg_splits == 0) return false; + + auto node_tips = compute_node_tips(tree, cd.n_words); + + // The internal tree is not guaranteed to be rooted on tip 0, so a bipartition + // can be displayed as EITHER side. Canonicalize each node's subtree mask to + // the tip-0-outside form (flipping if tip 0 is inside) before comparing to the + // canonical forbidden split; this matches a clade regardless of which side is + // the rooted subtree. + int n_tips = tree.n_tip; + int rem = n_tips % 64; + uint64_t last_mask = rem ? ((1ULL << rem) - 1) : ~0ULL; + + std::vector canon(cd.n_words); + for (int node : tree.postorder) { + const uint64_t* nd = &node_tips[static_cast(node) * cd.n_words]; + bool flip = (nd[0] & 1ULL) != 0; + for (int w = 0; w < cd.n_words; ++w) { + canon[w] = flip ? ~nd[w] : nd[w]; + } + if (flip) canon[cd.n_words - 1] &= last_mask; + + for (int s = 0; s < cd.n_neg_splits; ++s) { + const uint64_t* split = + &cd.neg_split_tips[static_cast(s) * cd.n_words]; + bool match = true; + for (int w = 0; w < cd.n_words; ++w) { + if (canon[w] != split[w]) { match = false; break; } + } + if (match) return true; + } + } + return false; +} + // ========================================================================= // Map constraint nodes: find which internal node holds each split // ========================================================================= @@ -252,7 +337,7 @@ void map_constraint_nodes(const TreeState& tree, ConstraintData& cd) // a rooted subtree, so the split is displayed whenever EITHER side is a // clade. Exactly one of the two is, except when the split is the root's own // bipartition (then both are): for an edge (parent(v), v) with v != root the - // two sides are desc(v) and its complement. build_constraint() canonicalises + // two sides are desc(v) and its complement. build_constraint() canonicalizes // A so that tip 0 is outside it, which makes A the clade side only when tip 0 // sits on the root's own edge -- true of a tip-0-rooted tree and of nothing // else. Testing the complement as well is what makes this mapping diff --git a/src/ts_constraint.h b/src/ts_constraint.h index 6ffd68e7b..7a6c80910 100644 --- a/src/ts_constraint.h +++ b/src/ts_constraint.h @@ -98,6 +98,16 @@ struct ConstraintData { int expected_score = 0; bool has_posthoc = false; + // --- Negative (converse) constraints: forbidden clades --- + // A tree is REJECTED if it displays ANY of these bipartitions. Used for + // Bremer support (the shortest tree that does NOT contain a given clade). + // Stored as canonicalized tip bitmasks (tip 0 outside), exactly like + // split_tips. Independent of the positive splits above: a ConstraintData + // may carry positive splits, negative splits, or (for Bremer) only negative. + bool neg_active = false; + int n_neg_splits = 0; + std::vector neg_split_tips; // n_neg_splits * n_words + // Per-clip workspace (reused across clips, sized at init) std::vector clip_zones; // [n_splits] std::vector clip_tip_mask; // [n_words] @@ -211,6 +221,27 @@ bool violates_constraint_posthoc(const TreeState& tree, // For tips: bit[t] = 1. For internal nodes: OR of children. std::vector compute_node_tips(const TreeState& tree, int n_words); +// --- Negative (converse) constraints --- + +// Add negative (converse) constraints from an R-side split matrix +// (n_neg rows x n_tips cols, column-major, 0/1 membership). A tree that +// displays any of these bipartitions is rejected. Initializes `cd` (active, +// n_words, DFS arrays, clip workspace) if it is not already built, so a +// negative-only ConstraintData is safe to pass through the positive-constraint +// machinery (which then runs as a no-op over zero positive splits). +void add_negative_constraint( + ConstraintData& cd, const int* neg_matrix, int n_neg, int n_tips); + +// Returns true if the tree currently displays any forbidden clade. A +// bipartition is displayed iff some internal node's subtree tip mask equals it +// on EITHER side, so each node mask is canonicalized to tip-0-outside form +// (flipped when tip 0 is inside) before comparison. This bilateral +// canonicalization is REQUIRED because the internal search tree is not +// guaranteed to be rooted on tip 0 -- unlike map_constraint_nodes(), which does +// a one-sided (already-tip-0-outside) compare for a positive split and so must +// not be assumed bilateral-safe. +bool displays_forbidden_clade(const TreeState& tree, const ConstraintData& cd); + // Repair constraint violations by minimal SPR moves. // // HEURISTIC, and it can fail: a pass bails out when the repair needs more than diff --git a/src/ts_data.cpp b/src/ts_data.cpp index 50bf960c4..c351b1fc0 100644 --- a/src/ts_data.cpp +++ b/src/ts_data.cpp @@ -420,8 +420,8 @@ DataSet build_dataset( // bit AND the applicable-state bit(s). Under BGS the applicability // character (binary Fitch, 0=inapplicable/1=applicable) is // reconstructed parsimoniously — resolving each {-,X} tip in tree - // context, applicable-preferred on ties (De Laet: maximise homology, - // not minimise homoplasy) — and the three passes then count state + // context, applicable-preferred on ties (De Laet: maximize homology, + // not minimize homoplasy) — and the three passes then count state // steps. // pure inapplicable -> {NA}; {-,X} -> {NA, X}; // missing {-,all} -> {NA, all applicable} (full wildcard); diff --git a/src/ts_driven.cpp b/src/ts_driven.cpp index 3a9708eb1..37d6e8291 100644 --- a/src/ts_driven.cpp +++ b/src/ts_driven.cpp @@ -204,6 +204,19 @@ ReplicateResult run_single_replicate( } } + // Converse-constraint (Bremer) search: guarantee a start tree that does NOT + // display any forbidden clade. A Wagner tree often builds the very clade we + // must avoid, and the guarded TBR below cannot climb out of it downhill, so + // the pool would stay empty. Random topologies almost never display an + // arbitrary clade, so fall back to them until one is clean. + if (cd && cd->neg_active && displays_forbidden_clade(result.tree, *cd)) { + int tries = 0; + do { + random_topology_tree(result.tree, ds); + } while (displays_forbidden_clade(result.tree, *cd) && ++tries < 100); + best_wag = score_tree(result.tree, ds); + } + // A start that breaks the constraint has to be repaired here, before anything // takes its score as a baseline. Constrained rearrangement cannot undo it: // regraft_violates_constraint() reads an unmapped split as "already @@ -797,6 +810,14 @@ DrivenResult driven_search(TreePool& pool, DataSet& ds, // Reset the candidate-evaluation counter for this search (serial path). ds.n_candidates_evaluated = 0; + // Converse-constraint (Bremer) search: make the pool reject any tree that + // displays a forbidden clade, so the returned best tree/score can never + // contain it regardless of which phase produced the candidate. The TBR + // hill-climber (above) additionally steers the search into that space. + if (cd && cd->neg_active) { + pool.set_forbidden(cd); + } + bool use_timeout = params.max_seconds > 0.0; auto start_time = std::chrono::steady_clock::now(); @@ -1046,7 +1067,7 @@ DrivenResult driven_search(TreePool& pool, DataSet& ds, // supplies the cross-replicate memory the per-replicate ratchet lacks, // letting the search discover the right perturbation strength for THIS // dataset at runtime instead of baking in a constant (which the - // benchmarks show does not generalise across datasets). Reset is implicit: + // benchmarks show does not generalize across datasets). Reset is implicit: // an improvement zeroes unsuccessful_reps below, so the next iteration // takes the else branch and restores base. stall_escalate_factor == 1.0 // (default) makes the whole rule inert — a true no-op. Authoritative over @@ -1319,13 +1340,19 @@ DrivenResult driven_search(TreePool& pool, DataSet& ds, } } } + bool fused_added = false; if (fused_ok) { std::vector fused_collapsed; compute_collapsed_flags(fused, ds, fused_collapsed); - pool.add_collapsed(fused, fused_score, fused_collapsed); + // The pool can reject the fused tree even when it scores better -- e.g. + // a converse-constraint (Bremer) search's backstop rejects any tree + // that displays the forbidden clade. Only treat it as an improvement + // (resetting the perturb-stop / target-hits counters) if it actually + // entered the pool; otherwise the stopping rules act on a phantom best. + fused_added = pool.add_collapsed(fused, fused_score, fused_collapsed); } - if (fused_ok && fused_score < best_before) { + if (fused_added && fused_score < best_before) { pool.set_hits_to_best(0); result.last_improved_rep = rep1; unsuccessful_reps = 0; // fuse found a better score; reset perturb-stop counter diff --git a/src/ts_fuse.h b/src/ts_fuse.h index 515ecf715..40d3c9ec4 100644 --- a/src/ts_fuse.h +++ b/src/ts_fuse.h @@ -41,7 +41,7 @@ FuseResult tree_fuse(TreeState& recipient, const DataSet& ds, // rooting-invariant, so this is safe to apply to a scored tree. // // Fuse needs it to compare differently-rooted trees; the constrained Wagner -// build needs it because build_constraint() canonicalises split masks with tip 0 +// build needs it because build_constraint() canonicalizes split masks with tip 0 // outside, so this rooting is the one in which map_constraint_nodes() can find // every split the tree displays. void reroot_at_tip0(TreeState& tree); diff --git a/src/ts_heartbeat.h b/src/ts_heartbeat.h index 3539c36aa..28bd62a80 100644 --- a/src/ts_heartbeat.h +++ b/src/ts_heartbeat.h @@ -80,7 +80,7 @@ struct HeartbeatScope { }; // Call from inside a long-running phase. `label` names the phase ("TBR", -// "Ratchet"); `score` is the phase's current best. `stride` amortises the clock +// "Ratchet"); `score` is the phase's current best. `stride` amortizes the clock // read: only every `stride`-th call consults the clock, so pass a large stride // from tight loops (per-clip) and 1 from coarse ones (per-ratchet-cycle). inline void heartbeat(const char* label, double score, int stride = 64) { diff --git a/src/ts_hsj.cpp b/src/ts_hsj.cpp index f26b652a6..cd04d04bf 100644 --- a/src/ts_hsj.cpp +++ b/src/ts_hsj.cpp @@ -33,7 +33,7 @@ std::vector partition_weights( // of SYMMETRIC dissimilarities over the branches of an UNROOTED tree (Hopkins // & St John 2021, p.3, p.6), so it may not depend on where the tree happens to // be rooted. The a(n)/p(n) DP below satisfies that already -- its branch -// costs are symmetric and it minimises over the root's own state. The +// costs are symmetric and it minimizes over the root's own state. The // secondary labelling did not: fitch_label_char() resolves ambiguous nodes // with a DELTRAN-style uppass whose direction, and with subtree support counts // whose subtrees, are both properties of the INPUT rooting. Two rootings of @@ -43,7 +43,7 @@ std::vector partition_weights( // topology and the data alone. Tip indices come from the dataset, not from // the rooting, so this is canonical. Note this is NOT the reporting-boundary // canonicalisation PR #278 applied to XFORM: this sits inside the kernel, so -// the objective the SEARCH optimises is itself rooting-invariant, and +// the objective the SEARCH optimizes is itself rooting-invariant, and // MaximizeParsimony()'s reported score agrees with TreeLength() of the trees // it returns by construction rather than by re-scoring at the boundary. // @@ -180,7 +180,7 @@ static int fitch_label_char( } // --- Downpass, in the canonical (tip-0-rooted) postorder --- - // Generalised over arity: the kernel's own root becomes an ordinary degree-2 + // Generalized over arity: the kernel's own root becomes an ordinary degree-2 // node here, which Fitch passes through unchanged, and the canonical root is // tip 0, which has one child AND an observation of its own to honour. int steps = 0; diff --git a/src/ts_ls.h b/src/ts_ls.h index 35c21fa04..7ead64515 100644 --- a/src/ts_ls.h +++ b/src/ts_ls.h @@ -4,7 +4,7 @@ // Least-squares (LS) branch-length fitting and topology search. // // Given a target symmetric dissimilarity matrix D over the tips and a fixed -// topology, fit branch lengths v that minimise the (optionally weighted) +// topology, fit branch lengths v that minimize the (optionally weighted) // residual sum of squares // // RSS = sum_{i stack; stack.reserve(n_internal); - IntegerVector scores(n_mc); - for (int rep = 0; rep < n_mc; rep++) { + IntegerVector scores(mcSamples); + for (int rep = 0; rep < mcSamples; rep++) { random_tree(parent.data(), left.data(), right.data(), &n); // Build postorder: collect preorder via DFS, then process in reverse. diff --git a/src/ts_nni_perturb.cpp b/src/ts_nni_perturb.cpp index 5c5f94a06..af7b80e66 100644 --- a/src/ts_nni_perturb.cpp +++ b/src/ts_nni_perturb.cpp @@ -124,6 +124,15 @@ NNIPerturbResult nni_perturb_search( if (cd->constraint_node[_s] < 0) { accept = false; break; } } } + // Negative (converse/Bremer) constraint: the blind NNI perturbation can + // rebuild a forbidden clade that the neg-guarded TBR could not climb back + // out of. Reject such a tree even if it scores better, reverting to the + // (clade-free) best_tree -- otherwise the replicate can strand on the clade, + // needlessly right-censoring its Bremer value. Soundness is guaranteed by + // the pool backstop regardless; this preserves reach. + if (accept && cd && cd->neg_active && displays_forbidden_clade(tree, *cd)) { + accept = false; + } if (accept) { best_score = tbr_result.best_score; diff --git a/src/ts_pool.cpp b/src/ts_pool.cpp index 397c0f6b6..0419618f6 100644 --- a/src/ts_pool.cpp +++ b/src/ts_pool.cpp @@ -1,4 +1,5 @@ #include "ts_pool.h" +#include "ts_constraint.h" // displays_forbidden_clade (negative-constraint filter) #include #include #include @@ -22,6 +23,11 @@ bool TreePool::add(const TreeState& tree, double score) { return false; } + // Reject any tree displaying a forbidden clade (converse-constraint search). + if (forbidden_ && displays_forbidden_clade(tree, *forbidden_)) { + return false; + } + SplitSet ss = compute_splits(tree); uint64_t hash = hash_splits(ss); @@ -105,6 +111,11 @@ bool TreePool::add_collapsed(const TreeState& tree, double score, return false; } + // Reject any tree displaying a forbidden clade (converse-constraint search). + if (forbidden_ && displays_forbidden_clade(tree, *forbidden_)) { + return false; + } + // Compute collapsed splits (skip zero-length edges) for dedup SplitSet css = compute_collapsed_splits(tree, collapsed); uint64_t chash = hash_splits(css); diff --git a/src/ts_pool.h b/src/ts_pool.h index 8a12e7c70..6216a374e 100644 --- a/src/ts_pool.h +++ b/src/ts_pool.h @@ -18,6 +18,8 @@ namespace ts { +struct ConstraintData; // fwd decl (ts_constraint.h); used only by pointer + // Per-split frequency table for conflict-guided sector selection. // Maps per-split hash → count across best-score pool trees. struct SplitFrequencyTable { @@ -46,6 +48,13 @@ class TreePool { best_score_(1e18), hits_to_best_(0), consensus_hash_(0), consensus_unchanged_(0) {} + // Register negative (converse) constraints: any tree displaying a forbidden + // clade is rejected by add()/add_collapsed(). Guarantees the pool -- and + // hence the returned best tree/score -- never contains a forbidden clade, + // regardless of which search phase produced the candidate. Pass nullptr + // (default) to disable. Used for Bremer converse-constraint searches. + void set_forbidden(const ConstraintData* f) { forbidden_ = f; } + // Add a tree if it's not a duplicate and meets score threshold. // Returns true if the tree was actually added. bool add(const TreeState& tree, double score); @@ -133,6 +142,9 @@ class TreePool { double best_score_; int hits_to_best_; + // Negative-constraint filter (not owned); nullptr disables. + const ConstraintData* forbidden_ = nullptr; + // Consensus stability tracking uint64_t consensus_hash_; int consensus_unchanged_; diff --git a/src/ts_prune_reinsert.cpp b/src/ts_prune_reinsert.cpp index 00b82ef91..fe9ba315c 100644 --- a/src/ts_prune_reinsert.cpp +++ b/src/ts_prune_reinsert.cpp @@ -438,7 +438,7 @@ void expand_and_reinsert( std::vector sa_pre; // Cumulative across all expand_and_reinsert() calls this session. thread_local: // expand_and_reinsert runs concurrently on parallel-search workers, so plain - // `static` would be an unsynchronised data race on these counters. thread_local + // `static` would be an unsynchronized data race on these counters. thread_local // gives each worker its own tally (per-thread partials on multi-thread runs); the // probe is a diagnostic normally run single-threaded, where this is exact. thread_local static long long sa_placements = 0, sa_delta_pos = 0, sa_delta_sum = 0, @@ -694,13 +694,21 @@ PruneReinsertResult prune_reinsert_search( if (cd->constraint_node[s] < 0) { accept = false; break; } } } + // Negative (converse/Bremer) constraint: the reduced-backbone TBR (step 4) + // runs cd-blind, so expand_and_reinsert can rebuild a forbidden clade. + // Reject such a tree even if it scores better, reverting to the (clade-free) + // pre-prune backup -- otherwise the replicate can strand on the clade. + // Soundness is already guaranteed by the pool backstop; this preserves reach. + if (accept && cd && cd->neg_active && displays_forbidden_clade(tree, *cd)) { + accept = false; + } if (accept) { current_score = new_score; result.best_score = new_score; ++result.n_improvements; } else { - tree = backup; // revert + tree = backup; // revert (worse, or would display the forbidden clade) // Re-sync constraint metadata after topology revert. // Same bug class as F-015 (ratchet), F-016 (NNI-perturb). if (cd) update_constraint(tree, *cd); diff --git a/src/ts_prune_reinsert.h b/src/ts_prune_reinsert.h index d411a3363..5bee2bc24 100644 --- a/src/ts_prune_reinsert.h +++ b/src/ts_prune_reinsert.h @@ -33,7 +33,7 @@ struct SplitFrequencyTable; // where the tip is fully ambiguous or inapplicable. // High-missingness taxa are hardest to score and most likely // to be trapped in suboptimal positions. -// COMBINED — product of INSTABILITY and MISSING scores (normalised): +// COMBINED — product of INSTABILITY and MISSING scores (normalized): // w(t) = instability(t) * (1 + miss_fraction(t)). // Targets taxa that are both unstably placed and data-poor. // Falls back to INSTABILITY when pool has < 2 trees, and to diff --git a/src/ts_ratchet.cpp b/src/ts_ratchet.cpp index 1e1fe33e8..ee83f8ee1 100644 --- a/src/ts_ratchet.cpp +++ b/src/ts_ratchet.cpp @@ -233,7 +233,17 @@ RatchetResult ratchet_search(TreeState& tree, DataSet& ds, nullptr, nullptr, check_timeout); total_moves += search_result.n_accepted; - if (search_result.best_score < best_score) { + // Negative (converse/Bremer) constraint defence-in-depth: ratchet perturbs + // only character WEIGHTS, so the topology moves solely through the two + // neg-guarded TBR searches above and the tree cannot acquire the forbidden + // clade. Check explicitly anyway, so the clade-free invariant is enforced + // here as well -- matching the nni_perturb / prune_reinsert accept paths and + // staying robust to any future topology-touching perturbation mode. + bool accept = search_result.best_score < best_score; + if (accept && cd && cd->neg_active && displays_forbidden_clade(tree, *cd)) { + accept = false; + } + if (accept) { best_score = search_result.best_score; best_tree = tree; ++n_escapes; diff --git a/src/ts_rcpp.cpp b/src/ts_rcpp.cpp index 9fcc11bf5..8f15ac130 100644 --- a/src/ts_rcpp.cpp +++ b/src/ts_rcpp.cpp @@ -319,7 +319,8 @@ static ts::ConstraintData build_constraint_from_r( Nullable consTipData, Nullable consWeight, Nullable consLevels, - int consExpectedScore); + int consExpectedScore, + Nullable consNegSplitMatrix = R_NilValue); // [[Rcpp::export]] double ts_fitch_score( @@ -1522,7 +1523,8 @@ static ts::ConstraintData build_constraint_from_r( Nullable consTipData, Nullable consWeight, Nullable consLevels, - int consExpectedScore) + int consExpectedScore, + Nullable consNegSplitMatrix) { ts::ConstraintData cd; if (consSplitMatrix.isNotNull()) { @@ -1559,6 +1561,25 @@ static ts::ConstraintData build_constraint_from_r( } } } + + // Negative (converse) constraints: forbidden clades (Bremer support). May be + // supplied alone (cd otherwise empty) or alongside positive splits. + if (consNegSplitMatrix.isNotNull()) { + IntegerMatrix nsm(consNegSplitMatrix.get()); + int n_neg = nsm.nrow(); + if (n_neg > 0) { + // A column-count mismatch would otherwise silently drop the negative + // constraint, running an UNCONSTRAINED converse search and reporting a + // clade's Bremer support as 0 -- a silent wrong answer. Fail loudly. + if (nsm.ncol() != n_tips) { + Rcpp::stop("consNegSplitMatrix has %d column(s) but the dataset has %d " + "tip(s); a negative-constraint matrix needs one column per " + "tip.", nsm.ncol(), n_tips); + } + ts::add_negative_constraint(cd, INTEGER(nsm), n_neg, n_tips); + } + } + return cd; } @@ -1848,6 +1869,9 @@ static ts::ConstraintData unpack_constraint(int n_tips, if (cc.containsElementNamed("consExpectedScore")) consExpectedScore = as(cc["consExpectedScore"]); + SEXP nsm = cc.containsElementNamed("consNegSplitMatrix") + ? SEXP(cc["consNegSplitMatrix"]) : R_NilValue; + return build_constraint_from_r( n_tips, Nullable(csm), @@ -1855,7 +1879,8 @@ static ts::ConstraintData unpack_constraint(int n_tips, Nullable(ctd), Nullable(cw), Nullable(cl), - consExpectedScore); + consExpectedScore, + Nullable(nsm)); } return ts::ConstraintData{}; } @@ -2112,7 +2137,14 @@ List ts_driven_search( ts::TreePool pool(params.pool_max_size, params.pool_suboptimal); ts::DrivenResult result; - if (nThreads != 1) { + // The parallel driven search has no negative-constraint (forbidden-clade) + // backstop: its shared pool is never set_forbidden and tree_fuse is + // constraint-blind, so it could return a tree that displays a forbidden clade. + // The R layer already forces nThreads = 1 (with a warning) whenever a negative + // constraint is set; force it here too, so a direct C++ caller cannot silently + // reach the unguarded path and report an unsound converse-search best. + const bool has_neg_constraint = cd_ptr && cd_ptr->neg_active; + if (nThreads != 1 && !has_neg_constraint) { result = ts::parallel_driven_search(pool, ds, params, cd_ptr, nThreads); } else { result = ts::driven_search(pool, ds, params, cd_ptr); @@ -2302,7 +2334,7 @@ List ts_collapse_pool( } // Group sizes depend only on the constraint, so they are counted once here // rather than per tree. A group of fewer than two taxa is skipped below: - // such a split is realised by a terminal edge, never a collapse candidate. + // such a split is realized by a terminal edge, never a collapse candidate. std::vector n_one_tips(cons_one.size(), 0); std::vector n_zero_tips(cons_one.size(), 0); for (size_t r = 0; r < cons_one.size(); ++r) { @@ -2361,7 +2393,7 @@ List ts_collapse_pool( ts::compute_collapsed_flags_aggressive(tree, ds, flags); - // Protect constraint splits: keep an internal edge that realises each + // Protect constraint splits: keep an internal edge that realizes each // constraint out of the contraction, so the enforced grouping stays // visible. Per-node descendant tip sets via a postorder OR; rooted on // tip 0, so every internal set excludes tip 0. @@ -2379,7 +2411,7 @@ List ts_collapse_pool( const uint64_t* R = &tb[static_cast(tree.right[ni]) * wps]; for (int w = 0; w < wps; ++w) dst[w] = L[w] | R[w]; } - // A node realises the split when it holds one whole group and none of the + // A node realizes the split when it holds one whole group and none of the // other -- ts::node_displays_split() (ts_constraint.h), the same predicate // the search's mapping and the Wagner build read, so the branch protected // here is the branch they enforce. With free tips that node is generally @@ -3575,7 +3607,7 @@ List ts_sankoff_test( // callers can compare average starting-tree quality across criteria. // // bias: 0 = RANDOM, 1 = GOLOBOFF, 2 = ENTROPY -// temperature: softmax temperature (0 = greedy; applied to [0,1]-normalised +// temperature: softmax temperature (0 = greedy; applied to [0,1]-normalized // scores so the parameter is dataset-independent) // n_reps: number of trees to build // run_tbr: if TRUE, run TBR convergence and record its score too diff --git a/src/ts_rng.h b/src/ts_rng.h index 80b13da23..2d3763796 100644 --- a/src/ts_rng.h +++ b/src/ts_rng.h @@ -40,7 +40,7 @@ bool check_interrupt(); // Direct random draws for algorithms that use unif_rand() directly // (e.g., Fisher-Yates in Wagner tree and resampling). // - Serial mode: calls unif_rand() (caller must manage RNG state) -// - Parallel mode: draws from *thread_rng (normalised to [0, 1)) +// - Parallel mode: draws from *thread_rng (normalized to [0, 1)) double thread_safe_unif(); // Begin RNG state access (serial mode: calls GetRNGstate()) diff --git a/src/ts_sankoff.h b/src/ts_sankoff.h index 16a514b5f..e4452b2fa 100644 --- a/src/ts_sankoff.h +++ b/src/ts_sankoff.h @@ -92,7 +92,7 @@ double sankoff_score_char( // optimal_states: output, length n_node. optimal_states[node] = chosen state. // // The uppass assigns root = argmin(node_costs[root][s]) (or forced_root_state), -// then propagates to children choosing the state that minimises +// then propagates to children choosing the state that minimizes // cost_matrix[parent_state][child_state] + node_costs[child][child_state]. void sankoff_uppass( const int* left, const int* right, diff --git a/src/ts_sector.cpp b/src/ts_sector.cpp index 25452eec2..be289d2fa 100644 --- a/src/ts_sector.cpp +++ b/src/ts_sector.cpp @@ -870,7 +870,7 @@ static ReducedDataset build_reduced_dataset_collapsed(const TreeState& tree, // Rebuild the sector's content topology from scratch by RAS Wagner: a random // taxon ORDER with greedy (best-edge) PLACEMENT, keeping the HTU anchored at // the synthetic root AND present throughout, so placements account for the -// rest-of-tree state it summarises. This is the operation +// rest-of-tree state it summarizes. This is the operation // TNT performs per sector (3 RAS+TBR restarts): it reaches sector topologies a // TBR on the *existing* sector subtree cannot, because TBR only locally // rearranges a tree the global TBR has already converged. @@ -1042,10 +1042,10 @@ static void build_ras_sector(ReducedDataset& rd, std::mt19937& rng) { // _free_htu_probe), NOT namespace-scope statics: a namespace-scope dynamic // initializer runs at DLL load, before R can Sys.setenv() after library(), so the // kill switch / stats would silently ignore an env var set from R. Function-local -// statics initialise on the first search_sector call, after any Sys.setenv. +// statics initialize on the first search_sector call, after any Sys.setenv. // Engagement instrumentation counters (drift solves / HTU-float reverts). A high // revert rate means anchored drift is inert -- the signal that motivates pinning -// the HTU (sector_mask). The ++ is unsynchronised across parallel workers, which +// the HTU (sector_mask). The ++ is unsynchronized across parallel workers, which // is acceptable for the opt-in serial trajectory-diff run. static long long g_sect_drift_solves = 0; static long long g_sect_drift_reverts = 0; @@ -1344,7 +1344,7 @@ static double search_sector(ReducedDataset& rd, const SectorParams& params, // start -- and only via the SAME reduced // score + root_ok gate + reinsert path as the drift starts (no reduced-score // shortcut: the reduced length is an HTU APPROXIMATION, so a lower reduced score - // is necessary but the full-tree gain is realised through reinsertion exactly as + // is necessary but the full-tree gain is realized through reinsertion exactly as // for godrift). Requires >= 2 distinct donors; on converged starts fuse is a // no-op and best-of-starts stands. if (do_fuse && fuse_pool.size() >= 2) { @@ -1527,6 +1527,16 @@ SectorResult rss_search(TreeState& tree, DataSet& ds, // (T-S6c micro-bank). static const bool _sect_debug = std::getenv("TS_SECT_DEBUG") != nullptr; bool constrained = cd && cd->active && cd->has_posthoc; + // Negative (converse/Bremer) constraint: the sector-internal reduced-dataset + // solve (search_sector/reinsert_sector) is constraint-blind, so a reinsertion + // can rebuild a forbidden clade even though the pre-sector tree lacked it. + // Reject such a reinsertion post-hoc (mirrors `constrained` above), keeping the + // live tree -- and hence every downstream phase -- in the space of trees + // lacking the clade. Soundness is already guaranteed by the pool backstop; + // this preserves REACH (an un-reverted reinsertion can strand the replicate on + // the clade with no improving escape move). Not covered by the <=7-tip oracle + // (sectors need >=12 tips). + bool neg_constrained = cd && cd->neg_active; // Seed RNG (from R in serial mode, from thread-local in parallel mode) std::mt19937 rng = ts::make_rng(); @@ -1772,6 +1782,16 @@ SectorResult rss_search(TreeState& tree, DataSet& ds, score_tree(tree, ds); continue; } + // Post-hoc negative (converse) constraint check: reject a reinsertion that + // rebuilt a forbidden clade. restore_clade undoes only the sector clade, + // which is the sole change since the pre-sector tree, so the reverted tree + // is clade-free. + if (neg_constrained && displays_forbidden_clade(tree, *cd)) { + restore_clade(tree, snap); + tree.build_postorder(); + score_tree(tree, ds); + continue; + } bool kept; if (new_score < result.best_score) { @@ -1918,6 +1938,11 @@ SectorResult xss_search(TreeState& tree, DataSet& ds, } bool constrained = cd && cd->active && cd->has_posthoc; + // Negative (converse/Bremer) constraint: the reduced-dataset sector solve is + // constraint-blind, so reject post-hoc any reinsertion that rebuilds a + // forbidden clade (see rss_search for the full rationale). REACH-preserving; + // soundness is already guaranteed by the pool backstop. + bool neg_constrained = cd && cd->neg_active; for (int round = 0; round < params.xss_rounds; ++round) { double score_before_round = result.best_score; @@ -1976,6 +2001,13 @@ SectorResult xss_search(TreeState& tree, DataSet& ds, score_tree(tree, ds); continue; } + // Post-hoc negative (converse) constraint check (see rss_search). + if (neg_constrained && displays_forbidden_clade(tree, *cd)) { + restore_clade(tree, snap); + tree.build_postorder(); + score_tree(tree, ds); + continue; + } if (new_score < result.best_score) { result.total_steps_saved += diff --git a/src/ts_simd.h b/src/ts_simd.h index 35a31e185..dffbd9126 100644 --- a/src/ts_simd.h +++ b/src/ts_simd.h @@ -44,7 +44,7 @@ namespace simd { // // Cached flag: evaluated once on first call. On non-x86, always false. -// AVX2 dispatch workaround for unoptimised builds (covr / gcov). +// AVX2 dispatch workaround for unoptimized builds (covr / gcov). // // Under covr on Windows (rtools45 GCC 14 + `-O0 --coverage`) any call // into the `__attribute__((target("avx2")))` helpers crashes the R diff --git a/src/ts_simplify.cpp b/src/ts_simplify.cpp index 6c5cc9a69..55b4c7d35 100644 --- a/src/ts_simplify.cpp +++ b/src/ts_simplify.cpp @@ -190,7 +190,7 @@ SimplificationResult simplify_patterns( // skip the transforms entirely. This also prevents Transform 2 from turning // an all-autapomorphy character (e.g. states 0..7, each in one tip) into a // wildcard-bearing token that the conservative ambiguous check can no - // longer recognise as uninformative — the score would stay correct, but the + // longer recognize as uninformative — the score would stay correct, but the // character would be needlessly rescored per tree. Informative unambiguous // characters fall through so genuine singleton autapomorphies are still // removed by Transform 2. @@ -289,7 +289,7 @@ SimplificationResult simplify_patterns( // reconstruction may resolve an ambiguous neighbour TO s to form a // cost-free clade (e.g. tip {s} beside {s,x} join at zero cost), so // removing s and charging +1 over-counts. The previous `has_dominant` - // guard only proved another state EXISTED, not that the optimiser + // guard only proved another state EXISTED, not that the optimizer // would avoid s — an unsound test that over-counted multistate // ambiguity. Ambiguity resolution is topology-dependent; // simplification must be topology-independent. diff --git a/src/ts_strategy.h b/src/ts_strategy.h index 03880335c..e608a6693 100644 --- a/src/ts_strategy.h +++ b/src/ts_strategy.h @@ -61,7 +61,7 @@ inline bool strategy_is_wagner(StartStrategy s) { class StrategyTracker { public: - // Initialise with default priors. + // Initialize with default priors. // RANDOM_TREE gets Beta(1,2) = pessimistic prior. // All others get Beta(1,1) = uniform. StrategyTracker() { diff --git a/src/ts_tbr.cpp b/src/ts_tbr.cpp index 019594853..693d7609b 100644 --- a/src/ts_tbr.cpp +++ b/src/ts_tbr.cpp @@ -133,7 +133,7 @@ static double full_rescore(TreeState& tree, const DataSet& ds) { // now rooted canonically at tip 0 inside the kernel (ts_hsj.cpp), so HSJ scores // are a function of the unrooted topology. Rebuilds postorder; does NOT refresh // Fitch state arrays, so the caller must full_rescore() afterwards. -// Generalises reroot_at_tip0() in ts_fuse.cpp to an arbitrary tip. +// Generalizes reroot_at_tip0() in ts_fuse.cpp to an arbitrary tip. // Declared in ts_tbr.h (used by the output-collapse kernel in ts_rcpp.cpp). void reroot_at_tip(TreeState& tree, int t) { const int n_tip = tree.n_tip; @@ -1347,7 +1347,7 @@ static void order_clips( case ClipOrder::ANTI_TIP: { // Non-tip clips (shuffled) first, tip clips (shuffled) last. - // Hypothesis: tips are under-productive; deprioritise them. + // Hypothesis: tips are under-productive; deprioritize them. // Inverse of TIPS_FIRST. auto tip_start = std::partition(clips.begin(), clips.end(), [n_tip](int node) { return node >= n_tip; }); // non-tips first @@ -1388,7 +1388,7 @@ static void order_clips( // collapsed / b2_ceiling). This template lifts that dispatch out of the hot // loop: it is instantiated ONCE per weight-class at dispatch (a runtime switch // in tbr_search selects the instantiation), so the dead-in-plain-EW branches -// compile away and the compiler picks a specialised kernel. +// compile away and the compiler picks a specialized kernel. // // The template is called ONLY in the plain-EW regime (no NA, no IW, no sector / // constraint / collapsed / b2), so keeping only the identity skip + scorer + @@ -1409,7 +1409,7 @@ static void order_clips( // Wrong is a positive-control ONLY: the <..,true> instantiations are emitted // solely under -DTS_EW_MONO_WRONG (see the dispatch), so a production binary // has NO code path — and no env var — that can corrupt the scorer. Under that -// build flag it corrupts the output so a live specialised path provably changes +// build flag it corrupts the output so a live specialized path provably changes // the result, distinguishing "path fired + output used" from a no-op that // passes the gate falsely (the false-0% trap documented in s7-fastpath-sizing). template @@ -1542,6 +1542,10 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, // Initialize constraint mapping if active bool constrained = cd && cd->active; + // Negative (converse) constraint: reject any accepted move whose result + // displays a forbidden clade, directing the hill-climb into the space of + // trees that lack it (used for Bremer support). + bool neg_constrained = cd && cd->neg_active; if (constrained) { update_constraint(tree, *cd); } @@ -1558,7 +1562,7 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, // getenv). t_clip_ns = the bail-INDEPENDENT per-clip IW precompute // (extract_char_steps + compute_iw base + iw_delta; no EW analog); t_scan_ns // = the bail-DEPENDENT per-candidate scan (EW x4-flat/popcount vs IW - // scalar-gather), normalised by n_evaluated. No-op unless the var is set. + // scalar-gather), normalized by n_evaluated. No-op unless the var is set. const bool iw_timing = std::getenv("TS_IW_TIMING") != nullptr; long long t_clip_ns = 0, t_spr_ns = 0, t_rer_ns = 0; long long n_clips_t = 0, n_spr_t = 0, n_rer_t = 0; @@ -1741,7 +1745,7 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, // path to a corrupted scorer in a production binary — the same discipline as // the removed TS_EW_MONO_ASSERT probe). Rebuild with -DTS_EW_MONO_WRONG in // PKG_CPPFLAGS, then TS_EW_MONO_WRONG=1 selects the corrupted instantiation - // (must DIVERGE — proves the specialised path's output drives the search). + // (must DIVERGE — proves the specialized path's output drives the search). const bool ew_mono_wrong = std::getenv("TS_EW_MONO_WRONG") != nullptr; #endif // Positive-control fire counters, split by weight class: proving A fast path @@ -2511,7 +2515,7 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, // cutoff is maintained across the clip (recomputed only on // improvement); byte-identical to the old per-batch recompute. int cutoff_b = cutoff; - // Initialise to cutoff_b so partial-batch trailing slots + // Initialize to cutoff_b so partial-batch trailing slots // never accidentally improve best_candidate. int scores[4] = {cutoff_b, cutoff_b, cutoff_b, cutoff_b}; @@ -2924,7 +2928,11 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, // clip phase (the rerooting changes which constraint tips // end up on which side of the attachment edge). Reject // any move that introduces a constraint violation. - if (constrained) { + // `cd->n_splits > 0`: a negative-only constraint (Bremer converse + // search) has `active == true` but zero positive splits, so this whole + // block is a no-op -- skip it to avoid a redundant compute_node_tips + // sweep on top of the displays_forbidden_clade sweep just below. + if (constrained && cd->n_splits > 0) { map_constraint_nodes(tree, *cd); bool violation = false; for (int _s = 0; _s < cd->n_splits; ++_s) { @@ -2943,6 +2951,21 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, } } + // Negative-constraint guard: reject any move whose resulting tree + // displays a forbidden clade. Mirrors the positive post-hoc block + // above (revert the applied move and skip), keeping every accepted + // tree -- and hence best_score and the pool -- free of the clade. + if (neg_constrained && displays_forbidden_clade(tree, *cd)) { + restore_topology(tree, snap); + state_snap.restore(tree); + score_fresh = true; + if (constrained && cd->n_splits > 0) { + map_constraint_nodes(tree, *cd); + compute_dfs_timestamps(tree, *cd); + } + continue; + } + // Compute topology hash for tabu checking uint64_t tree_hash = 0; if (tabu.active()) { @@ -3112,7 +3135,7 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, // Piggyback the existing per-clip poll rather than adding a clock read to // the candidate loop: the profiling campaign left the hot path at-limit, - // and a per-candidate chrono::now() would reopen it. Stride 64 amortises + // and a per-candidate chrono::now() would reopen it. Stride 64 amortizes // the clock read across clips, which are individually cheap. // Silent unless the caller labelled this search -- see TBRParams. if (params.heartbeat_label != nullptr) { @@ -3160,6 +3183,17 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, // fast additive for EW / apply+rescore for IW). NA: the indirect scan is // only approximate, so an EXACT full-neighbourhood sweep is required to // certify a true unrooted-TBR optimum (see exact_verify_sweep). + // Negative (converse) constraint: the root-edge / full-neighbourhood + // sweep below is NOT constraint-guarded (unlike the inner clip loop), so + // its best move can introduce a forbidden clade (a root-edge TBR move + // changes the unrooted topology). Snapshot the clade-free tree first; if + // the improved tree displays the clade, keep the snapshot and stop. We + // are then at a local optimum within the space of trees lacking the clade + // -- conservative (an improving clade-free root-edge move is forgone, + // costing reach) but never unsound (the pool backstop already guarantees + // soundness; this keeps best_score consistent with a ¬C tree). + TopoSnapshot neg_pre_snap; + if (neg_constrained) save_topology(tree, neg_pre_snap); bool improved; if (has_na && !certify_na) { // This caller does not need a certified optimum (see @@ -3194,6 +3228,13 @@ TBRResult tbr_search(TreeState& tree, const DataSet& ds, } else { improved = try_root_edge_moves(tree, ds, best_score, ew_directional); } + if (neg_constrained && improved && displays_forbidden_clade(tree, *cd)) { + restore_topology(tree, neg_pre_snap); + tree.build_postorder(); + best_score = full_rescore(tree, ds); // pre-sweep score; refreshes states + score_fresh = true; + break; + } if (!improved) break; score_fresh = true; if (!collapsed.empty()) { diff --git a/src/ts_tree.cpp b/src/ts_tree.cpp index 353b265b5..c6d884384 100644 --- a/src/ts_tree.cpp +++ b/src/ts_tree.cpp @@ -101,7 +101,7 @@ void TreeState::load_tip_states(const DataSet& ds) { std::memcpy(final_.data(), ds.tip_states.data(), tip_bytes); } - // Initialise tip subtree_actives: applicable states only (NA word = 0). + // Initialize tip subtree_actives: applicable states only (NA word = 0). // For {-,X} tips, applicable state bits are preserved here; the // tip uppass (Pass 2) will clear them if the tip resolves as NA. for (int tip = 0; tip < n_tip; ++tip) { diff --git a/src/ts_wagner.cpp b/src/ts_wagner.cpp index 451c62462..05bb80d1e 100644 --- a/src/ts_wagner.cpp +++ b/src/ts_wagner.cpp @@ -854,7 +854,7 @@ WagnerResult wagner_tree(TreeState& tree, const DataSet& ds, } // Hand the tree on in the rooting the rest of the constraint machinery - // assumes. build_constraint() canonicalises every split mask so that tip 0 is + // assumes. build_constraint() canonicalizes every split mask so that tip 0 is // *outside* it, and map_constraint_nodes() then looks for a node whose subtree // is exactly that mask. Rooting on tip 0 makes the non-tip-0 side of every // displayed split a clade, so that search always succeeds when the tree really @@ -989,7 +989,7 @@ std::vector wagner_entropy_scores(const DataSet& ds) { } // Softmax-weighted sampling without replacement. -// Scores are normalised to [0, 1] before applying temperature so that +// Scores are normalized to [0, 1] before applying temperature so that // the parameter is dataset-independent. temperature == 0 → greedy argmax. // Returns a permutation of 0..n_tip-1 in the sampled addition order. static std::vector softmax_sample_order( @@ -1009,7 +1009,7 @@ static std::vector softmax_sample_order( return idx; } - // Normalise scores to [0, 1] so temperature is dataset-independent + // Normalize scores to [0, 1] so temperature is dataset-independent double mn = *std::min_element(scores.begin(), scores.end()); double mx = *std::max_element(scores.begin(), scores.end()); double rng = mx - mn; diff --git a/src/ts_wagner.h b/src/ts_wagner.h index ab5d786f1..3629e8813 100644 --- a/src/ts_wagner.h +++ b/src/ts_wagner.h @@ -39,15 +39,15 @@ WagnerResult random_wagner_tree(TreeState& tree, const DataSet& ds, // Criterion for biasing taxon addition order. // // RANDOM — uniform random (existing behaviour) -// GOLOBOFF — prioritise taxa with more non-ambiguous parsimony-informative +// GOLOBOFF — prioritize taxa with more non-ambiguous parsimony-informative // characters (Goloboff 2014 "informative" addition sequence) -// ENTROPY — prioritise taxa with more specific (lower-entropy) state codings; +// ENTROPY — prioritize taxa with more specific (lower-entropy) state codings; // score(t) = Σ_c (n_states_c - |state_set of t at c|) // // Both scored criteria use softmax-weighted sampling WITHOUT replacement so // that multiple starts are diverse while biasing toward better basins. // temperature controls selectivity: 0 → greedy argmax; large → near-uniform. -// Temperature is applied to scores normalised to [0, 1] so that behaviour +// Temperature is applied to scores normalized to [0, 1] so that behaviour // is consistent across datasets regardless of character count. enum class WagnerBias { RANDOM = 0, GOLOBOFF = 1, ENTROPY = 2 }; diff --git a/tests/testthat/test-AdditionTree.R b/tests/testthat/test-AdditionTree.R index f73cd86b3..ad0928a9e 100644 --- a/tests/testthat/test-AdditionTree.R +++ b/tests/testthat/test-AdditionTree.R @@ -52,7 +52,7 @@ test_that("ts_wagner_tree: real min_steps changes score, not topology (#5, T-369 concavity <- 10 # Exercises the C++ engine directly (not AdditionTree(), which discards - # `result$score`) to characterise what a `min_steps` of 0 versus the + # `result$score`) to characterize what a `min_steps` of 0 versus the # dataset's real per-character minimum corrupts. at <- attributes(ds) tipData <- matrix(unlist(ds, use.names = FALSE), nrow = length(taxa), @@ -363,7 +363,7 @@ test_that("AdditionTree() verifies its own output against the constraint", { # when the filter rejected *every* edge, which the T-364/T-370 leak never did, # so violating trees came back mutely -- and AdditionTree() never sets # `has_posthoc`, so unlike the search path it has no reshuffle to fall back - # on. Disabling complement enforcement makes 425 of 1334 randomised cases + # on. Disabling complement enforcement makes 425 of 1334 randomized cases # violate; the check caught all 425 and warned on none of the other 909. # # The risk of adding a verifier is spurious warnings, so sweep for silence. diff --git a/tests/testthat/test-Bremer.R b/tests/testthat/test-Bremer.R new file mode 100644 index 000000000..96f98dbd9 --- /dev/null +++ b/tests/testthat/test-Bremer.R @@ -0,0 +1,598 @@ +library("TreeTools", quietly = TRUE) + +data("inapplicable.phyData", package = "TreeSearch") +vinther <- inapplicable.phyData[["Vinther2008"]] + +# Exact Bremer by exhaustive enumeration of every binary tree on the taxa +# (feasible for <= 7 tips): the minimum length among trees LACKING each +# reference clade, minus the global minimum length. The gold-standard oracle +# for the converse-constraint engine. +allBinaryTrees <- function(tips) { + trees <- list(ape::read.tree( + text = paste0("(", tips[1], ",", tips[2], ",", tips[3], ");"))) + for (k in 4:length(tips)) { + trees <- unlist(lapply(trees, function(tr) { + AddTipEverywhere(tr, tips[k], includeRoot = FALSE) + }), recursive = FALSE) + } + trees +} + +oracleBremer <- function(reference, dataset, concavity = Inf) { + tips <- names(dataset) + trees <- allBinaryTrees(tips) + trees <- structure(lapply(trees, RootTree, tips[1]), class = "multiPhylo") + lengths <- TreeLength(trees, dataset, concavity = concavity) + Lstar <- min(lengths) + refFreq <- SplitFrequency(reference, trees) + nSplits <- length(refFreq) + disp <- vapply(seq_along(trees), function(j) { + SplitFrequency(reference, trees[j]) + }, double(nSplits)) + dim(disp) <- c(nSplits, length(trees)) + out <- vapply(seq_len(nSplits), function(i) { + lacking <- disp[i, ] < 0.5 + if (any(lacking)) min(lengths[lacking]) - Lstar else Inf + }, double(1)) + setNames(out, names(refFreq)) +} + +# Enumeration oracle scoring under the inapplicable "bgs" kernel (the DEFAULT +# scoring mode). Validates the converse-constraint engine on data with +# inapplicable (`-`) tokens -- the path that runs exact_verify_sweep. +oracleBremerNA <- function(reference, dataset) { + tips <- names(dataset) + trees <- structure(lapply(allBinaryTrees(tips), RootTree, tips[1]), + class = "multiPhylo") + lengths <- TreeLength(trees, dataset, inapplicable = "bgs") + Lstar <- min(lengths) + refFreq <- SplitFrequency(reference, trees) + nSplits <- length(refFreq) + disp <- vapply(seq_along(trees), function(j) { + SplitFrequency(reference, trees[j]) + }, double(nSplits)) + dim(disp) <- c(nSplits, length(trees)) + setNames(vapply(seq_len(nSplits), function(i) { + lacking <- disp[i, ] < 0.5 + if (any(lacking)) min(lengths[lacking]) - Lstar else Inf + }, double(1)), names(refFreq)) +} + +# --- method = "pool" (approximate engine) --- +# A multiPhylo of several MPTs is summarized by its strict consensus; support +# is calculated for the consensus's resolved clades only. + +test_that("Bremer(method = 'pool') returns node-keyed non-negative support", { + set.seed(3418) + trees <- MaximizeParsimony(vinther, maxReplicates = 6L, targetHits = 2L, + verbosity = 0L) + refCons <- if (length(trees) == 1L) { + trees[[1]] + } else { + TreeTools::Consensus(trees, p = 1) + } + + decay <- Bremer(trees, vinther, method = "pool", maxBremer = 6, + maxReplicates = 6L, targetHits = 2L, verbosity = 0L) + + expect_type(decay, "double") + # One value per resolved clade of the consensus reference. + expect_length(decay, NSplits(refCons)) + # Names are internal node numbers. + expect_true(all(as.integer(names(decay)) > NTip(vinther))) + + # Finite (uncensored) support values are non-negative. + expect_true(all(decay[is.finite(decay)] >= 0)) + + # Censored attribute marks exactly the infinite entries. + cens <- attr(decay, "censored") + expect_type(cens, "logical") + expect_length(cens, length(decay)) + expect_equal(unname(cens), unname(!is.finite(decay))) +}) + +test_that("Bremer(format = 'character') yields a node.label-shaped vector", { + set.seed(3418) + trees <- MaximizeParsimony(vinther, maxReplicates = 6L, targetHits = 2L, + verbosity = 0L) + refCons <- if (length(trees) == 1L) { + trees[[1]] + } else { + TreeTools::Consensus(trees, p = 1) + } + + lab <- Bremer(trees, vinther, method = "pool", maxBremer = 6, + format = "character", maxReplicates = 6L, targetHits = 2L, + verbosity = 0L) + expect_type(lab, "character") + expect_length(lab, refCons[["Nnode"]]) + # Assignable straight onto the reference tree. + refCons[["node.label"]] <- lab + expect_length(refCons[["node.label"]], refCons[["Nnode"]]) +}) + +test_that("Bremer accepts a single phylo reference with an explicit L*", { + set.seed(3418) + trees <- MaximizeParsimony(vinther, maxReplicates = 6L, targetHits = 2L, + verbosity = 0L) + Lstar <- attr(trees, "score") + decay <- Bremer(trees[[1]], vinther, method = "pool", maxBremer = 6, + optimalScore = Lstar, + maxReplicates = 6L, targetHits = 2L, verbosity = 0L) + expect_type(decay, "double") + expect_length(decay, NSplits(trees[[1]])) + expect_true(all(decay[is.finite(decay)] >= 0)) + expect_setequal(names(decay), rownames(as.matrix(as.Splits(trees[[1]])))) +}) + +test_that("Bremer errors on a bad reference type", { + expect_error(Bremer("not a tree", vinther, method = "pool"), + "must be a `phylo` or `multiPhylo`") +}) + +# --- method = "constraint" (rigorous engine): validated against brute force --- + +test_that("the oracle enumerates every binary tree (guards the gate itself)", { + # (2n - 5)!! unrooted binary trees; a gappy enumerator would make the oracle + # itself an over-estimate that could rubber-stamp an over-estimating search. + doubleFactorial <- function(m) prod(seq(m, 1, by = -2)) + expect_equal(length(allBinaryTrees(LETTERS[1:6])), doubleFactorial(2 * 6 - 5)) + expect_equal(length(allBinaryTrees(c(LETTERS[1:6], "out"))), + doubleFactorial(2 * 7 - 5)) +}) + +test_that("constraint Bremer matches enumeration when L* is computed internally", { + # Exercises the full-strength L* search path (no optimalScore supplied). + dat <- StringToPhyDat( + "1110000 1110000 1110000 1100000 0001100 0001100 0000110 1111111", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) + ref <- mpts[[1]] + con <- Bremer(ref, dat, method = "constraint", # no optimalScore -> internal L* + maxReplicates = 25L, verbosity = 0L) + orc <- oracleBremer(ref, dat) + expect_equal(as.numeric(con[names(orc)]), as.numeric(orc), tolerance = 1e-6) +}) + +test_that("constraint Bremer matches exact enumeration (graded pectinate)", { + dat <- StringToPhyDat( + "1100000 1110000 1111000 1111100 1100000 1110000 1111000 1111100 1001000", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) + ref <- mpts[[1]] + con <- Bremer(ref, dat, method = "constraint", + optimalScore = attr(mpts, "score"), + maxReplicates = 25L, verbosity = 0L) + orc <- oracleBremer(ref, dat) + expect_equal(as.numeric(con[names(orc)]), as.numeric(orc), tolerance = 1e-6) +}) + +test_that("constraint Bremer matches enumeration in the adversarial case", { + # {A,B,C} is supported by three characters (decay 3); the optimum DISPLAYS it + # and the nearest tree lacking it is three steps longer -- the case a stalling + # post-hoc filter would over-estimate. + dat <- StringToPhyDat( + "1110000 1110000 1110000 1100000 0001100 0001100 0000110 1111111", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) + ref <- mpts[[1]] + con <- Bremer(ref, dat, method = "constraint", + optimalScore = attr(mpts, "score"), + maxReplicates = 25L, verbosity = 0L) + orc <- oracleBremer(ref, dat) + expect_equal(as.numeric(con[names(orc)]), as.numeric(orc), tolerance = 1e-6) + expect_gte(max(con), 3) +}) + +test_that("constraint Bremer matches enumeration under implied weights", { + dat <- StringToPhyDat( + "1110000 1110000 1110000 1100000 0001100 0001100 0000110 1111111", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, concavity = 3, maxReplicates = 8L, + verbosity = 0L) + ref <- mpts[[1]] + con <- Bremer(ref, dat, method = "constraint", concavity = 3, + optimalScore = attr(mpts, "score"), + maxReplicates = 25L, verbosity = 0L) + orc <- oracleBremer(ref, dat, concavity = 3) + expect_equal(as.numeric(con[names(orc)]), as.numeric(orc), tolerance = 1e-6) + # Fractional decay under implied weights. + expect_false(all(con == round(con))) +}) + +test_that("constraint is the default and never looser than an uncensored pool", { + # The matrix leaves parts of the tree unresolved, so a fully-resolved reference + # has decay-0 clades that OTHER equally-optimal trees lack. A best-score pool + # therefore breaks them, making the cross-engine comparison NON-vacuous (guards + # the earlier bug where a consensus reference left every shared clade censored + # and the key assertion compared empty vectors). + dat <- StringToPhyDat("1111000 1111000 1100000", 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(42) + mptsFull <- MaximizeParsimony(dat, collapse = FALSE, maxReplicates = 12L, + verbosity = 0L) + ref <- mptsFull[[1]] # a fully-resolved (binary) MPT + Lstar <- attr(mptsFull, "score") + + con <- Bremer(ref, dat, optimalScore = Lstar, maxReplicates = 20L, + verbosity = 0L) # default method (constraint) + expect_type(con, "double") + expect_true(all(con >= 0)) + expect_true(all(is.finite(con))) + + pool <- Bremer(ref, dat, method = "pool", optimalScore = Lstar, maxBremer = 10, + maxReplicates = 50L, verbosity = 0L) + shared <- intersect(names(con), names(pool)) + uncensored <- shared[is.finite(pool[shared])] + expect_gt(length(uncensored), 0L) # the pool broke at least one clade + # Both are upper bounds on the true decay; the directed constraint search is + # never looser than the incidental pool where the pool is uncensored. + expect_true(all(con[uncensored] <= pool[uncensored] + 1e-6)) +}) + +test_that("Bremer warns (does not error) on an inconsistent optimalScore, then proceeds", { + dat <- StringToPhyDat("1100000 1110000 1111000 1111100 1001000", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) + ref <- mpts[[1]] + # A supplied L* inconsistent with the reference length -> single WARNING. + expect_warning( + con <- Bremer(ref, dat, method = "constraint", + optimalScore = attr(mpts, "score") - 3, + maxReplicates = 20L, verbosity = 0L), + "3. differs from 6, the score of `tree`") + expect_type(con, "double") +}) + +test_that("a contradictory optimalScore warns even with a matching signature (B-F1)", { + dat <- StringToPhyDat("1100000 1110000 1111000 1111100 1001000", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) + # The whole multiPhylo carries a "scoring" signature. A matching signature + # validates the scoring MODE but not the numeric VALUE of a supplied L*, so a + # contradictory optimalScore must STILL warn. + expect_warning( + con <- Bremer(mpts, dat, method = "constraint", + optimalScore = attr(mpts, "score") - 2, + maxReplicates = 20L, verbosity = 0L), + "4. differs from 6, the score of `tree`") + expect_type(con, "double") + # Control: the default (correct L*, matching signature) does not false-warn. + set.seed(1) + expect_warning(Bremer(mpts, dat, maxReplicates = 12L, verbosity = 0L), NA) +}) + +test_that("Bremer strips a managed `collapse` from ... with a warning (B-F4)", { + dat <- StringToPhyDat("1111000 1111000 1100000", 1:7, byTaxon = FALSE) + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 6L, verbosity = 0L) + ref <- mpts[[1]] + # A reserved MaximizeParsimony formal passed through `...` previously collided + # in do.call ("matched by multiple actual arguments"); it is now stripped with + # a warning, mirroring SuboptimalTrees(). + expect_warning( + con <- Bremer(ref, dat, method = "constraint", collapse = FALSE, + optimalScore = attr(mpts, "score"), + maxReplicates = 12L, verbosity = 0L), + "manages") + expect_type(con, "double") +}) + +test_that("constraint Bremer matches enumeration on inapplicable (bgs) data", { + # The default scoring mode (inapplicable = "bgs") previously had no ground- + # truth oracle. Validate the negative-constraint machinery + exact_verify_sweep + # against exact enumeration on a 6-taxon matrix with inapplicable (`-`) tokens. + dat <- StringToPhyDat("111000 111000 111000 110000 000110 -00-11 -00-11", + 1:6, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:5], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 10L, verbosity = 0L) + ref <- mpts[[1]] + con <- Bremer(ref, dat, method = "constraint", + optimalScore = attr(mpts, "score"), + maxReplicates = 30L, verbosity = 0L) + orc <- oracleBremerNA(ref, dat) + expect_equal(as.numeric(con[names(orc)]), as.numeric(orc), tolerance = 1e-6) +}) + +test_that("Bremer warns on a likely scoring-units mismatch (IW trees vs EW default)", { + dat <- StringToPhyDat( + "1110000 1110000 1110000 1100000 0001100 0001100 0000110 1111111", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, concavity = 3, maxReplicates = 8L, verbosity = 0L) + # Trees found under implied weights; the default equal-weights Bremer would + # subtract an IW optimum from EW lengths. The recorded scoring signature makes + # this an EXACT criterion mismatch (concavity 3 vs Inf), so it WARNS (accepting + # and proceeding) rather than erroring or silently mis-scoring (BR-1/B-7). + expect_warning(Bremer(mpts, dat, maxReplicates = 8L, verbosity = 0L), + "found under") + # Passing the matching concavity is accepted without the mismatch warning. + ok <- suppressWarnings( + Bremer(mpts, dat, concavity = 3, maxReplicates = 12L, verbosity = 0L)) + expect_type(ok, "double") +}) + +test_that("MaximizeParsimony records a scoring signature Bremer trusts (B-7 provenance)", { + dat <- StringToPhyDat( + "1110000 1110000 1110000 1100000 0001100 0001100 0000110 1111111", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) # EW default + sig <- attr(mpts, "scoring") + expect_false(is.null(sig)) + expect_true(is.infinite(sig[["concavity"]])) # equal weights + expect_identical(sig[["inapplicable"]], "bgs") # default kernel + + # Matching scoring arguments -> the exact provenance check passes -> no warning + # (and, being all defaults here, this is the canonical Bremer() call). + expect_warning( + con <- Bremer(mpts, dat, maxReplicates = 12L, verbosity = 0L), NA) + expect_true(all(is.finite(con))) + + # An IW analysis under matching concavity also matches its own signature. + set.seed(1) + mptsIW <- MaximizeParsimony(dat, concavity = 5, maxReplicates = 8L, + verbosity = 0L) + expect_equal(attr(mptsIW, "scoring")[["concavity"]], 5) + expect_warning( + Bremer(mptsIW, dat, concavity = 5, maxReplicates = 8L, verbosity = 0L), NA) +}) + +test_that("the scoring-mismatch warning catches a small gap the 5% band missed (B-7)", { + dat <- StringToPhyDat( + "1110000 1110000 1110000 1100000 0001100 0001100 0000110 1111111", + 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) + ref <- mpts[[1]] + Lstar <- attr(mpts, "score") + # A gap of 0.3 is still a gap and should warn once. + expect_warning( + Bremer(ref, dat, method = "constraint", optimalScore = Lstar - 0.3, + maxReplicates = 15L, verbosity = 0L), + "7.7. differs from 8, the score of `tree`") +}) + +test_that("Bremer ignores (with a warning) drift/anneal args in a converse search", { + dat <- StringToPhyDat("1111000 1111000 1100000", 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) + # driftCycles is a documented forwarded arg but unsound in a converse search; + # it must be ignored (with a warning), not silently empty the pool (BR-2). + expect_warning( + con <- Bremer(mpts, dat, driftCycles = 5L, annealCycles = 3L, + maxReplicates = 12L, verbosity = 0L), + "always disabled") + expect_true(all(is.finite(con))) +}) + +# --- method = "constraint" on >= 12 tips: exercises the SECTORIAL search path --- +# rss_search / xss_search engage once a tree has >= 12 tips. Their reduced- +# dataset sector solve is constraint-blind, so a reinsertion could rebuild a +# forbidden clade; the fix rejects such a reinsertion post-hoc (a reach guard -- +# the pool backstop already guaranteed soundness). This whole path is +# structurally invisible to the <= 7-tip enumeration oracle above (sectors need +# >= 12 tips), so it needs its own coverage. + +# A graded pectinate matrix on `nIn` ingroup taxa + one outgroup: nested clades +# {t1..tk} (k = 2..nIn-1), each duplicated for graded support. +sectorFixture <- function(nIn = 13) { + tips <- c(paste0("t", seq_len(nIn)), "out") + nTip <- length(tips) + mkChar <- function(k) paste0(c(rep("1", k), rep("0", nTip - k)), collapse = "") + chars <- unlist(lapply(rep(2:(nIn - 1L), each = 2L), mkChar)) + dat <- StringToPhyDat(paste(chars, collapse = " "), seq_len(nTip), + byTaxon = FALSE) + names(dat) <- tips + dat +} + +test_that("converse search on >= 12 tips (sectors engaged) returns clade-free trees", { + skip_on_cran() + dat <- sectorFixture(13) # 14 tips -> sectorial search engages + expect_gte(length(dat), 12L) + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 6L, verbosity = 0L) + ref <- mpts[[1]] + splits <- as.Splits(ref, tipLabels = names(dat)) + splitNames <- rownames(as.matrix(splits)) + expect_gt(length(splitNames), 0L) + + # Directly drive the negative-constraint sector path for a spread of clades and + # confirm each returned tree genuinely LACKS its forbidden clade. The pool + # backstop guarantees this regardless of the sector fix, so it is a robust + # soundness assertion; the fix's REACH benefit (fewer stranded replicates) is a + # Hamilton-scale measurement, not asserted here. + probe <- unique(round(seq(1, length(splitNames), + length.out = min(4L, length(splitNames))))) + for (i in probe) { + set.seed(100L + i) + res <- MaximizeParsimony(dat, collapse = TRUE, nThreads = 1L, + .negativeConstraint = splits[[i]], + driftCycles = 0L, sectorGoDrift = 0L, + sectorDriftCycles = 0L, annealCycles = 0L, + maxReplicates = 12L, verbosity = 0L) + tr <- if (inherits(res, "phylo")) res else res[[1L]] + disp <- SplitFrequency(ref, structure(list(tr), class = "multiPhylo")) + expect_lt(unname(disp[splitNames[i]]), 0.5) # returned tree lacks the clade + } +}) + +test_that("a negative constraint forces serial search with a warning (A-F2/A-F3)", { + skip_on_cran() + dat <- sectorFixture(13) + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 6L, verbosity = 0L) + ref <- mpts[[1]] + splits <- as.Splits(ref, tipLabels = names(dat)) + splitNames <- rownames(as.matrix(splits)) + # The parallel driven search has no forbidden-clade backstop, so a negative + # constraint must be clamped to serial (with a warning) at the R layer -- and, + # defensively, in C++. Confirm the warning fires and the returned tree still + # lacks the forbidden clade even though nThreads > 1 was requested. + set.seed(2) + expect_warning( + res <- MaximizeParsimony(dat, collapse = TRUE, nThreads = 2L, + .negativeConstraint = splits[[1]], + driftCycles = 0L, sectorGoDrift = 0L, + sectorDriftCycles = 0L, annealCycles = 0L, + maxReplicates = 8L, verbosity = 0L), + regexp = "serial", ignore.case = TRUE) + tr <- if (inherits(res, "phylo")) res else res[[1L]] + disp <- SplitFrequency(ref, structure(list(tr), class = "multiPhylo")) + expect_lt(unname(disp[splitNames[1]]), 0.5) +}) + +test_that("constraint Bremer on >= 12 tips is non-negative and <= pool", { + skip_on_cran() + dat <- sectorFixture(13) + set.seed(2) + mpts <- MaximizeParsimony(dat, maxReplicates = 6L, verbosity = 0L) + ref <- mpts[[1]] + Lstar <- attr(mpts, "score") + + con <- Bremer(ref, dat, method = "constraint", optimalScore = Lstar, + maxReplicates = 10L, verbosity = 0L) + expect_type(con, "double") + expect_true(any(is.finite(con))) # the sector path did not strand + expect_true(all(con[is.finite(con)] >= 0)) # correct L* (no negative decay) + + pool <- Bremer(ref, dat, method = "pool", optimalScore = Lstar, maxBremer = 12, + maxReplicates = 30L, verbosity = 0L) + shared <- intersect(names(con), names(pool)) + uncensored <- shared[is.finite(pool[shared]) & is.finite(con[shared])] + # Both are upper bounds on the true decay; the directed constraint search is + # never looser than the incidental pool where both are finite. + expect_true(all(con[uncensored] <= pool[uncensored] + 1e-6)) +}) + +# --- input validation / format consistency (red-team round 2) --- + +test_that("Bremer rejects a non-finite or non-scalar optimalScore (B-8)", { + dat <- StringToPhyDat("1111000 1111000 1100000", 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, maxReplicates = 8L, verbosity = 0L) + # NA_real_ slips past is.null() and previously crashed the one-sided scoring + # guard with "missing value where TRUE/FALSE needed"; now a clear error. + expect_error( + Bremer(mpts, dat, optimalScore = NA_real_, maxReplicates = 6L, + verbosity = 0L), + "single finite number") + expect_error( + Bremer(mpts, dat, optimalScore = c(1, 2), maxReplicates = 6L, + verbosity = 0L), + "single finite number") +}) + +test_that("Bremer on a fully-unresolved reference warns and returns nothing (TA-9)", { + dat <- StringToPhyDat("111000 110000 001100", 1:6, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:5], "out") + star <- ape::stree(length(dat), type = "star", tip.label = names(dat)) + # No resolved internal clades -> nothing to calculate; early return, no search. + expect_warning( + res <- Bremer(star, dat, verbosity = 0L), + "no resolved internal clades") + expect_type(res, "double") + expect_length(res, 0L) +}) + +test_that("Bremer(format='character') always carries a censored attribute (B-9)", { + dat <- StringToPhyDat("1111000 1111000 1100000", 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, collapse = FALSE, maxReplicates = 8L, + verbosity = 0L) + ref <- mpts[[1]] + Lstar <- attr(mpts, "score") + # method = "constraint" never censors, so this is the "nothing censored" path + # where the character format previously dropped the attribute entirely while + # the numeric format kept an all-FALSE vector. + lab <- Bremer(ref, dat, optimalScore = Lstar, format = "character", + maxReplicates = 12L, verbosity = 0L) + cens <- attr(lab, "censored") + expect_false(is.null(cens)) # attribute is present... + expect_type(cens, "logical") + expect_length(cens, ref[["Nnode"]]) # ...node.label-shaped... + expect_false(any(cens)) # ...and correctly all-FALSE. + # The numeric format for the same call also carries a censored attribute. + num <- Bremer(ref, dat, optimalScore = Lstar, + maxReplicates = 12L, verbosity = 0L) + expect_false(is.null(attr(num, "censored"))) +}) + +# --- engine-failure handling, via a test seam (no real search needed) --- +# These deterministically drive the NA / clade-displaying paths that the real +# engine (C++ guard + pool backstop) is designed never to reach, so they cannot +# be provoked by an ordinary search. + +test_that(".BremerProcessResult maps engine sentinels to NA (BR-6/TA-4)", { + dat <- StringToPhyDat("1111000 1111000 1100000", 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, collapse = FALSE, maxReplicates = 8L, + verbosity = 0L) + ref <- mpts[[1]] + sn <- rownames(as.matrix(as.Splits(ref, tipLabels = names(dat))))[[1]] + + # "No tree lacking the clade found" sentinel (negative / non-finite) -> NA. + expect_identical( + TreeSearch:::.BremerProcessResult(list(score = -1, tree = NULL), ref, sn), + NA_real_) + expect_identical( + TreeSearch:::.BremerProcessResult(list(score = NA_real_, tree = NULL), ref, sn), + NA_real_) + + # A returned tree that STILL displays the clade (the reference does) -> the + # BR-6 defence-in-depth check warns and returns NA rather than a deflated decay. + expect_warning( + r <- TreeSearch:::.BremerProcessResult(list(score = 5, tree = ref), ref, sn), + "still displays it") + expect_identical(r, NA_real_) + + # A genuinely clade-free tree (a star displays no clade) -> score passes through. + star <- ape::stree(length(dat), type = "star", tip.label = names(dat)) + expect_identical( + TreeSearch:::.BremerProcessResult(list(score = 5, tree = star), ref, sn), 5) +}) + +test_that(".BremerConstraint emits the aggregate NA warning (VF-4/TA-3)", { + dat <- StringToPhyDat("1111000 1111000 1100000", 1:7, byTaxon = FALSE) + names(dat) <- c(LETTERS[1:6], "out") + set.seed(1) + mpts <- MaximizeParsimony(dat, collapse = FALSE, maxReplicates = 8L, + verbosity = 0L) + ref <- TreeSearch:::.BremerReference(mpts, dat, + optimalScore = attr(mpts, "score")) + expect_gte(length(ref$splitNames), 2L) + sa <- list(concavity = Inf, extended_iw = TRUE, xpiwe_r = 0.5, xpiwe_max_f = 5, + hierarchy = NULL, inapplicable = "bgs", hsj_alpha = 1.0) + + # Inject an engine stub returning the "no tree found" sentinel for every clade, + # so every converse search NAs and the aggregate warning fires. Lstar is + # supplied via the reference, so no real search runs. + stubNA <- function(negSplit) list(score = -1, tree = NULL) + expect_warning( + res <- TreeSearch:::.BremerConstraint(ref, dat, sa, Inf, list(), + .runConverse = stubNA), + "returned NA") + expect_length(res$bremer, length(ref$splitNames)) + expect_true(all(is.na(res$bremer))) +}) diff --git a/tests/testthat/test-Concordance.R b/tests/testthat/test-Concordance.R index 99a36bbdb..f4a6ac271 100644 --- a/tests/testthat/test-Concordance.R +++ b/tests/testthat/test-Concordance.R @@ -216,7 +216,7 @@ test_that("QuartetConcordance() unit = 'nrqs' scores 1 iff split displayed", { # A multistate character need not be *identical* to a split to score 1: it # scores full marks for every split its own tree displays (each state block # wholly on one side; the split-orthogonal state-pair drops via M = 0). This - # generalises "only identical scores 1" from binary to multistate -- a cleanly + # generalizes "only identical scores 1" from binary to multistate -- a cleanly # refining reproductive character fully supports the clade it refines. tree <- ape::read.tree(text = "((((t1,t2),t3),((t4,t5),t6)),((t7,t8),t9));") char <- MatrixToPhyDat(matrix( diff --git a/tests/testthat/test-MaddisonSlatkin.R b/tests/testthat/test-MaddisonSlatkin.R index 7e126826d..0fbfd18b2 100644 --- a/tests/testthat/test-MaddisonSlatkin.R +++ b/tests/testthat/test-MaddisonSlatkin.R @@ -194,17 +194,11 @@ test_that("StepInformation() falls back instead of hanging when the exact memo c # (observed as a 6 h --run-donttest CI timeout). The solver must now detect # the impending overflow and fall back to the MC approximation instead. # - # This character no longer overflows a memo table -- its peak demand is ~12k - # entries, within the capacity the tables now reserve -- but neither does it - # fit the 2 s budget, so it still falls back, now by the clock rather than by - # the table. Which guard wins is deliberately not asserted, nor that one - # wins at all: that is a property of how fast the machine is, and a quick - # enough one will simply finish. A fallback warning is likewise NOT - # required; requiring one would fail on exactly the machines that need no - # fallback. What the hang violated -- terminates, with usable values -- is - # what is asserted. - char <- rep(c("0", "1", "2"), c(42L, 9L, 2L)) # == inapplicable Agnarsson2004 col 83 - si <- StepInformation(char, n_mc = 1000L) + # This character does not overflow a memo table, but takes more than 2 s + # (on some machines!), so may succeed or fall back to mc by the clock. + # The assertion simply requires that the call terminates without hanging. + char <- rep(c("0", "1", "2"), c(42L, 9L, 2L)) # == Agnarsson2004 col 83 + si <- suppressWarnings(StepInformation(char, maxSeconds = 0.01, mcSamples = 24L)) expect_type(si, "double") expect_true(length(si) >= 1L && all(is.finite(si))) }) @@ -229,7 +223,7 @@ test_that("An oversized exact recursion falls back rather than running away", { char <- rep(c("0", "1", "2"), c(60L, 12L, 3L)) si <- NULL warningsSeen <- capture_warnings( - si <- StepInformation(char, approx = "exact", n_mc = 1000L)) + si <- StepInformation(char, approx = "exact", mcSamples = 1000L)) expect_type(si, "double") expect_true(length(si) >= 1L && all(is.finite(si))) expect_match(paste(warningsSeen, collapse = "\n"), "exceeded") diff --git a/tests/testthat/test-MaximizeParsimony-features.R b/tests/testthat/test-MaximizeParsimony-features.R index 91daaadbe..85d3ef08f 100644 --- a/tests/testthat/test-MaximizeParsimony-features.R +++ b/tests/testthat/test-MaximizeParsimony-features.R @@ -1,7 +1,7 @@ library("TreeTools", quietly = TRUE) data("inapplicable.phyData", package = "TreeSearch") -ds <- inapplicable.phyData[["Vinther2008"]] +vinther <- inapplicable.phyData[["Vinther2008"]] # --- Input validation --- @@ -19,24 +19,24 @@ test_that("MaximizeParsimony rejects maxReplicates < 1 (T-341)", { # empty-pool fallback silently returned the random starting tree tagged # with that bogus score instead of erroring. expect_error( - MaximizeParsimony(ds, maxReplicates = 0L, targetHits = 1L, + MaximizeParsimony(vinther, maxReplicates = 0L, targetHits = 1L, verbosity = 0L), "`maxReplicates` must be" ) expect_error( - MaximizeParsimony(ds, maxReplicates = -1L, targetHits = 1L, + MaximizeParsimony(vinther, maxReplicates = -1L, targetHits = 1L, verbosity = 0L), "`maxReplicates` must be" ) expect_error( - MaximizeParsimony(ds, maxReplicates = NA_integer_, targetHits = 1L, + MaximizeParsimony(vinther, maxReplicates = NA_integer_, targetHits = 1L, verbosity = 0L), "`maxReplicates` must be" ) }) test_that("replicate-adequacy warning uses unscaled character count (T-342)", { - # `weight` is the .ScaleWeight()-integerised value (up to ~1260x for + # `weight` is the .ScaleWeight()-integerized value (up to ~1260x for # fractional weights); the printed `nChars` must reflect the true number # of characters, not that internal scale factor. The warning only fires # for nTip >= 30, so use a synthetic dataset large enough to trigger it. @@ -46,11 +46,11 @@ test_that("replicate-adequacy warning uses unscaled character count (T-342)", { dimnames = list(paste0("t", 1:30), NULL))) attr(dat, "weight") <- rep(0.5, attr(dat, "nr")) nCharsTrue <- sum(attr(dat, "weight")) - expect_warning( - MaximizeParsimony(dat, maxReplicates = 1L, targetHits = 1L, - verbosity = 1L), - paste0(nCharsTrue, " characters") - ) + capture.output(suppressMessages( + expect_warning(MaximizeParsimony(dat, maxReplicates = 1L, + targetHits = 1L, verbosity = 1L), + paste0(nCharsTrue, " characters")) + )) }) # --- Strategy presets --- @@ -58,13 +58,13 @@ test_that("replicate-adequacy warning uses unscaled character count (T-342)", { test_that("the sprint rung runs and returns a valid result", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(3418) - result <- MaximizeParsimony(ds, effort = -9L, + result <- MaximizeParsimony(vinther, effort = -9L, maxReplicates = 2L, targetHits = 1L, verbosity = 0L) expect_s3_class(result, "multiPhylo") expect_true(is.finite(attr(result, "score"))) expect_true(attr(result, "score") > 0) - expect_equal(NTip(result[[1]]), NTip(ds)) + expect_equal(NTip(result[[1]]), NTip(vinther)) }) test_that("candidates_evaluated attribute is reported for serial search", { @@ -72,7 +72,7 @@ test_that("candidates_evaluated attribute is reported for serial search", { # Diagnostic counter (TNT "rearrangements examined" analogue): a positive, # finite scalar for a single-threaded search. See MaximizeParsimony @return. set.seed(3418) - result <- MaximizeParsimony(ds, effort = -9L, + result <- MaximizeParsimony(vinther, effort = -9L, maxReplicates = 2L, targetHits = 1L, nThreads = 1L, verbosity = 0L) ce <- attr(result, "candidates_evaluated") @@ -84,7 +84,7 @@ test_that("candidates_evaluated attribute is reported for serial search", { test_that("the default rung runs and returns a valid result", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(5726) - result <- MaximizeParsimony(ds, .rung = "default", + result <- MaximizeParsimony(vinther, .rung = "default", maxReplicates = 2L, targetHits = 1L, verbosity = 0L) expect_s3_class(result, "multiPhylo") @@ -94,7 +94,7 @@ test_that("the default rung runs and returns a valid result", { test_that("the thorough rung runs and returns a valid result", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(8103) - result <- MaximizeParsimony(ds, .rung = "thorough", + result <- MaximizeParsimony(vinther, .rung = "thorough", maxReplicates = 1L, targetHits = 1L, verbosity = 0L) expect_s3_class(result, "multiPhylo") @@ -134,7 +134,7 @@ test_that("effort = 0 selects the rung from dataset size", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(2944) # Vinther2008 has 23 tips -> should auto-select "sprint" - result <- MaximizeParsimony(ds, effort = 0L, + result <- MaximizeParsimony(vinther, effort = 0L, maxReplicates = 2L, targetHits = 1L, verbosity = 0L) expect_s3_class(result, "multiPhylo") @@ -144,7 +144,7 @@ test_that("effort = 0 selects the rung from dataset size", { test_that("the no-preset escape uses raw parameter defaults", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(6017) - result <- MaximizeParsimony(ds, .rung = "none", + result <- MaximizeParsimony(vinther, .rung = "none", maxReplicates = 2L, targetHits = 1L, ratchetCycles = 1L, driftCycles = 0L, verbosity = 0L) @@ -156,7 +156,7 @@ test_that("explicit params override the rung preset", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(1589) # Sprint has driftCycles=0; override to 1 - result <- MaximizeParsimony(ds, effort = -9L, + result <- MaximizeParsimony(vinther, effort = -9L, driftCycles = 1L, maxReplicates = 2L, targetHits = 1L, verbosity = 0L) @@ -169,13 +169,13 @@ test_that("`effort` rejects a non-integer offset", { # "unknown strategy" fallback to warn about. What can go wrong instead is a # value that is not a whole number, which must error rather than silently # truncate -- a search quietly run at the wrong rung is worse than a stop. - expect_error(MaximizeParsimony(ds, effort = 1.5, maxReplicates = 1L, + expect_error(MaximizeParsimony(vinther, effort = 1.5, maxReplicates = 1L, verbosity = 0L), "whole number") - expect_error(MaximizeParsimony(ds, effort = "thorough", maxReplicates = 1L, + expect_error(MaximizeParsimony(vinther, effort = "thorough", maxReplicates = 1L, verbosity = 0L), "whole number") - expect_error(MaximizeParsimony(ds, effort = c(1L, 2L), maxReplicates = 1L, + expect_error(MaximizeParsimony(vinther, effort = c(1L, 2L), maxReplicates = 1L, verbosity = 0L), "whole number") }) @@ -254,10 +254,10 @@ test_that("a raised enumMaxTrees can return more trees than poolMaxSize", { # DrivenParams -> TreePool::raise_max_size(). skip_on_cran() set.seed(90210) - small <- MaximizeParsimony(ds, maxReplicates = 3L, targetHits = 99L, + small <- MaximizeParsimony(vinther, maxReplicates = 3L, targetHits = 99L, poolMaxSize = 4L, verbosity = 0L) set.seed(90210) - big <- MaximizeParsimony(ds, maxReplicates = 3L, targetHits = 99L, + big <- MaximizeParsimony(vinther, maxReplicates = 3L, targetHits = 99L, poolMaxSize = 4L, enumMaxTrees = 40L, verbosity = 0L) expect_lte(length(small), 4L) @@ -287,7 +287,7 @@ test_that("maxSeconds stops search before maxReplicates", { # The timeout is checked between replicates, so the first may complete # but subsequent ones won't start. - result <- MaximizeParsimony(ds, maxReplicates = 1000L, targetHits = 1000L, + result <- MaximizeParsimony(vinther, maxReplicates = 1000L, targetHits = 1000L, maxSeconds = 0.001, verbosity = 0L) expect_s3_class(result, "multiPhylo") expect_lt(attr(result, "replicates"), 1000L) @@ -296,7 +296,7 @@ test_that("maxSeconds stops search before maxReplicates", { test_that("maxSeconds = 0 means no timeout", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(8456) - result <- MaximizeParsimony(ds, maxReplicates = 2L, targetHits = 1L, + result <- MaximizeParsimony(vinther, maxReplicates = 2L, targetHits = 1L, maxSeconds = 0, verbosity = 0L) expect_s3_class(result, "multiPhylo") expect_false(attr(result, "timed_out")) @@ -313,7 +313,7 @@ test_that("verbosity = 1 prints 'Search complete' summary to console", { msg_lines <- character() stdout_lines <- capture.output( msg_lines <- capture.output( - MaximizeParsimony(ds, maxReplicates = 2L, targetHits = 1L, + MaximizeParsimony(vinther, maxReplicates = 2L, targetHits = 1L, verbosity = 1L), type = "message" ) @@ -328,7 +328,7 @@ test_that("verbosity = 1 prints 'Search complete' summary to console", { test_that("nThreads = 1 (serial) runs correctly", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(5193) - result <- MaximizeParsimony(ds, maxReplicates = 2L, targetHits = 1L, + result <- MaximizeParsimony(vinther, maxReplicates = 2L, targetHits = 1L, nThreads = 1L, verbosity = 0L) expect_s3_class(result, "multiPhylo") expect_true(is.finite(attr(result, "score"))) @@ -337,7 +337,7 @@ test_that("nThreads = 1 (serial) runs correctly", { test_that("nThreads = 2 (parallel) runs correctly", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(6274) - result <- MaximizeParsimony(ds, maxReplicates = 2L, targetHits = 1L, + result <- MaximizeParsimony(vinther, maxReplicates = 2L, targetHits = 1L, nThreads = 2L, verbosity = 0L) expect_s3_class(result, "multiPhylo") expect_true(is.finite(attr(result, "score"))) @@ -351,17 +351,17 @@ test_that("user tree is used as warm start", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md # Build a known tree set.seed(9847) - user_tree <- RandomTree(ds, root = TRUE) + user_tree <- RandomTree(vinther, root = TRUE) user_tree <- Preorder(user_tree) - result <- MaximizeParsimony(ds, tree = user_tree, + result <- MaximizeParsimony(vinther, tree = user_tree, maxReplicates = 1L, targetHits = 1L, verbosity = 0L) expect_s3_class(result, "multiPhylo") result_score <- attr(result, "score") # Result should be at least as good as the input tree - input_score <- TreeLength(user_tree, ds) + input_score <- TreeLength(user_tree, vinther) expect_true(result_score <= input_score) }) @@ -371,39 +371,39 @@ test_that("multiPhylo warm starts survive tip renumbering and polytomies", { # every start must be normalized (renumbered, resolved, rerooted) before it # reaches the engine, not just the first. set.seed(1002) - shuffled <- RandomTree(ds, root = TRUE) + shuffled <- RandomTree(vinther, root = TRUE) shuffled <- KeepTip(shuffled, rev(TipLabels(shuffled))) - polytomous <- CollapseNode(Preorder(RandomTree(ds, root = TRUE)), - NTip(ds) + 3L) + polytomous <- CollapseNode(Preorder(RandomTree(vinther, root = TRUE)), + NTip(vinther) + 3L) trees <- structure(list(shuffled, polytomous), class = "multiPhylo") set.seed(4004) - result <- MaximizeParsimony(ds, tree = trees, maxReplicates = 2L, + result <- MaximizeParsimony(vinther, tree = trees, maxReplicates = 2L, targetHits = 99L, verbosity = 0L) expect_s3_class(result, "multiPhylo") - expect_equal(attr(result, "score"), TreeLength(result[[1]], ds)) - expect_setequal(TipLabels(result[[1]]), names(ds)) + expect_equal(attr(result, "score"), TreeLength(result[[1]], vinther)) + expect_setequal(TipLabels(result[[1]]), names(vinther)) }) test_that("multiPhylo warm starts are validated", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(6003) - t1 <- Preorder(RandomTree(ds, root = TRUE)) - t2 <- Preorder(RandomTree(ds, root = TRUE)) + t1 <- Preorder(RandomTree(vinther, root = TRUE)) + t2 <- Preorder(RandomTree(vinther, root = TRUE)) t2[["tip.label"]][[1]] <- "not_a_taxon" expect_error( - MaximizeParsimony(ds, tree = structure(list(), class = "multiPhylo"), + MaximizeParsimony(vinther, tree = structure(list(), class = "multiPhylo"), maxReplicates = 1L, verbosity = 0L), "contains no trees" ) expect_error( - MaximizeParsimony(ds, tree = structure(list(t1, t2), class = "multiPhylo"), + MaximizeParsimony(vinther, tree = structure(list(t1, t2), class = "multiPhylo"), maxReplicates = 1L, verbosity = 0L), "same tip labels" ) expect_error( - MaximizeParsimony(ds, tree = structure(list(t1, "not a tree"), + MaximizeParsimony(vinther, tree = structure(list(t1, "not a tree"), class = "multiPhylo"), maxReplicates = 1L, verbosity = 0L), "class 'phylo'" @@ -415,18 +415,18 @@ test_that("multiPhylo warm starts are validated", { # error has to come first. The index tells the user which tree is at fault. broken <- ape::unroot(t1) expect_error( - MaximizeParsimony(ds, tree = broken, maxReplicates = 1L, verbosity = 0L), + MaximizeParsimony(vinther, tree = broken, maxReplicates = 1L, verbosity = 0L), "`tree` is not a valid tree" ) expect_error( - MaximizeParsimony(ds, tree = structure(list(t1, broken), + MaximizeParsimony(vinther, tree = structure(list(t1, broken), class = "multiPhylo"), maxReplicates = 1L, verbosity = 0L), "`tree\\[\\[2\\]\\]` is not a valid tree" ) # A genuinely unrooted tree is fine; only the corrupted object is refused. expect_silent( - MaximizeParsimony(ds, tree = RandomTree(ds, root = FALSE), + MaximizeParsimony(vinther, tree = RandomTree(vinther, root = FALSE), maxReplicates = 1L, targetHits = 1L, verbosity = 0L) ) @@ -438,16 +438,16 @@ test_that("multiPhylo warm starts are validated", { # legitimate degree-3 trifurcation as a polytomy, corrupting the tree and # previously surfacing downstream as "argument is of length zero". set.seed(9) - apeUnrooted <- ape::unroot(ape::rtree(NTip(ds), tip.label = names(ds))) + apeUnrooted <- ape::unroot(ape::rtree(NTip(vinther), tip.label = names(vinther))) expect_silent( - MaximizeParsimony(ds, tree = apeUnrooted, + MaximizeParsimony(vinther, tree = apeUnrooted, maxReplicates = 1L, targetHits = 1L, verbosity = 0L) ) # Unused pool members are reported against the replicates actually run, # which targetHits can cut short well below maxReplicates. expect_warning( - MaximizeParsimony(ds, tree = structure(list(t1, t1, t1), + MaximizeParsimony(vinther, tree = structure(list(t1, t1, t1), class = "multiPhylo"), maxReplicates = 9L, targetHits = 1L, verbosity = 0L), "of the 3 trees supplied" @@ -459,7 +459,7 @@ test_that("multiPhylo warm starts are validated", { test_that("timings attribute is returned", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(2689) - result <- MaximizeParsimony(ds, maxReplicates = 2L, targetHits = 1L, + result <- MaximizeParsimony(vinther, maxReplicates = 2L, targetHits = 1L, verbosity = 0L) timings <- attr(result, "timings") expect_false(is.null(timings)) @@ -474,12 +474,12 @@ test_that("timings attribute is returned", { test_that("IW mode works with effort rungs", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(4012) - result <- MaximizeParsimony(ds, concavity = 10, effort = -9L, + result <- MaximizeParsimony(vinther, concavity = 10, effort = -9L, maxReplicates = 2L, targetHits = 1L, verbosity = 0L) expect_s3_class(result, "multiPhylo") cpp_score <- attr(result, "score") - tl_score <- TreeLength(result[[1]], ds, concavity = 10) + tl_score <- TreeLength(result[[1]], vinther, concavity = 10) expect_equal(cpp_score, tl_score, tolerance = 0.01) }) @@ -491,11 +491,11 @@ test_that("concavity as a numeric-coercible string behaves like the number", { # min_steps (uncorrected homoplasy) -- it should match concavity = 10 # exactly, seed-for-seed. set.seed(4012) - asString <- MaximizeParsimony(ds, concavity = "10", effort = -9L, + asString <- MaximizeParsimony(vinther, concavity = "10", effort = -9L, maxReplicates = 2L, targetHits = 1L, verbosity = 0L) set.seed(4012) - asNumber <- MaximizeParsimony(ds, concavity = 10, effort = -9L, + asNumber <- MaximizeParsimony(vinther, concavity = 10, effort = -9L, maxReplicates = 2L, targetHits = 1L, verbosity = 0L) expect_equal(attr(asString, "score"), attr(asNumber, "score")) @@ -505,21 +505,27 @@ test_that("concavity as a numeric-coercible string behaves like the number", { test_that("concavity = 'Profile'/'prof' route to profile mode like 'profile'", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(4012) - canonical <- MaximizeParsimony(ds, concavity = "profile", effort = -9L, - maxReplicates = 2L, targetHits = 1L, - verbosity = 0L) + expect_message( + canonical <- MaximizeParsimony(vinther, concavity = "profile", effort = -9L, + maxReplicates = 2L, targetHits = 1L, + verbosity = 0L), + "Inapplicable .* treated as ambiguous" + ) for (spelling in c("Profile", "prof")) { set.seed(4012) - result <- MaximizeParsimony(ds, concavity = spelling, effort = -9L, - maxReplicates = 2L, targetHits = 1L, - verbosity = 0L) + expect_message( + result <- MaximizeParsimony(vinther, concavity = spelling, effort = -9L, + maxReplicates = 2L, targetHits = 1L, + verbosity = 0L), + "Inapplicable .* treated as ambiguous" + ) expect_equal(attr(result, "score"), attr(canonical, "score")) } }) test_that("an invalid concavity string errors cleanly instead of silently using EW", { expect_error( - MaximizeParsimony(ds, concavity = "banana", maxReplicates = 1L, + MaximizeParsimony(vinther, concavity = "banana", maxReplicates = 1L, targetHits = 1L, verbosity = 0L), "`concavity` must be a single positive number" ) @@ -530,14 +536,14 @@ test_that("an invalid concavity string errors cleanly instead of silently using test_that("output trees have valid preorder numbering", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(8734) - result <- MaximizeParsimony(ds, maxReplicates = 2L, targetHits = 1L, + result <- MaximizeParsimony(vinther, maxReplicates = 2L, targetHits = 1L, verbosity = 0L) for (tree in result) { # Trees should have correct number of tips and edges - expect_equal(NTip(tree), NTip(ds)) - expect_equal(nrow(tree$edge), 2 * (NTip(ds) - 1)) + expect_equal(NTip(tree), NTip(vinther)) + expect_equal(nrow(tree$edge), 2 * (NTip(vinther) - 1)) # Tip labels should match - expect_true(all(TipLabels(tree) %in% names(ds))) + expect_true(all(TipLabels(tree) %in% names(vinther))) } }) @@ -601,7 +607,7 @@ test_that("Constrained Wagner tree works with multiple seeds", { test_that("intraFuse runs without error", { skip_on_cran() # mixed-tier exception, see tests/testing-strategy.md set.seed(8517) - result <- MaximizeParsimony(ds, effort = -9L, + result <- MaximizeParsimony(vinther, effort = -9L, maxReplicates = 5L, targetHits = 2L, maxSeconds = 3, intraFuse = TRUE, verbosity = 0L, nThreads = 1L) @@ -694,14 +700,14 @@ test_that("collapse = TRUE is a no-op when no branch is unsupported", { # Vinther2008 MPTs are fully resolved (no zero-length branches): collapse must # leave every tree binary and topologically unchanged. set.seed(3418) - resolved <- MaximizeParsimony(ds, effort = -9L, + resolved <- MaximizeParsimony(vinther, effort = -9L, maxReplicates = 3L, targetHits = 1L, verbosity = 0L, collapse = FALSE) set.seed(3418) - collapsed <- MaximizeParsimony(ds, effort = -9L, + collapsed <- MaximizeParsimony(vinther, effort = -9L, maxReplicates = 3L, targetHits = 1L, verbosity = 0L, collapse = TRUE) - nTip <- NTip(ds) + nTip <- NTip(vinther) # No spurious collapse: every returned tree stays fully binary. expect_true(all(vapply(collapsed, function(t) t[["Nnode"]] == nTip - 1L, logical(1)))) diff --git a/tests/testthat/test-PolEscapa.R b/tests/testthat/test-PolEscapa.R index 480e59020..b99832334 100644 --- a/tests/testthat/test-PolEscapa.R +++ b/tests/testthat/test-PolEscapa.R @@ -347,7 +347,10 @@ test_that("LengthAdded(concavity = 'profile') handles an uninformative character char <- ProfileChar(c("0", "0", "0", "0", "1")) trees <- ProfileTrees(paste0("t", 1:5)) - expect_no_error(added <- LengthAdded(trees, char, concavity = "profile")) + expect_message(expect_message( + expect_no_error(added <- LengthAdded(trees, char, concavity = "profile")), + "No informative characters"), + "No informative characters") expect_named(added, names(char)) expect_equal(unname(added), rep(0, length(char))) }) diff --git a/tests/testthat/test-Ratchet.R b/tests/testthat/test-Ratchet.R index 738596d15..c3f833916 100644 --- a/tests/testthat/test-Ratchet.R +++ b/tests/testthat/test-Ratchet.R @@ -55,8 +55,10 @@ test_that("Ratchet(stopAtScore=) met mid-search returns a consistent tree", { }) test_that("MultiRatchet() survives an already-met stopAtScore", { - result <- MultiRatchet(startTree, preparedData, stopAtScore = startScore, - nSearch = 2, verbosity = 0) + expect_silent(result <- MultiRatchet(startTree, preparedData, + stopAtScore = startScore, + nSearch = 2, verbosity = 0) + ) expect_s3_class(result, "multiPhylo") for (phy in result) { expect_equal(TreeLength(phy, dataset), attr(phy, "score")) diff --git a/tests/testthat/test-SuboptimalTrees.R b/tests/testthat/test-SuboptimalTrees.R new file mode 100644 index 000000000..741e33b46 --- /dev/null +++ b/tests/testthat/test-SuboptimalTrees.R @@ -0,0 +1,92 @@ +library("TreeTools", quietly = TRUE) + +data("inapplicable.phyData", package = "TreeSearch") +ds <- inapplicable.phyData[["Vinther2008"]] + +# --- Scores surfaced only when collapse = FALSE --- + +test_that("MaximizeParsimony(collapse = TRUE) attaches no per-tree scores", { + set.seed(3418) + result <- MaximizeParsimony(ds, maxReplicates = 2L, targetHits = 1L, + verbosity = 0L) # collapse = TRUE by default + expect_null(attr(result, "scores")) + expect_true(is.finite(attr(result, "score"))) +}) + +test_that("MaximizeParsimony(collapse = FALSE) surfaces aligned pool scores", { + set.seed(3418) + result <- MaximizeParsimony(ds, maxReplicates = 3L, targetHits = 1L, + verbosity = 0L, collapse = FALSE, + control = SearchControl(poolSuboptimal = 5, + poolMaxSize = 500L)) + scores <- attr(result, "scores") + expect_type(scores, "double") + expect_length(scores, length(result)) + + # Each tree carries a matching `score` attribute (so Suboptimality() works). + perTree <- vapply(result, attr, double(1), "score") + expect_equal(perTree, scores) + + # Scores match an independent re-scoring of the trees. + expect_equal(scores, TreeLength(result, ds), tolerance = 1e-8) + + # The best of the pool equals the reported best score, and every retained + # tree is within the requested suboptimality window. + expect_equal(min(scores), attr(result, "score")) + expect_true(all(scores <= attr(result, "score") + 5 + 1e-8)) +}) + +# --- SuboptimalTrees() accessor --- + +test_that("SuboptimalTrees() returns a scored, bounded pool", { + set.seed(3418) + trees <- SuboptimalTrees(ds, maxSuboptimal = 3, maxPool = 200L, + maxReplicates = 3L, targetHits = 1L, + verbosity = 0L) + expect_s3_class(trees, "multiPhylo") + scores <- attr(trees, "scores") + expect_false(is.null(scores)) + expect_length(scores, length(trees)) + expect_true(length(trees) <= 200L) + expect_equal(min(scores), attr(trees, "score")) + expect_true(all(scores <= attr(trees, "score") + 3 + 1e-8)) + + # Suboptimality() reads the per-tree `score` attribute we attached. + subopt <- Suboptimality(trees) + expect_equal(subopt, scores - min(scores)) + expect_true(all(subopt >= 0)) +}) + +test_that("SuboptimalTrees() validates its arguments", { + expect_error(SuboptimalTrees(ds, maxSuboptimal = -1), + "must be a single non-negative number") + expect_error(SuboptimalTrees(ds, maxPool = 0L), + "must be a single positive integer") +}) + +test_that("SuboptimalTrees() keeps scores aligned under pool eviction (TA-5)", { + # A tiny maxPool forces the diversity-eviction path; the scores attribute -- and + # each tree's own score attribute -- must stay aligned with the evicted-down set. + set.seed(3418) + trees <- SuboptimalTrees(ds, maxSuboptimal = 8, maxPool = 5L, + maxReplicates = 4L, targetHits = 1L, verbosity = 0L) + expect_true(length(trees) <= 5L) + scores <- attr(trees, "scores") + expect_length(scores, length(trees)) + expect_equal(vapply(trees, attr, double(1), "score"), scores) # alignment held + expect_equal(scores, TreeLength(trees, ds), tolerance = 1e-8) # independent re-score +}) + +test_that("SuboptimalTrees() warns and ignores raw poolSuboptimal/poolMaxSize (B-10)", { + # These raw SearchControl fields would silently override the values set from + # maxSuboptimal/maxPool via MaximizeParsimony()'s dots-override-control merge. + set.seed(3418) + expect_warning( + SuboptimalTrees(ds, maxSuboptimal = 2, poolSuboptimal = 9, + maxReplicates = 2L, targetHits = 1L, verbosity = 0L), + "maxSuboptimal") + expect_warning( + SuboptimalTrees(ds, maxPool = 100L, poolMaxSize = 5L, + maxReplicates = 2L, targetHits = 1L, verbosity = 0L), + "maxPool") +}) diff --git a/tests/testthat/test-data_manipulation.R b/tests/testthat/test-data_manipulation.R index 4cceafcec..8044da740 100644 --- a/tests/testthat/test-data_manipulation.R +++ b/tests/testthat/test-data_manipulation.R @@ -89,7 +89,7 @@ test_that("PrepareDataProfile()", { data("Lobo", package = "TreeTools") - prep <- suppressMessages(PrepareDataProfile(Lobo.phy, n_mc = 1000L)) + prep <- suppressMessages(PrepareDataProfile(Lobo.phy, mcSamples = 1000L)) info_dims <- dim(attr(prep, "info.amounts")) expect_equal(info_dims[2], attr(prep, "nr")) expect_true(info_dims[1] >= 1) diff --git a/tests/testthat/test-pp-info_extra_step.R b/tests/testthat/test-pp-info_extra_step.R index c47ee0c94..81e8280e7 100644 --- a/tests/testthat/test-pp-info_extra_step.R +++ b/tests/testthat/test-pp-info_extra_step.R @@ -16,6 +16,11 @@ test_that("StepInformation() works", { ambiguousTokens = 3)))) expect_true(all(is.finite(StepInformation( char = rep.int(1:2, times = c(600, 600)))))) + expect_true(all(is.finite(StepInformation(rep.int(1:3, 2L), maxSeconds = 200)))) + expect_error(StepInformation(rep.int(1:3, 2L), maxSeconds = -1), + "non-negative") + expect_error(StepInformation(rep.int(1:3, 2L), maxSeconds = Inf), + "non-negative") }) test_that("Carter1() matches profile counts", { diff --git a/tests/testthat/test-pp-multistate.R b/tests/testthat/test-pp-multistate.R index 960b57c64..031858249 100644 --- a/tests/testthat/test-pp-multistate.R +++ b/tests/testthat/test-pp-multistate.R @@ -53,7 +53,7 @@ test_that("infeasible multi-state uses MC preserving all states", { # k=3 n=38 (13,13,12): sc=140 >> 75 char3 <- rep(c("a", "b", "c"), c(13, 13, 12)) - info3 <- StepInformation(char3, n_mc = 5000L) + info3 <- StepInformation(char3, mcSamples = 5000L) expect_true(length(info3) >= 1) # MC preserves 3 states: min steps = k - 1 = 2 @@ -63,14 +63,14 @@ test_that("infeasible multi-state uses MC preserving all states", { # k=4 n=24 (7,6,6,5): sc=224 >> 50 char4 <- rep(c("x", "y", "z", "w"), c(7, 6, 6, 5)) - info4 <- StepInformation(char4, n_mc = 5000L) + info4 <- StepInformation(char4, mcSamples = 5000L) expect_true(length(info4) >= 1) expect_equal(as.integer(names(info4)[1L]), 3L) expect_true(all(info4 >= 0)) # k=5 n=15 (4,3,3,3,2): sc=143 >> 35 char5 <- rep(c("0", "1", "2", "3", "4"), c(4, 3, 3, 3, 2)) - info5 <- StepInformation(char5, n_mc = 5000L) + info5 <- StepInformation(char5, mcSamples = 5000L) expect_true(length(info5) >= 1) expect_equal(as.integer(names(info5)[1L]), 4L) expect_true(all(info5 >= 0)) @@ -82,7 +82,7 @@ test_that("approx='mc' matches exact within 1 bit for feasible character", { set.seed(4412) info_exact <- StepInformation(char, approx = "exact") - info_mc <- StepInformation(char, approx = "mc", n_mc = 10000L) + info_mc <- StepInformation(char, approx = "mc", mcSamples = 10000L) common <- intersect(names(info_exact), names(info_mc)) expect_true(length(common) >= 1L) @@ -101,7 +101,7 @@ test_that("approx='mc' returns multi-state step range for infeasible char", { char <- rep(c("0", "1", "2"), c(13, 13, 12)) set.seed(7731) - info_mc <- StepInformation(char, approx = "mc", n_mc = 2000L) + info_mc <- StepInformation(char, approx = "mc", mcSamples = 2000L) expect_true(length(info_mc) >= 1L) # Min steps = k - 1 = 2 (not 1 as binary fallback would give) @@ -128,8 +128,8 @@ test_that("PrepareDataProfile preserves multi-state patterns", { # Step ranges may differ (exact has the true distribution; MC estimates # via sampling and log-quadratic tail interpolation), so we don't # compare dimensions — just check both produce valid, finite output. - info_auto <- PrepareDataProfile(dat, approx = "auto", n_mc = 5000L) - info_mc <- PrepareDataProfile(dat, approx = "mc", n_mc = 5000L) + info_auto <- PrepareDataProfile(dat, approx = "auto", mcSamples = 5000L) + info_mc <- PrepareDataProfile(dat, approx = "mc", mcSamples = 5000L) expect_true(all(is.finite(attr(info_auto, "info.amounts")))) expect_true(all(is.finite(attr(info_mc, "info.amounts")))) @@ -140,7 +140,7 @@ test_that("PrepareDataProfile preserves multi-state patterns", { test_that(">5 state characters handled via MC without truncation", { char <- rep(c("a", "b", "c", "d", "e", "f"), c(4, 3, 3, 2, 2, 2)) # No warning: >5 states now route to MC directly instead of truncating - info <- StepInformation(char, n_mc = 5000L) + info <- StepInformation(char, mcSamples = 5000L) expect_true(length(info) >= 1) expect_true(all(info >= 0)) @@ -203,7 +203,7 @@ test_that("multi-state info is always >= 0", { ) for (i in seq_along(test_chars)) { - info <- suppressWarnings(StepInformation(test_chars[[i]], n_mc = 5000L)) + info <- suppressWarnings(StepInformation(test_chars[[i]], mcSamples = 5000L)) expect_true(all(info >= 0), label = paste("test char", i)) expect_true(all(is.finite(info)), label = paste("finite char", i)) } @@ -232,7 +232,7 @@ test_that("MC approximation matches exact within 2 bits at boundary", { info_exact <- StepInformation(char, approx = "exact") set.seed(5072) - info_mc <- StepInformation(char, approx = "mc", n_mc = 50000L) + info_mc <- StepInformation(char, approx = "mc", mcSamples = 50000L) common <- intersect(names(info_exact), names(info_mc)) expect_true(length(common) >= 1L) @@ -252,7 +252,7 @@ test_that("log-quadratic interpolation produces monotone IC", { # decreasing IC (non-increasing). set.seed(2849) char <- rep(c("0", "1", "2"), c(13, 13, 12)) - info <- StepInformation(char, n_mc = 10000L) + info <- StepInformation(char, mcSamples = 10000L) expect_true(length(info) >= 1L) expect_true(all(info >= 0)) @@ -280,7 +280,7 @@ test_that("Monte Carlo step information is reproducible under set.seed()", { # The same property must hold one level up, through StepInformation()'s MC # path (infeasible 3-state character forces approx = "mc"). char <- rep(c("0", "1", "2"), c(13, 13, 12)) - set.seed(123L); s1 <- StepInformation(char, approx = "mc", n_mc = 20000L) - set.seed(123L); s2 <- StepInformation(char, approx = "mc", n_mc = 20000L) + set.seed(123L); s1 <- StepInformation(char, approx = "mc", mcSamples = 20000L) + set.seed(123L); s2 <- StepInformation(char, approx = "mc", mcSamples = 20000L) expect_identical(s1, s2) }) diff --git a/tests/testthat/test-tree_length.R b/tests/testthat/test-tree_length.R index cb3f2ecd3..b78b3942c 100644 --- a/tests/testthat/test-tree_length.R +++ b/tests/testthat/test-tree_length.R @@ -221,8 +221,10 @@ test_that("TreeLength() scores an uninformative character under profile parsimon tree <- PectinateTree(names(char)) trees <- RootTree(c(tree, tree), 1) - expect_equal(TreeLength(tree, char, "profile"), 0) - expect_equal(TreeLength(trees, char, "profile"), c(0, 0)) + expect_message(expect_equal(TreeLength(tree, char, "profile"), 0), + "No informative characters") + expect_message(expect_equal(TreeLength(trees, char, "profile"), c(0, 0)), + "No informative characters") }) test_that("CharacterLength() fails gracefully", { diff --git a/tests/testthat/test-ts-constraint-free-taxa.R b/tests/testthat/test-ts-constraint-free-taxa.R index 805eaf2e3..03a8fb4ab 100644 --- a/tests/testthat/test-ts-constraint-free-taxa.R +++ b/tests/testthat/test-ts-constraint-free-taxa.R @@ -153,7 +153,7 @@ test_that("MaximizeParsimony honours and searches under a `?` constraint", { test_that("the collapse pass keeps the enforced grouping visible", { # The one place the strict reading did return a wrong answer. A constraint # is external evidence for a grouping, so ts_collapse_pool() protects the - # branch that realises it from contraction — but it identified that branch by + # branch that realizes it from contraction — but it identified that branch by # matching a node's tip set to the `1` group EXACTLY. With free taxa the # realising node is generally not that set, so nothing was protected and the # separating edge was contracted away: the returned tree broke the documented diff --git a/tests/testthat/test-ts-constraint-rooting.R b/tests/testthat/test-ts-constraint-rooting.R index 66cc9ebed..b9bacd774 100644 --- a/tests/testthat/test-ts-constraint-rooting.R +++ b/tests/testthat/test-ts-constraint-rooting.R @@ -4,7 +4,7 @@ skip_on_cran() ## T-384: constraint mapping must not depend on the tree's rooting. ## ## A constraint split is an *unrooted* bipartition, so a tree displays it -## whenever EITHER side is a rooted clade. build_constraint() canonicalises +## whenever EITHER side is a rooted clade. build_constraint() canonicalizes ## every split mask to exclude tip 0, and map_constraint_nodes() used to accept ## only that side as a clade — true of a tip-0-rooted tree and of no other. In ## any other rooting the mapping returned -1, which regraft_violates_constraint() @@ -56,7 +56,7 @@ test_that("T-384: constrained TBR searches from every rooting of one tree", { ds <- make_ts_data(dataset) # One constraint split; which side is stored is immaterial, as - # build_constraint() canonicalises it to exclude tip 0. + # build_constraint() canonicalizes it to exclude tip 0. inGroup <- labels[1:6] splitMatrix <- matrix(as.integer(labels %in% inGroup), nrow = 1) diff --git a/tests/testthat/test-ts-degenerate-shapes.R b/tests/testthat/test-ts-degenerate-shapes.R index 70e2a6af6..a5b172718 100644 --- a/tests/testthat/test-ts-degenerate-shapes.R +++ b/tests/testthat/test-ts-degenerate-shapes.R @@ -204,12 +204,13 @@ test_that("the user-facing API survives zero Fitch words", { dataset <- zeroWordSets[[nm]] tree <- FixedTree(dataset, 1L) for (concavity in list(Inf, 10, "profile")) { - trees <- suppressWarnings( + trees <- suppressMessages(suppressWarnings( MaximizeParsimony(dataset, tree = tree, concavity = concavity, - maxReplicates = 2L, verbosity = 0L)) + maxReplicates = 2L, verbosity = 0L))) expect_s3_class(trees[[1]], "phylo") expect_setequal(trees[[1]]$tip.label, names(dataset)) - expect_true(is.numeric(TreeLength(tree, dataset, concavity = concavity))) + expect_true(is.numeric( + suppressMessages(TreeLength(tree, dataset, concavity = concavity)))) } # One length per character, not per distinct pattern. expect_length(CharacterLength(tree, dataset), diff --git a/tests/testthat/test-ts-na-ambig.R b/tests/testthat/test-ts-na-ambig.R index 1a03f8a0c..c94c341e2 100644 --- a/tests/testthat/test-ts-na-ambig.R +++ b/tests/testthat/test-ts-na-ambig.R @@ -76,7 +76,7 @@ test_that("applicability tie breaks toward applicable, not the cheaper -", { library(TreeTools) # (t1,(t2,(t3,(t4,t5)))); 1 2 1 {-,2} - : making t4 applicable vs inapplicable # is EQUALLY parsimonious on the applicability character (a genuine tie), so - # BGS takes the applicable resolution (maximise homology) -> t4 = 2 is counted + # BGS takes the applicable resolution (maximize homology) -> t4 = 2 is counted # -> length 2, even though resolving t4 = "-" would score 1. This is the # deliberate consequence of Fitch-optimising applicability first. tree <- read.tree(text = "(t1, (t2, (t3, (t4, t5))));") diff --git a/tests/testthat/test-ts-na-oracle.R b/tests/testthat/test-ts-na-oracle.R index b143dfd0d..3b7032e0d 100644 --- a/tests/testthat/test-ts-na-oracle.R +++ b/tests/testthat/test-ts-na-oracle.R @@ -22,7 +22,7 @@ skip_on_cran() # (inapplicable) or an applicable state, so their applicability is fixed: this # is the regime in which the criterion is tie-break-free. With partially # ambiguous {-,X} tips BGS additionally prefers the APPLICABLE resolution on a -# tie (maximise homology; see test-ts-na-ambig.R), so the engine may +# tie (maximize homology; see test-ts-na-ambig.R), so the engine may # legitimately exceed this minimum there -- hence the second test asserts only # the one-sided bound that holds in every regime. diff --git a/tests/testthat/test-ts-random-constrained-free.R b/tests/testthat/test-ts-random-constrained-free.R index 53750a487..573fa7757 100644 --- a/tests/testthat/test-ts-random-constrained-free.R +++ b/tests/testthat/test-ts-random-constrained-free.R @@ -33,7 +33,7 @@ rctSplits <- function(tree, tips) { ## Does some edge put all of `together` on one side and all of `apart` on the ## other? This is the documented contract, stated without reference to which -## group the machinery happens to canonicalise as "inside". +## group the machinery happens to canonicalize as "inside". ## ## Both groups must hold at least two taxa, and that is checked rather than ## assumed. A group of one is separated from everything by its own pendant @@ -125,7 +125,7 @@ test_that("random_constrained_tree samples every legal topology", { }) ## The user-facing route in: `?`-coded taxa in a constraint phyDat, through -## .PrepareConstraint(), with the group the machinery canonicalises as "inside" +## .PrepareConstraint(), with the group the machinery canonicalizes as "inside" ## chosen by tip 0's coding rather than by the test. test_that("`?` taxa start inside a constrained group as well as outside", { tips <- letters[1:8] @@ -243,7 +243,7 @@ test_that("a constraint with no free taxa still builds exact clades", { ## (n_tip - 1 internal nodes, none allocated twice, none left dangling) is worth ## pinning: an over-run would corrupt the tree rather than fail loudly. ## Two splits, and a free tip 0 -- the tip whose position build_constraint() -## canonicalises the split masks around. +## canonicalizes the split masks around. test_that("scattered free tips leave a well-formed tree", { tips <- paste0("t", 1:9) ds <- rctDataset(tips) diff --git a/tests/testthat/test-ts-ratchet-search.R b/tests/testthat/test-ts-ratchet-search.R index 95608433d..7d3b4c9d1 100644 --- a/tests/testthat/test-ts-ratchet-search.R +++ b/tests/testthat/test-ts-ratchet-search.R @@ -66,7 +66,7 @@ test_that("Ratchet does not worsen score vs starting tree", { tree <- as.phylo(1, 15) # The guaranteed invariant is that ratchet never exceeds the initial - # (unoptimised) starting score. Comparing against a separate ts_tbr() call + # (unoptimized) starting score. Comparing against a separate ts_tbr() call # is not valid: both TBR and ratchet's internal TBR use different R RNG # states and can converge to different local optima. initial_score <- ts_score(tree, ds) diff --git a/tests/testthat/test-ts-sector-scaling.R b/tests/testthat/test-ts-sector-scaling.R index ed7e815cd..4a2cccca5 100644 --- a/tests/testthat/test-ts-sector-scaling.R +++ b/tests/testthat/test-ts-sector-scaling.R @@ -42,7 +42,7 @@ test_that("scaling knobs are OFF by default (deterministic baseline)", { test_that("TS_SECT_GROW=0 kill-switch is identical to default-off", { set.seed(1); base <- ts_rss(scal_tree, scal_ds) - # Growth knobs are supplied but the kill-switch must neutralise them entirely. + # Growth knobs are supplied but the kill-switch must neutralize them entirely. set.seed(1) off <- withr::with_envvar(c(TS_SECT_GROW = "0", TS_SECT_GROW_INC = "50", TS_SECT_GROW_SELFACT = "40", TS_SECT_GROW_START = "6"), diff --git a/tests/testthat/test-ts-xform.R b/tests/testthat/test-ts-xform.R index f5606918d..5e07ed948 100644 --- a/tests/testthat/test-ts-xform.R +++ b/tests/testthat/test-ts-xform.R @@ -468,8 +468,8 @@ test_that("TreeLength xform is rooting-invariant (T-385)", { }, numeric(1)) expect_equal(diff(range(lengths)), 0) - # The multiPhylo method must canonicalise identically to the single-tree one: - # it previously rooted only trees that arrived unrooted. + # The multiPhylo method must canonicalize identically to the single-tree one: + # rooting trees whether they arrive rooted or unrooted. multi <- TreeLength( structure(lapply(taxa, function(taxon) RootTree(tree, taxon)), class = "multiPhylo"), diff --git a/tests/testthat/test-ts-xpiwe.R b/tests/testthat/test-ts-xpiwe.R index f9a5de367..e2041efa5 100644 --- a/tests/testthat/test-ts-xpiwe.R +++ b/tests/testthat/test-ts-xpiwe.R @@ -14,7 +14,7 @@ ts_score <- function(...) TreeSearch:::ts_fitch_score(...) # 8 taxa, 6 characters. # Chars 1-3: fully observed; Chars 4-6: taxa G,H are "?". # Tree chosen so chars 4-6 have homoplasy > 1 (needed for XPIWE to differ, -# since Φ-rescaling normalises the first step identically). +# since Φ-rescaling normalizes the first step identically). make_missing_data <- function() { mat <- matrix(c( # C1 C2 C3 C4 C5 C6 diff --git a/vignettes/inapplicable.Rmd b/vignettes/inapplicable.Rmd index 403c95230..8ab8cd28c 100644 --- a/vignettes/inapplicable.Rmd +++ b/vignettes/inapplicable.Rmd @@ -157,7 +157,7 @@ xform_trees <- MaximizeParsimony(dataset, hierarchy = hierarchy, | Criterion | Brazeau | HSJ | X-transformation | |-----------|---------|-----|------------------| | Hierarchy required | No | Yes | Yes | -| Secondary char. variation | Not modelled | Penalised (via \U{03B1}) | Penalised (via Hamming) | +| Secondary char. variation | Not modelled | Penalized (via \U{03B1}) | Penalized (via Hamming) | | Gain/loss asymmetry | No | No | Yes (n+1 : 1) | | Rooting-invariant score | Yes | Yes | No | | Tuning parameter | None | `hsj_alpha` | None | diff --git a/vignettes/profile-scores.Rmd b/vignettes/profile-scores.Rmd index e8356af4f..177ef3c81 100644 --- a/vignettes/profile-scores.Rmd +++ b/vignettes/profile-scores.Rmd @@ -138,7 +138,7 @@ definition of the nature of the quantity that is being approximated. ### An information theoretic basis -I argue that the quantity that we should seek to minimise is the "surprise" of +I argue that the quantity that we should seek to minimize is the "surprise" of a tree. Information theory is the science of quantifying unexpectedness. Information is usually measured in _bits_. One bit is the amount of information generated by tossing a fair coin: to record the outcome of a coin toss, I must diff --git a/vignettes/profile.Rmd b/vignettes/profile.Rmd index 4921631eb..c4bea0dc1 100644 --- a/vignettes/profile.Rmd +++ b/vignettes/profile.Rmd @@ -92,7 +92,7 @@ Let's see the resultant tree, and its score: ```{r ratchet-search-results} TreeLength(betterTrees[[1]], myMatrix, "profile") par(mar = rep(0.25, 4), cex = 0.75) # make plot easier to read -plot(ape::consensus(betterTrees)) +plot(TreeTools::Consensus(betterTrees)) ``` Type `?MaximizeParsimony` to view all search parameters, including effort @@ -120,7 +120,7 @@ typically more reliable, summary of the signal with the phylogenetic dataset ```{r plot-suboptimal-consensus} par(mar = rep(0.25, 4), cex = 0.75) table(signif(TreeLength(suboptimals, myMatrix, "profile"))) -plot(ape::consensus(suboptimals)) +plot(TreeTools::Consensus(suboptimals)) ``` diff --git a/vignettes/search-algorithm.Rmd b/vignettes/search-algorithm.Rmd index b10751cdb..d5f47d10f 100644 --- a/vignettes/search-algorithm.Rmd +++ b/vignettes/search-algorithm.Rmd @@ -181,7 +181,7 @@ probability that a better score exists is bounded by $(1 - K/R)^R$. **Thompson sampling.** Each arm $i$ maintains a Beta-distributed estimate of its success probability, parameterized by success count $\alpha_i$ and failure count -$\beta_i$ (both initialised to 1, giving a uniform prior). +$\beta_i$ (both initialized to 1, giving a uniform prior). The _random tree_ arm receives a pessimistic prior $\text{Beta}(1, 2)$ reflecting the expectation that random topologies are usually worse. @@ -265,7 +265,7 @@ Where the repair does not succeed, the start is discarded in favour of a fresh constrained Wagner build. The same check gates each replicate's finished tree on its way into the pool, so a tree that breaks the constraint is never returned; and the enforced splits -are protected from the final collapse pass, whichever branch happens to realise +are protected from the final collapse pass, whichever branch happens to realize them. A start that already separates the two groups is left alone, however its free @@ -378,12 +378,12 @@ minimum, and in simulation 87--98% of rootings attain that minimum exactly, with the excess never exceeding the total number of secondary characters across hierarchy blocks. It is not, however, reproducible from the returned tree, whose rooting the -output stage has since canonicalised. +output stage has since canonicalized. `MaximizeParsimony()` therefore re-scores the returned trees at that same canonical rooting -- the first taxon of the dataset -- before reporting, and -`TreeLength()` canonicalises identically, so a reported x-transformation score +`TreeLength()` canonicalizes identically, so a reported x-transformation score is the length of the tree in hand, and one topology has one length. -This affects reporting only: the quantity the search optimises is unchanged. +This affects reporting only: the quantity the search optimizes is unchanged. Because pool membership is still decided on scores taken at differing rootings, the returned trees need not all share that canonical length; diff --git a/vignettes/tree-search.Rmd b/vignettes/tree-search.Rmd index 1cd83b07c..c894377c2 100644 --- a/vignettes/tree-search.Rmd +++ b/vignettes/tree-search.Rmd @@ -106,7 +106,7 @@ We can plot the best tree(s) that we've found, and check its parsimony score ```{r plot-tree} par(mar = rep(0.25, 4), cex = 0.75) # make plot easier to read -plot(ape::consensus(bestTrees)) +plot(TreeTools::Consensus(bestTrees)) TreeLength(bestTrees[[1]], vinther) ``` @@ -117,7 +117,7 @@ sampled most-parsimonious trees: ```{r plot-label-nodes} par(mar = rep(0.25, 4), cex = 0.75) # make plot easier to read -majCons <- ape::consensus(bestTrees, p = 0.5) +majCons <- TreeTools::Consensus(bestTrees, p = 0.5) splitFreqs <- TreeTools::SplitFrequency(majCons, bestTrees) / length(bestTrees) plot(majCons) TreeTools::LabelSplits(majCons, round(splitFreqs * 100), unit = "%", @@ -158,7 +158,7 @@ parallel threads: nReplicates <- 10 jackTrees <- Resample(vinther, bestTrees, nReplicates = nReplicates) -strict <- ape::consensus(bestTrees, p = 1) +strict <- TreeTools::Consensus(bestTrees, p = 1) par(mar = rep(0, 4), cex = 0.8) # Take the strict consensus of all trees for each replicate @@ -231,7 +231,7 @@ The most informative single summary tree is thus provided by: if (rogueInstalled) { par(mar = rep(0, 4), cex = 0.8) noWiwaxia <- lapply(bestTrees, TreeTools::DropTip, "Wiwaxia") - plot(ape::consensus(noWiwaxia), + plot(TreeTools::Consensus(noWiwaxia), tip.color = Rogue::ColByStability(noWiwaxia)) } else { message("Install the 'Rogue' package to render this block.") @@ -373,7 +373,7 @@ constant, _k_: ```{r iw-search, message = FALSE} iwTrees <- MaximizeParsimony(vinther, concavity = 10) par(mar = rep(0.25, 4), cex = 0.75) # make plot easier to read -plot(ape::consensus(iwTrees)) +plot(TreeTools::Consensus(iwTrees)) ``` Note that we recommend a default value of 10, somewhat higher than the default diff --git a/vignettes/tree-space.Rmd b/vignettes/tree-space.Rmd index 5c63e8dab..429ed43fa 100644 --- a/vignettes/tree-space.Rmd +++ b/vignettes/tree-space.Rmd @@ -267,7 +267,7 @@ nClusters <- length(unique(clustering)) ## Mapping tree space -Now we can map tree space into a few dimensions and visualise the distribution +Now we can map tree space into a few dimensions and visualize the distribution of trees: ```{r map-space, fig.keep = "none"}