diff --git a/.github/workflows/SpellCheck.yml b/.github/workflows/SpellCheck.yml new file mode 100644 index 0000000..c344f6d --- /dev/null +++ b/.github/workflows/SpellCheck.yml @@ -0,0 +1,13 @@ +name: Spell Check + +on: [pull_request] + +jobs: + typos-check: + name: Spell Check with Typos + runs-on: ubuntu-latest + steps: + - name: Checkout Actions Repository + uses: actions/checkout@v4 + - name: Check spelling + uses: crate-ci/typos@master diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a33d5bd..e0d4365 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,19 +17,19 @@ jobs: fail-fast: false matrix: version: - - '1.0' + - 'lts' - '1' os: - ubuntu-latest arch: - x64 steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 + - uses: actions/checkout@v7 + - uses: julia-actions/setup-julia@v3 with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: julia-actions/cache@v1 + - uses: julia-actions/cache@v2 env: cache-name: cache-artifacts with: @@ -42,6 +42,22 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v7 with: - file: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + verbose: true + docs: + name: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: julia-actions/setup-julia@v3 + with: + version: '1' + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-docdeploy@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} + diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 0000000..c62bedf --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,9 @@ +coverage: + status: + project: + default: + threshold: 0.5% + removed_code_behavior: fully_covered_patch + patch: + default: + target: 80 diff --git a/Project.toml b/Project.toml index 406f0d7..6eced04 100644 --- a/Project.toml +++ b/Project.toml @@ -1,20 +1,33 @@ name = "IterationControl" uuid = "b3c1a2ee-3fec-4384-bf48-272ea71de57c" -version = "0.5.4" +version = "0.6.0" authors = ["Anthony D. Blaom "] +[workspace] +projects = ["docs", "NotebookManagementTools"] + [deps] EarlyStopping = "792122b4-ca99-40de-a6bc-6742525f08b6" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +[weakdeps] +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[extensions] +TablesExt = "Tables" + [compat] -EarlyStopping = "0.3" -REPL = "1.11.0" -julia = "1" +Dates = "1" +EarlyStopping = "1" +REPL = "1" +Tables = "1.6" +julia = "1.10" [extras] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [targets] -test = ["Test"] +test = ["Dates", "Tables", "Test"] diff --git a/README.md b/README.md index d833640..85f6fa7 100644 --- a/README.md +++ b/README.md @@ -15,272 +15,4 @@ inspired by Other related software: [DynamicIterators.jl](https://github.com/mschauer/DynamicIterators.jl). -## Installation - -```julia -using Pkg -Pkg.add("IterationControl") -``` - -## Basic idea - -Suppose you have [some kind of object](/examples/square_rooter/), -`SquareRooter(x)`, for iteratively computing approximations to the -square root of `x`: - -```julia -model = SquareRooter(9) - -julia> model.root -1.0 - -train!(model, 2) # train for 2 iterations - -julia> model.root -3.4 - -train!(model, 1) # train for 1 more iteration - -julia> model.root -3.023529411764706 -``` - -Then we can replace the integer argument `n` in `train!(model, n)` -with a number of more sophisticated *controls* by "lifting" the method -`train!` to the `IterationControl.train!` method defined in this -package: - -```julia -using IterationControl -IterationControl.train!(model::SquareRooter, n) = train!(model, n) # lifting -``` -By definition, the lifted `train!` has the same functionality as the original one: - -```julia -model = SquareRooter(9) -IterationControl.train!(model, 2) - -julia> model.root -3.4 -``` -But now we can also do this: - -```julia -julia> IterationControl.train!(model, Step(2), NumberLimit(3), Info(m->m.root)); -[ Info: 3.4 -[ Info: 3.00009155413138 -[ Info: 3.0 -[ Info: Stop triggered by NumberLimit(3) stopping criterion. -``` - -Here each control is repeatedly applied in sequence until one of them -triggers a stop. The first control `Step(2)` says, "Train the model -two more iterations"; the second asks, "Have I been applied 3 times -yet?", signaling a stop (at the end of the current control cycle) if -so; and the third logs the value of the function `m -> m.root`, -evaluated on `model`, to `Info`. In this example only the second -control can terminate model iteration. - -If `model` admits a method returning a loss (in this case the -difference between `x` and the square of `root`) then we can lift -that method to `IterationControl.loss` to enable control using -loss-based stopping criteria, such as a loss threshold. In the -demonstration below, we also include a callback: - -```julia -model = SquareRooter(4) -train!(model, 1) - -julia> loss(model) -2.25 - -IterationControl.loss(model::SquareRooter) = loss(model) # lifting - -losses = Float64[] -callback(model) = push!(losses, loss(model)) - -julia> IterationControl.train!(model, Step(1), Threshold(0.0001), Callback(callback)); -[ Info: Stop triggered by Threshold(0.0001) stopping criterion. - -julia> losses -2-element Array{Float64,1}: - 0.002439396192741583 - 3.716891878724482e-7 -``` - -In many applications to machine learning, "loss" will be an -out-of-sample loss, computed after some iterations. If `model` -additionally generates user-inspectable "training losses" (one per -iteration) then similarly lifting the appropriate access function to -`IterationControl.training_losses` enables Prechelt's -progress-modified generalization loss stopping criterion, `PQ` (see -Table 1 below). - -`PQ` is the only criterion from the -[EarlyStopping.jl](https://github.com/ablaom/EarlyStopping.jl) package -not otherwise enabled when `IterationControl.loss` is overloaded as -above. - -*Reference.* [Prechelt, Lutz - (1998)](https://link.springer.com/chapter/10.1007%2F3-540-49430-8_3): - "Early Stopping - But When?", in *Neural Networks: Tricks of the - Trade*, ed. G. Orr, Springer. - -The interface just described is sufficient for controlling -conventional machine learning models with an iteration parameter, as -[this](/examples/tree_booster/) tree boosting example shows. - - -## Online and incremental training - -For online or incremental training, lift the method for ingesting data -into the model to `IterationControl.ingest!(model, datum)` and use the -control `Data(data)`. Here `data` is any iterator generating the -`datum` items to be ingested (one per application of the control). By -default, the `Data` control becomes passive after `data` is -exhausted. Do `?Data` for details. (See [Access to model through a -wrapper](#access-to-model-through-a-wrapper) below on dealing with any -model wrapping necessary to implement data ingestion.) - -A simple particle tracking example is given -[here](/examples/particle/). - - -## Verbose logging and inspecting control reports - -The `IterationControl.train!` method can be given the keyword argument -`verbosity=...`, defaulting to `1`. The larger `verbosity`, the noisier. - -The return value of `IterationControl.train!` is a tuple of `(control, -report)` tuples, where `report` is generated by `control` at the end -of training. For example, the final loss can be accessed from the -report of the `WithLossDo()` control: - -```julia -model = SquareRooter(9) -reports = IterationControl.train!(model, Step(1), WithLossDo(println), NumberLimit(3)); - -julia> last(reports[2]) -(loss = 0.1417301038062284, done = false, log = "") - -julia> last(reports[2]).loss - 0.1417301038062284 -``` - - -## Controls provided - -Controls are repeatedly applied in sequence until a control triggers a -stop. Each control type has a detailed doc-string. Below is a short -summary, with some advanced options omitted. - -control | description | enabled if these are overloaded | can trigger a stop | notation in Prechelt -------------------------|-----------------------------------------------------------------------------------------|-----------------------------------|-------|--------------- -`Step(n=1)` | Train model for `n` iterations |`train!` | no | -`Info(f=identity)` | Log to `Info` the value of `f(model)` |`train!` | no | -`Warn(predicate, f="")` | Log to `Warn` the value of `f` or `f(model)` if `predicate(model)` holds |`train!` | no | -`Error(predicate, f="")`| Log to `Error` the value of `f` or `f(model)` if `predicate(model)` holds and then stop |`train!` | yes | -`Callback(f=_->nothing)`| Call `f(model)` |`train!` | yes | -`TimeLimit(t=0.5)` | Stop after `t` hours |`train!` | yes | -`NumberLimit(n=100)` | Stop after `n` applications of the control |`train!` | yes | -`NumberSinceBest(n=6)` | Stop when best loss occurred `n` control applications ago |`train!` | yes | -`WithNumberDo(f=n->@info(n))` | Call `f(n + 1)` where `n` is the number of complete control cycles so far |`train!` | yes | -`WithLossDo(f=x->@info("loss: $x"))` | Call `f(loss)` where `loss` is the current loss |`train!`, `loss` | yes | -`WithTrainingLossesDo(f=v->@info(v))`| Call `f(v)` where `v` is the current batch of training losses |`train!`, `training_losses` | yes | -`InvalidValue()` | Stop when `NaN`, `Inf` or `-Inf` loss/training loss encountered |`train!` | yes | -`Threshold(value=0.0)` | Stop when `loss < value` |`train!`, `loss` | yes | -`GL(alpha=2.0)` | Stop after "Generalization Loss" exceeds `alpha` |`train!`, `loss` | yes | ``GL_α`` -`Patience(n=5)` | Stop after `n` consecutive loss increases |`train!`, `loss` | yes | ``UP_s`` -`PQ(alpha=0.75, k=5)` | Stop after "Progress-modified GL" exceeds `alpha` |`train!`, `loss`, `training_losses`| yes | ``PQ_α`` -`Warmup(c; n=1)` | Wait for `n` loss updates before checking criteria `c` |`train!` | no | -`Data(data)` | Call `ingest!(model, item)` on the next `item` in the iterable `data`. |`train!`, `ingest!` | yes | - -> Table 1. Atomic controls - -**Stopping option.** All the following controls trigger a stop if the -provided function `f` returns `true` and `stop_if_true=true` is -specified in the constructor: `Callback`, `WithNumberDo`, `WithLossDo`, -`WithTrainingLossesDo`. - -There are also three control wrappers to modify a control's behavior: - -wrapper | description ----------------------------------------------------|------------------------------------------------------------------------- -`IterationControl.skip(control; predicate=1)` | Apply `control` every `predicate` applications of the control wrapper (can also be a function; see doc-string) -`IterationControl.louder(control; by=1)` | Increase the verbosity level of `control` by the specified value (negative values lower verbosity) -`IterationControl.with_state_do(control; f=...)` | Apply control *and* call `f(x)` where `x` is the internal state of control; useful for debugging. Default `f` logs state to `Info`. **Warning**: internal control state is not yet part of public API. -`IterationControl.composite(controls...)` | Apply each `control` in `controls` in sequence; mostly for under-the-hood use - -> Table 2. Wrapped controls - - -## Access to model through a wrapper - -Note that functions ordinarily applied to `model` by some control -(e.g., a `Callback`) will instead be applied to -`IterationControl.expose(model)` if `IterationControl.expose` is -appropriately overloaded. - - -## Implementing new controls - -There is no abstract control type; any object can be a -control. Behavior is implemented using a functional style interface -with six methods. Only the first two are compulsory (the fallbacks for -`done`, `takedown`, `needs_loss` and `needs_training_losses` always -return `false` and `NamedTuple()` respectively.): - -```julia -update!(control, model, verbosity, n) -> state # initialization -update!(control, model, verbosity, n, state) -> state -done(control, state)::Bool -takedown(control, verbosity, state) -> human_readable_named_tuple -``` - -Here `n` is the control cycle count, i.e., one more than the the -number of completed control cycles. - -If it is nonsensical to apply `control` to any model for which -`loss(model)` has not been overloaded, and we want an error thrown -when this is attempted, then declare `needs_loss(control::MyControl) = -true` to take value true. Otherwise `control` is applied anyway, and -`loss`, if called, returns `nothing`. - -A second trait `needs_training_losses(control)` serves an analogous -purpose for training losses. - -Here's a simplified version of how `IterationControl.train!` calls -these methods: - -```julia -function train!(model, controls...; verbosity::Int=1) - - control = composite(controls...) - - # before training: - verbosity > 1 && @info "Using these controls: $(flat(control)). " - - # first training event: - n = 1 # counts control cycles - state = update!(control, model, verbosity, n) - finished = done(control, state) - - # checks that model supports control: - if needs_loss(control) && loss(model) === nothing - throw(ERR_NEEDS_LOSS) - end - if needs_training_losses(control) && training_losses(model) === nothing - throw(ERR_NEEDS_TRAINING_LOSSES) - end - - # subsequent training events: - while !finished - n += 1 - state = update!(control, model, verbosity, n, state) - finished = done(control, state) - end - - # finalization: - return takedown(control, verbosity, state) -end -``` +Documentation is [here](https://juliaai.github.io/IterationControl.jl/dev/). diff --git a/docs/Project.toml b/docs/Project.toml new file mode 100644 index 0000000..cf68713 --- /dev/null +++ b/docs/Project.toml @@ -0,0 +1,18 @@ +[workspace] +projects = ["src/notebooks/particle","src/notebooks/anteaters", "src/notebooks/tree_booster"] + +[deps] +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +DocumenterCodeBlocks = "b6400b83-267f-4b55-9fd8-bffc853bdfd8" +EarlyStopping = "792122b4-ca99-40de-a6bc-6742525f08b6" +IterationControl = "b3c1a2ee-3fec-4384-bf48-272ea71de57c" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" + +[compat] +DataFrames = "1" +Documenter = "1" +DocumenterCodeBlocks = "1.5" +EarlyStopping = "1" +Tables = "1.6" +julia = "1.10" diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..81fcb32 --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,34 @@ +using Documenter +using DocumenterCodeBlocks +using IterationControl +import EarlyStopping + +const REPO = Remotes.GitHub("JuliaAI", "IterationControl.jl") + +makedocs( + modules=[IterationControl, EarlyStopping], + format=Documenter.HTML( + prettyurls = true, + collapselevel = 1, + ), + plugins = [CodeBlocks(),], + pages=[ + "Overview" => "index.md", + "Controls" => "controls.md", + "Adding New Controls" => "adding_new_controls.md", + "Demos" => [ + "Particle Tracker" => "notebooks/particle/notebook.md", + "Tree Booster" => "notebooks/tree_booster/notebook.md", + "Anteaters" => "notebooks/anteaters/notebook.md", + ], + ], + sitename="IterationControl.jl", + warnonly = [:cross_references, :missing_docs], + repo = Remotes.GitHub("JuliaAI", "IterationControl.jl"), +) + +deploydocs( + devbranch="dev", + push_preview=false, + repo="github.com/JuliaAI/IterationControl.jl.git", +) diff --git a/docs/src/adding_new_controls.md b/docs/src/adding_new_controls.md new file mode 100644 index 0000000..45fdf98 --- /dev/null +++ b/docs/src/adding_new_controls.md @@ -0,0 +1,65 @@ +# Adding New Controls + +## Implementing new controls + +There is no abstract control type; any object can be a control. Behavior is implemented +using a functional style interface documented below. Only the first two methods below are +compulsory. + +Main methods: + +- [`IterationControl.update!(control, model, verbosity, n)`](@ref)` -> state` # initialization +- [`IterationControl.update!(control, model, verbosity, n, state)`](@ref)` -> state` +- [`EarlyStopping.public_state(control, state)`](@ref) +- [`IterationControl.done(control, state)`](@ref)`::Bool` +- [`IterationControl.takedown(control, verbosity, state)`](@ref)` -> message` + +Here `n` is the control cycle count, i.e., one more than the the +number of completed control cycles. + +Control traits (fallback to `false`): + +- [`IterationControl.needs_loss(control)`](@ref) +- [`IterationControl.needs_training_losses(control)`](@ref) +- [`IterationControl.needs_ingest(control)`](@ref) + + +Simplified version of how `IterationControl.train!` calls these methods: + +```julia +function train!(model, controls...; verbosity::Int=1) + + control = composite(controls...) + + # first training event: + n = 1 # counts control cycles + state = update!(control, model, verbosity, n) + finished = done(control, state) + + + + + # subsequent training events: + while !finished + n += 1 + state = update!(control, model, verbosity, n, state) + finished = done(control, state) + + end + + + + + +end +``` + +```@docs +IterationControl.update! +EarlyStopping.public_state +IterationControl.done +IterationControl.takedown +IterationControl.needs_loss +IterationControl.needs_training_losses +IterationControl.needs_ingest +``` diff --git a/docs/src/controls.md b/docs/src/controls.md new file mode 100644 index 0000000..3b49eda --- /dev/null +++ b/docs/src/controls.md @@ -0,0 +1,70 @@ +# Controls + +Controls are repeatedly applied in sequence until a control triggers a +stop. Each control type has a detailed doc-string. Below is a short +summary, with some advanced options omitted. + +| control | description | enabled if these are overloaded | can trigger a stop | notation in Prechelt | +|-----------------------------------------------|-----------------------------------------------------------------------------------------|-------------------------------------|--------------------|----------------------| +| [`Step(n=1)`](@ref) | Train model for `n` iterations | `train!` | no | | +| [`Info(f=identity)`](@ref) | Log to `Info` the value of `f(model)` | `train!` | no | | +| [`Warn(predicate, f="")`](@ref) | Log to `Warn` the value of `f` or `f(model)` if `predicate(model)` holds | `train!` | no | | +| [`Error(predicate, f="")`](@ref) | Log to `Error` the value of `f` or `f(model)` if `predicate(model)` holds and then stop | `train!` | yes | | +| [`Callback(f=identity)`](@ref) | Call `f(model)` | `train!` | yes | | +| [`TimeLimit(t=0.5)`](@ref) | Stop after `t` hours | `train!` | yes | | +| [`NumberLimit(n=100)`](@ref) | Stop after `n` applications of the control | `train!` | yes | | +| [`NumberSinceBest(n=6)`](@ref) | Stop when best loss occurred `n` control applications ago | `train!` | yes | | +| [`WithNumberDo(f=identity)`](@ref) | Call `f(n + 1)` where `n` is the number of complete control cycles so far | `train!` | yes | | +| [`WithLossDo(f=x->@info("loss: $x"))`](@ref) | Call `f(loss)` where `loss` is the current loss | `train!`, `loss` | yes | | +| [`WithTrainingLossesDo(f=v->@info(v))`](@ref) | Call `f(v)` where `v` is the current batch of training losses | `train!`, `training_losses` | yes | | +| [`InvalidValue()`](@ref) | Stop when `NaN`, `Inf` or `-Inf` loss/training loss encountered | `train!` | yes | | +| [`Threshold(value=0.0)`](@ref) | Stop when `loss < value` | `train!`, `loss` | yes | | +| [`GL(alpha=2.0)`](@ref) | Stop after "Generalization Loss" exceeds `alpha` | `train!`, `loss` | yes | ``GL_α`` | +| [`Patience(n=5)`](@ref) | Stop after `n` consecutive loss increases | `train!`, `loss` | yes | ``UP_s`` | +| [`PQ(alpha=0.75, k=5)`](@ref) | Stop after "Progress-modified GL" exceeds `alpha` | `train!`, `loss`, `training_losses` | yes | ``PQ_α`` | +| [`Warmup(c; n=1)`](@ref) | Wait for `n` loss updates before checking criteria `c` | `train!` | no | | +| [`Data(data)`](@ref) | Call `ingest!(model, datum)` on the next `item` in the iterable `data`. | `train!`, `ingest!` | yes | | +| | | | | | + +> Table 1. Atomic controls + +**Stopping option.** All the following controls trigger a stop if the +provided function `f` returns `true` and `stop_if_true=true` is +specified in the constructor: `Callback`, `WithNumberDo`, `WithLossDo`, +`WithTrainingLossesDo`. + +There are also three control wrappers to modify a control's behavior: + +| wrapper | description | +|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| +| [`IterationControl.skip(control; predicate=1)`](@ref) | Apply `control` every `predicate` applications of the control wrapper (can also be a function; see doc-string) | +| [`IterationControl.louder(control; by=1)`](@ref) | Increase the verbosity level of `control` by the specified value (negative values lower verbosity) | +| [`IterationControl.with_state_do`](@ref)`(control; f=...)` | Apply control *and* call `f(x)` where `x` is the internal (public) state of control; useful for debugging. Default `f` logs state to `Info`. | +| [`IterationControl.composite(controls...)`](@ref) | Apply each `control` in `controls` in sequence; mostly for internal use | + +> Table 2. Wrapped controls + +```@docs +IterationControl.Step +IterationControl.Info +IterationControl.Warn +IterationControl.Error +IterationControl.Callback +EarlyStopping.TimeLimit +EarlyStopping.NumberLimit +EarlyStopping.NumberSinceBest +IterationControl.WithNumberDo +IterationControl.WithLossDo +IterationControl.WithTrainingLossesDo +EarlyStopping.InvalidValue +EarlyStopping.Threshold +EarlyStopping.GL +EarlyStopping.Patience +EarlyStopping.PQ +EarlyStopping.Warmup +IterationControl.Data +IterationControl.skip +IterationControl.louder +IterationControl.with_state_do +IterationControl.composite +``` diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 0000000..3dd57b9 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,221 @@ +# IterationControl.jl + +A lightweight package for controlling iterative algorithms, with a +view to training and optimizing machine learning models. + +Builds on +[EarlyStopping.jl](https://github.com/ablaom/EarlyStopping.jl) and +inspired by +[LearningStrategies.jl](https://github.com/JuliaML/LearningStrategies.jl). + +Other related software: +[DynamicIterators.jl](https://github.com/mschauer/DynamicIterators.jl). + + +## Installation + +```julia +using Pkg +Pkg.add("IterationControl") +Pkg.add("Tables") # optional +``` + +## Quick Start + +Suppose you have some kind of object `SquareRooter(x)`, for iteratively computing +approximations to the square root of `x`: + +```@setup quick +using IterationControl.SquareRooterExample +``` + +```@example quick +model = SquareRooter(9) +model.root +``` + +```@example quick +train!(model, 2) # train for 2 iterations +model.root +``` + +```@example quick +train!(model, 1) # train for 1 more iteration +model.root +``` + +Then you can replace the integer argument `n` in `train!(model, n)` +with a number of more sophisticated **controls** by "lifting" the method +`train!` to the `IterationControl.train!` method defined in this +package: + +```@example quick +using IterationControl +IterationControl.train!(model::SquareRooter, n::Int) = train!(model, n) # lifting +``` +By definition, the lifted `train!` has the same functionality as the original one: + +```@example quick +model = SquareRooter(9) +IterationControl.train!(model, 2) +model.root +``` + +But now we can also do this: + +```@example quick +model = SquareRooter(9) +IterationControl.train!(model, Step(2), NumberLimit(3), Info(m->m.root)); +nothing # hide +``` + +Here each control is repeatedly applied in sequence until one of them triggers a stop: + +- The first control `Step(2)` says, "Train the model two more iterations" +- The `NumberLimit(3)` control asks, "Have I been applied 3 times yet?", signaling a stop + (at the end of the current control cycle) if so +- The `Info` control logs the value of the function `m -> m.root`, evaluated on + `model`, to `Info`. + +In this example only the `NumberLimit(3)` control can terminate model iteration. + +!!! info + + To try out the example above, you can use a built-in version of `SquareRooter`. Before + executing the code above, run `using IterationControl.SquareRooterExample`. + +The [Controls](@ref) section documents all the controls provided by IterationControl.jl, +including those from EarlyStopping.jl. + + +## Loss-based stopping criteria + +If `model` admits a method returning a loss then we can lift that method to +`IterationControl.loss` to enable control using a loss-based stopping criteria, such as a +loss threshold. Let's demonstrate this for the built-in square rooting example, for which +`loss` returns the difference between the root approximation and the actual 64-bit +precision square root: + +```@example quick +using IterationControl +using IterationControl.SquareRooterExample +model = SquareRooter(4) +train!(model, 1) +loss(model) +``` + +```@example quick +IterationControl.loss(model::SquareRooter) = loss(model) # lifting +traces = IterationControl.train!( + model, + Step(1), + Threshold(0.0001), + Callback(m->m.root), + WithLossDo(), +); +nothing # hide +``` + +```@repl quick +traces +``` + +## Control traces + +In the demonstration above we used `Callback` instead of `Info` to suppress the logging of +the root value to `stdout`, and we added a special callback, `WithLossDo()`, which calls +the `loss` function on `model`. (More generally, `WithLossDo(f)` applies `f` to each +loss.) The return values of callbacks can be inspected from the **control traces** +returned by [`IterationControl.train!`](@ref): + +```@example quick +step, threshold, callback, withlossdo = traces; +nothing # hide +``` + +```@repl quick +callback +``` + +```@repl quick +withlossdo +``` + +```@repl quick +withlossdo.loss +``` + +Control traces have a property called `table`, returning a named tuple of vectors, but +they also implement the [Tables.jl](https://tables.juliadata.org/stable/) interface, +allowing conversion to common Julia table formats: + +```@repl quick +threshold.table +``` + +```@repl quick +using DataFrames +DataFrame(threshold) +``` + +It is possible to inspect traces "live" by running [`IterationControl.train!`](@ref) in an +asynchronous task. See the document string for details. + + +## Training losses + +In many applications to machine learning, "loss" will be an out-of-sample loss, computed +after some iterations. If `model` additionally generates user-inspectable "training +losses" (one per iteration) then similarly lifting the appropriate access function to +`IterationControl.training_losses` enables Prechelt's progress-modified generalization +loss stopping criterion, [`PQ`](@ref). + +`PQ` is the only criterion from the +[EarlyStopping.jl](https://github.com/ablaom/EarlyStopping.jl) package not otherwise +enabled when `IterationControl.loss` is overloaded as above. + +*Reference.* [Prechelt, Lutz + (1998)](https://link.springer.com/chapter/10.1007%2F3-540-49430-8_3): "Early Stopping - + But When?", in *Neural Networks: Tricks of the Trade*, ed. G. Orr, Springer. + +The interface just described is sufficient for controlling conventional machine learning +models with an iteration parameter (see [Early Stopping for a Tree Booster](@ref) for a +demo). + + +## Online and incremental training + +For online or incremental training, lift the method for ingesting data into the model to +[`IterationControl.ingest!(model, datum)`](@ref) and use the control `Data(data)`. Here +`data` is any iterator generating the `datum` items to be ingested (one per application of +the control). By default, the [`Data`](@ref) control becomes passive after `data` is +exhausted. See [Access to model through a wrapper](@ref) below on dealing with any model +wrapping necessary to implement data ingestion. + +For a demonstration, see [Controlling a Particle Tracker](@ref). + + +## Access to model through a wrapper + +Note that functions ordinarily applied to `model` by some control (e.g., a `Callback`) +will instead be applied to [`IterationControl.expose(model)`](@ref) if this function has +been appropriately overloaded. + + +## Reference + +- [`IterationControl.train!`](@ref) +- [`IterationControl.loss`](@ref) +- [`IterationControl.training_losses`](@ref) +- [`IterationControl.expose`](@ref) +- [`IterationControl.ControlTrace`](@ref) + + +```@docs +IterationControl.train! +IterationControl.loss +IterationControl.training_losses +IterationControl.ingest! +IterationControl.expose +IterationControl.ControlTrace +``` diff --git a/docs/src/notebooks/anteaters/Project.toml b/docs/src/notebooks/anteaters/Project.toml new file mode 100644 index 0000000..86b6ccb --- /dev/null +++ b/docs/src/notebooks/anteaters/Project.toml @@ -0,0 +1,8 @@ +[deps] +IterationControl = "b3c1a2ee-3fec-4384-bf48-272ea71de57c" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[compat] +Plots = "1" +Random = "1" diff --git a/docs/src/notebooks/anteaters/anteaters.svg b/docs/src/notebooks/anteaters/anteaters.svg new file mode 100644 index 0000000..4c73ae4 --- /dev/null +++ b/docs/src/notebooks/anteaters/anteaters.svg @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/notebooks/anteaters/notebook.jl b/docs/src/notebooks/anteaters/notebook.jl new file mode 100644 index 0000000..935902e --- /dev/null +++ b/docs/src/notebooks/anteaters/notebook.jl @@ -0,0 +1,106 @@ +# # Anteaters competing for food + +# > A demonstration of controlling multiple agents using IterationControl.jl + +# The source for this demonstration can be found +# [here](https://github.com/JuliaAI/IterationControl.jl/tree/dev/docs/src/notebooks/eaters/.). + +# Controlling competing iterative models in parallel is useful in optimization. For +# example, one might try several different optimization strategies in parallel, and divert +# resources to the strategy or strategies that are performing best. + +# In this toy illustration, several anteaters of varying health compete for food from a +# common supply - a stream of random "ant" or inedible "seed" items. An anteater's +# `health` attribute is the probability he is able to successfully catch an ant when the +# source makes it available. + +# If an anteater's score (the number of ants consumed) falls too far behind that of the +# leader, he stops consuming (his "training" stops early). An anteater also stops if he +# reaches the specified goal. + +# Anteaters forage for food in parallel using Julia multi-threading. + +using IterationControl +using .Threads +using Random +Random.seed!(123) +using Plots + +# ## Create a channel to supply food + +N = 1_000_000 +food = Channel{String}() do ch + foreach(i -> put!(ch, rand(["ant", "seed"])), 1:N) +end + +[take!(food) for i in 1:3] + +# ## Define anteaters and how to "train" them + +mutable struct Anteater + health::Float64 # 0.0 = dead; 1.0 = excellent health + n::Int # number of ants consumed + Anteater(health) = new(health, 0) +end + +function eat!(eater::Anteater) + take!(food) == "ant" && rand() <= eater.health && (eater.n += 1) + return eater +end; + +# "Training" for `n` iterations means take `n` items from the `food` +# source and try to catch and eat each item if it is an "ant": + +function IterationControl.train!(eater::Anteater, n::Int) + foreach(i->eat!(eater), 1:n) + return eater +end; + +# ## Create a collection of competitors + +eaters = [Anteater(0.7), Anteater(0.8), Anteater(0.9), Anteater(1.0)] + +#- + +n_eaters = length(eaters); + + +# ## Define the controls that will define the rules of the competition + +const LEAD_TRIGGERING_DEFEAT = 5 +const ANTS_TO_WIN = 100 + +scores(eaters) = [e.n for e in eaters] + +is_too_far_behind(eater) = + maximum(scores(eaters)) - eater.n > LEAD_TRIGGERING_DEFEAT + +has_won(eater) = eater.n >= ANTS_TO_WIN + +controls = [ + Step(1), + Callback(is_too_far_behind, stop_if_true=true), + Callback(has_won, stop_if_true=true), + Callback(eater->eater.n), + NumberLimit(1000), +]; + + +# ## Run the competition + +tasks = [Threads.@spawn IterationControl.train!(e, controls...) for e in eaters] +histories = map(fetch.(tasks)) do traces + traces[4].called_back +end + + +# ## Plot the results + +fig = plot() +for (j, e) in enumerate(eaters) + plot!(histories[j], label=repr(e), linewidth=2); +end +ylabel!(fig, "ants eaten"); +savefig(fig, "anteaters.svg"); + +# ![](anteaters.svg) diff --git a/docs/src/notebooks/anteaters/notebook.md b/docs/src/notebooks/anteaters/notebook.md new file mode 100644 index 0000000..eada6f0 --- /dev/null +++ b/docs/src/notebooks/anteaters/notebook.md @@ -0,0 +1,151 @@ +```@meta +EditURL = "notebook.jl" +``` + +# Anteaters competing for food + +> A demonstration of controlling multiple agents using IterationControl.jl + +The source for this demonstration can be found +[here](https://github.com/JuliaAI/IterationControl.jl/tree/dev/docs/src/notebooks/eaters/.). + +Controlling competing iterative models in parallel is useful in optimization. For +example, one might try several different optimization strategies in parallel, and divert +resources to the strategy or strategies that are performing best. + +In this toy illustration, several anteaters of varying health compete for food from a +common supply - a stream of random "ant" or inedible "seed" items. An anteater's +`health` attribute is the probability he is able to successfully catch an ant when the +source makes it available. + +If an anteater's score (the number of ants consumed) falls too far behind that of the +leader, he stops consuming (his "training" stops early). An anteater also stops if he +reaches the specified goal. + +Anteaters forage for food in parallel using Julia multi-threading. + +````@julia +using IterationControl +using .Threads +using Random +Random.seed!(123) +using Plots +```` + +## Create a channel to supply food + +````@julia +N = 1_000_000 +food = Channel{String}() do ch + foreach(i -> put!(ch, rand(["ant", "seed"])), 1:N) +end + +[take!(food) for i in 1:3] +```` + +```` +3-element Vector{String}: + "seed" + "seed" + "ant" +```` + +## Define anteaters and how to "train" them + +````@julia +mutable struct Anteater + health::Float64 # 0.0 = dead; 1.0 = excellent health + n::Int # number of ants consumed + Anteater(health) = new(health, 0) +end + +function eat!(eater::Anteater) + take!(food) == "ant" && rand() <= eater.health && (eater.n += 1) + return eater +end; +```` + +"Training" for `n` iterations means take `n` items from the `food` + source and try to catch and eat each item if it is an "ant": + +````@julia +function IterationControl.train!(eater::Anteater, n::Int) + foreach(i->eat!(eater), 1:n) + return eater +end; +```` + +## Create a collection of competitors + +````@julia +eaters = [Anteater(0.7), Anteater(0.8), Anteater(0.9), Anteater(1.0)] +```` + +```` +4-element Vector{Main.var"##277".Anteater}: + Main.var"##277".Anteater(0.7, 0) + Main.var"##277".Anteater(0.8, 0) + Main.var"##277".Anteater(0.9, 0) + Main.var"##277".Anteater(1.0, 0) +```` + +````@julia +n_eaters = length(eaters); +```` + +## Define the controls that will define the rules of the competition + +````@julia +const LEAD_TRIGGERING_DEFEAT = 5 +const ANTS_TO_WIN = 100 + +scores(eaters) = [e.n for e in eaters] + +is_too_far_behind(eater) = + maximum(scores(eaters)) - eater.n > LEAD_TRIGGERING_DEFEAT + +has_won(eater) = eater.n >= ANTS_TO_WIN + +controls = [ + Step(1), + Callback(is_too_far_behind, stop_if_true=true), + Callback(has_won, stop_if_true=true), + Callback(eater->eater.n), + NumberLimit(1000), +]; +```` + +## Run the competition + +````@julia +tasks = [Threads.@spawn IterationControl.train!(e, controls...) for e in eaters] +histories = map(fetch.(tasks)) do traces + traces[4].called_back +end +```` + +```` +4-element Vector{Vector{Int64}}: + [1, 1, 1, 2, 3, 3, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 7, 7, 7, 7, 8, 8, 9, 9, 9, 9, 9, 10, 10, 11, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13] + [0, 0, 0, 1, 1, 1, 2, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 9, 9, 10, 10, 10, 10] + [0, 0, 0, 0, 1, 1, 1, 1, 2, 3, 4, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 11, 11, 12, 13, 13, 14, 15, 16, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 20, 21, 21, 22, 23, 23, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26] + [0, 0, 0, 1, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 9, 9, 10, 11, 12, 13, 13, 13, 14, 15, 16, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 23, 24, 24, 24, 25, 26, 26, 26, 26, 26, 26, 27, 28, 28, 29, 30, 31, 31, 31, 32, 32, 33, 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, 38, 38, 39, 40, 41, 42, 42, 43, 43, 44, 45, 45, 45, 46, 47, 48, 48, 49, 49, 49, 50, 51, 51, 51, 52, 53, 53, 53, 53, 54, 54, 55, 55, 56, 57, 57, 57, 57, 58, 58, 59, 59, 60, 60, 61, 62, 63, 63, 64, 65, 66, 66, 67, 68, 68, 68, 69, 69, 70, 71, 71, 72, 73, 73, 73, 74, 75, 75, 76, 76, 76, 77, 77, 78, 79, 80, 81, 81, 82, 82, 83, 84, 84, 85, 86, 87, 88, 89, 89, 89, 90, 91, 91, 91, 91, 92, 92, 92, 93, 94, 94, 94, 94, 95, 95, 96, 97, 98, 98, 99, 100] +```` + +## Plot the results + +````@julia +fig = plot() +for (j, e) in enumerate(eaters) + plot!(histories[j], label=repr(e), linewidth=2); +end +ylabel!(fig, "ants eaten"); +savefig(fig, "anteaters.svg"); +```` + +![](anteaters.svg) + +--- + +*This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).* + diff --git a/docs/src/notebooks/particle/Project.toml b/docs/src/notebooks/particle/Project.toml new file mode 100644 index 0000000..54a62f2 --- /dev/null +++ b/docs/src/notebooks/particle/Project.toml @@ -0,0 +1,6 @@ +[deps] +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +IterationControl = "b3c1a2ee-3fec-4384-bf48-272ea71de57c" + +[compat] +Plots = "1" diff --git a/docs/src/notebooks/particle/notebook.jl b/docs/src/notebooks/particle/notebook.jl new file mode 100644 index 0000000..d8d5dd1 --- /dev/null +++ b/docs/src/notebooks/particle/notebook.jl @@ -0,0 +1,120 @@ +# # Controlling A Particle Tracker + +# > A demonstration of the `Data` control for online training, with callbacks + +# The source for this demonstration can be found +# [here](https://github.com/JuliaAI/IterationControl.jl/tree/dev/docs/src/notebooks/particle/). + +using Plots +using IterationControl + +# Consider an object that tracks a particle in one dimension, moving +# in discrete time at a speed proportional to the distance away from +# some moving target. We are viewing this particle as a toy example of +# online learning and call the constant of proportionality `η` a +# *learning rate*. + +# The particle is initially at rest at the origin, which is also where +# the target begins. + +# Calling `train!` on the particle moves it along for the specified +# number of time steps, while calling `ingest!` updates the target +# position ("data ingestion"). + +mutable struct Particle + position::Float64 + target::Float64 + velocity::Float64 + η::Float64 # learning rate + Particle(η) = new(0.0, 0.0, 0.0, η) +end + +Particle(; η=0.1) = Particle(η) + +loss(model::Particle) = abs(model.target - model.position) + +# A method to train for one time step: + +function train!(model::Particle) + model.velocity = model.η*(model.target - model.position) + model.position = model.position + model.velocity +end; + +# And for multiple time steps: + +function train!(model, n) + foreach(1:n) do _ + train!(model) + end + return nothing +end; + +# A method for updating the target: + +function ingest!(model::Particle, target) + model.target = target + return nothing +end + +# Lifting the `train!`, `loss` and `ingest!` methods to IterationControl.jl: + +IterationControl.train!(model::Particle, n::Int) = train!(model, n) +IterationControl.loss(model::Particle) = loss(model) +IterationControl.ingest!(model::Particle, datum) = ingest!(model, datum); + +# Lets `train` our particle by alternatively setting the target position to -1 or +1 every +# 5 time steps, for a total of 40 time steps (four -1/+1 cycles). We will then then allow +# our particle to "cruise" with the target fixed thereafter (the provided `data` iterator +# having been exhausted). This cruising is allowed to continue until the particle is +# within 0.01 units of the target. + +# We use a learning rate of 0.1. + +model = Particle(0.1) +data = repeat([-1, 1], outer=4) + +#- + +traces = IterationControl.train!( + model, + Data(data), + Step(5), + Callback(model->model.target), + Callback(model->model.position), + Threshold(0.01), + TimeLimit(0.005), +) + +#- + +target_positions = traces[3].called_back +particle_positions = traces[4].called_back +fig = scatter(target_positions, label="target", ms=6); +scatter!(fig, particle_positions, label="particle", ms=6); +savefig(fig, "particle.svg"); + +# ![](particle.svg) + +losses = traces[5].loss + +# Switching the target just once more to the -1 position and +# continuing training until the particle and target are within 0.01: + +data = [-1,] + +traces = IterationControl.train!( + model, + Data(data), + Step(5), + Callback(model->model.target), + Callback(model->model.position), + Threshold(0.01), + TimeLimit(0.005) +) +append!(target_positions, traces[3].called_back) +append!(particle_positions,traces[4].called_back) +fig = scatter(target_positions, label="target", ms=6); +scatter!(fig, particle_positions, label="particle", ms=6); +savefig(fig, "particle2.svg"); + +# ![](particle2.svg) diff --git a/docs/src/notebooks/particle/notebook.md b/docs/src/notebooks/particle/notebook.md new file mode 100644 index 0000000..5c21a36 --- /dev/null +++ b/docs/src/notebooks/particle/notebook.md @@ -0,0 +1,204 @@ +```@meta +EditURL = "notebook.jl" +``` + +# Controlling A Particle Tracker + +> A demonstration of the `Data` control for online training, with callbacks + +The source for this demonstration can be found +[here](https://github.com/JuliaAI/IterationControl.jl/tree/dev/docs/src/notebooks/particle/). + +````@julia +using Plots +using IterationControl +```` + +Consider an object that tracks a particle in one dimension, moving +in discrete time at a speed proportional to the distance away from +some moving target. We are viewing this particle as a toy example of +online learning and call the constant of proportionality `η` a +*learning rate*. + +The particle is initially at rest at the origin, which is also where +the target begins. + +Calling `train!` on the particle moves it along for the specified +number of time steps, while calling `ingest!` updates the target +position ("data ingestion"). + +````@julia +mutable struct Particle + position::Float64 + target::Float64 + velocity::Float64 + η::Float64 # learning rate + Particle(η) = new(0.0, 0.0, 0.0, η) +end + +Particle(; η=0.1) = Particle(η) + +loss(model::Particle) = abs(model.target - model.position) +```` + +```` +loss (generic function with 1 method) +```` + +A method to train for one time step: + +````@julia +function train!(model::Particle) + model.velocity = model.η*(model.target - model.position) + model.position = model.position + model.velocity +end; +```` + +And for multiple time steps: + +````@julia +function train!(model, n) + foreach(1:n) do _ + train!(model) + end + return nothing +end; +```` + +A method for updating the target: + +````@julia +function ingest!(model::Particle, target) + model.target = target + return nothing +end +```` + +```` +ingest! (generic function with 1 method) +```` + +Lifting the `train!`, `loss` and `ingest!` methods to IterationControl.jl: + +````@julia +IterationControl.train!(model::Particle, n::Int) = train!(model, n) +IterationControl.loss(model::Particle) = loss(model) +IterationControl.ingest!(model::Particle, datum) = ingest!(model, datum); +```` + +Lets `train` our particle by alternatively setting the target position to -1 or +1 every +5 time steps, for a total of 40 time steps (four -1/+1 cycles). We will then then allow +our particle to "cruise" with the target fixed thereafter (the provided `data` iterator +having been exhausted). This cruising is allowed to continue until the particle is +within 0.01 units of the target. + +We use a learning rate of 0.1. + +````@julia +model = Particle(0.1) +data = repeat([-1, 1], outer=4) +```` + +```` +8-element Vector{Int64}: + -1 + 1 + -1 + 1 + -1 + 1 + -1 + 1 +```` + +````@julia +traces = IterationControl.train!( + model, + Data(data), + Step(5), + Callback(model->model.target), + Callback(model->model.position), + Threshold(0.01), + TimeLimit(0.005), +) +```` + +```` +6-element Vector{Any}: + ControlTrace{<:Data} + ControlTrace{<:Step}, ending (new_iterations = 85, new_steps = 17) + ControlTrace{<:Callback}, ending (called_back = 1.0,) + ControlTrace{<:Callback}, ending (called_back = 0.9934860451342596,) + ControlTrace{<:Threshold}, ending (loss = 0.006513954865740357, excess = -0.0034860451342596434) + ControlTrace{<:TimeLimit}, ending (time = Dates.DateTime("2026-10-19T09:25:05.875"),) +```` + +````@julia +target_positions = traces[3].called_back +particle_positions = traces[4].called_back +fig = scatter(target_positions, label="target", ms=6); +scatter!(fig, particle_positions, label="particle", ms=6); +savefig(fig, "particle.svg"); +```` + +![](particle.svg) + +````@julia +losses = traces[5].loss +```` + +```` +17-element Vector{Float64}: + 0.59049 + 0.8323015599 + 0.689514251894649 + 0.7738287293987287 + 0.7240418735773446 + 0.7534405140713137 + 0.7360809108460299 + 0.7463315829545278 + 0.44070133641881915 + 0.26022973214194844 + 0.15366305453249895 + 0.09073649707089537 + 0.05357899415539302 + 0.031637860258817985 + 0.018681840104229352 + 0.011031439763146467 + 0.006513954865740357 +```` + +Switching the target just once more to the -1 position and +continuing training until the particle and target are within 0.01: + +````@julia +data = [-1,] + +traces = IterationControl.train!( + model, + Data(data), + Step(5), + Callback(model->model.target), + Callback(model->model.position), + Threshold(0.01), + TimeLimit(0.005) +) +append!(target_positions, traces[3].called_back) +append!(particle_positions,traces[4].called_back) +fig = scatter(target_positions, label="target", ms=6); +scatter!(fig, particle_positions, label="particle", ms=6); +savefig(fig, "particle2.svg"); +```` + +```` +[ Info: final loss: 0.006066681833463705 +[ Info: Stop triggered by EarlyStopping.Threshold(0.01) stopping criterion. + +```` + +![](particle2.svg) + +--- + +*This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).* + diff --git a/docs/src/notebooks/particle/particle.svg b/docs/src/notebooks/particle/particle.svg new file mode 100644 index 0000000..8332343 --- /dev/null +++ b/docs/src/notebooks/particle/particle.svg @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/notebooks/particle/particle2.svg b/docs/src/notebooks/particle/particle2.svg new file mode 100644 index 0000000..1bd1ba3 --- /dev/null +++ b/docs/src/notebooks/particle/particle2.svg @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/notebooks/tree_booster/Project.toml b/docs/src/notebooks/tree_booster/Project.toml new file mode 100644 index 0000000..bb31cc2 --- /dev/null +++ b/docs/src/notebooks/tree_booster/Project.toml @@ -0,0 +1,14 @@ +[deps] +EvoTrees = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5" +IterationControl = "b3c1a2ee-3fec-4384-bf48-272ea71de57c" +MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +StatisticalMeasures = "a19d573c-0a75-4610-95b3-7071388c7541" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[compat] +EvoTrees = "0.19" +MLJBase = "1.14" +Random = "1" +StatisticalMeasures = "0.3" +Statistics = "1" diff --git a/docs/src/notebooks/tree_booster/notebook.jl b/docs/src/notebooks/tree_booster/notebook.jl new file mode 100644 index 0000000..fa1ff72 --- /dev/null +++ b/docs/src/notebooks/tree_booster/notebook.jl @@ -0,0 +1,76 @@ +# # Early Stopping for a Tree Booster + +# > Implementing early stopping for an iterative [MLJ](https://juliaml.ai) model. + +# In actual practice, an MLJ user will use +# [`IteratedModel`](https://juliaai.github.io/MLJ.jl/stable/controlling_iterative_models/) +# wrapper, which is built on top of IterationControl.jl, because this is more convenient +# and compositional. + +# The source for this demonstration can be found +# [here](https://github.com/JuliaAI/IterationControl.jl/tree/dev/docs/src/notebooks/tree_booster/). + + +import MLJBase +import EvoTrees.EvoTreeClassifier +using IterationControl +import StatisticalMeasures.log_loss + +using Statistics +using Random +Random.seed!(123) + +# Loading some data and splitting observation indices into test/train: + +X, y = MLJBase.@load_iris; # a table and a vector +train, test = MLJBase.partition(eachindex(y), 0.7, shuffle=true); + +# Instantiate an MLJ model: + +booster = EvoTreeClassifier(nrounds=1) +Booster = typeof(booster); + +# Note that in MLJ a "model" is just a container for hyper-parameters. The objects we will +# iteratively train here are MLJ +# [*machines*](https://JuliaAI.github.io/MLJBase.jl/dev/machines/); these bind the model +# to train/test data and, in the case of iterative models, can be trained using a +# warm-restart. + +# Creating a machine: + +mach = MLJBase.machine(booster, X, y); + +# Lifting MLJ's `fit!(::Machine)` method to `IterativeControl.train!`: + +function IterationControl.train!(mach::MLJBase.Machine{<:Booster}, n::Int) + mlj_model = mach.model + mlj_model.nrounds = mlj_model.nrounds + n + MLJBase.fit!(mach, rows=train, verbosity=0) +end; + +# Lifting the out-of-sample loss: + +function IterationControl.loss(mach::MLJBase.Machine{<:Booster}) + mlj_model = mach.model + yhat = MLJBase.predict(mach, rows=test) + return log_loss(yhat, y[test]) +end; + +# Iterating with controls: + +IterationControl.train!( + mach, + Step(5), + GL(), + Info(IterationControl.loss), +); + +# Continuing iteration with a different stopping criterion: + +IterationControl.train!( + mach, + Step(5), + NumberLimit(10), + Info(IterationControl.loss), +); + diff --git a/docs/src/notebooks/tree_booster/notebook.md b/docs/src/notebooks/tree_booster/notebook.md new file mode 100644 index 0000000..1a3d1b3 --- /dev/null +++ b/docs/src/notebooks/tree_booster/notebook.md @@ -0,0 +1,139 @@ +```@meta +EditURL = "notebook.jl" +``` + +# Early Stopping for a Tree Booster + +> Implementing early stopping for an iterative [MLJ](https://juliaml.ai) model. + +In actual practice, an MLJ user will use +[`IteratedModel`](https://juliaai.github.io/MLJ.jl/stable/controlling_iterative_models/) +wrapper, which is built on top of IterationControl.jl, because this is more convenient +and compositional. + +The source for this demonstration can be found +[here](https://github.com/JuliaAI/IterationControl.jl/tree/dev/docs/src/notebooks/tree_booster/). + +````@julia +import MLJBase +import EvoTrees.EvoTreeClassifier +using IterationControl +import StatisticalMeasures.log_loss + +using Statistics +using Random +Random.seed!(123) +```` + +```` +Random.TaskLocalRNG() +```` + +Loading some data and splitting observation indices into test/train: + +````@julia +X, y = MLJBase.@load_iris; # a table and a vector +train, test = MLJBase.partition(eachindex(y), 0.7, shuffle=true); +```` + +Instantiate an MLJ model: + +````@julia +booster = EvoTreeClassifier(nrounds=1) +Booster = typeof(booster); +```` + +Note that in MLJ a "model" is just a container for hyper-parameters. The objects we will +iteratively train here are MLJ +[*machines*](https://JuliaAI.github.io/MLJBase.jl/dev/machines/); these bind the model +to train/test data and, in the case of iterative models, can be trained using a +warm-restart. + +Creating a machine: + +````@julia +mach = MLJBase.machine(booster, X, y); +```` + +Lifting MLJ's `fit!(::Machine)` method to `IterativeControl.train!`: + +````@julia +function IterationControl.train!(mach::MLJBase.Machine{<:Booster}, n::Int) + mlj_model = mach.model + mlj_model.nrounds = mlj_model.nrounds + n + MLJBase.fit!(mach, rows=train, verbosity=0) +end; +```` + +Lifting the out-of-sample loss: + +````@julia +function IterationControl.loss(mach::MLJBase.Machine{<:Booster}) + mlj_model = mach.model + yhat = MLJBase.predict(mach, rows=test) + return log_loss(yhat, y[test]) +end; +```` + +Iterating with controls: + +````@julia +IterationControl.train!( + mach, + Step(5), + GL(), + Info(IterationControl.loss), +); +```` + +```` +[ Info: 0.39655590607605007 +[ Info: 0.25803830912951475 +[ Info: 0.20369477186348553 +[ Info: 0.17930965646052324 +[ Info: 0.15988563045516432 +[ Info: 0.14971082736701255 +[ Info: 0.1419051277333237 +[ Info: 0.13624241869102463 +[ Info: 0.12996659052675014 +[ Info: 0.12758077552017272 +[ Info: 0.1241560953359005 +[ Info: 0.12118229064975733 +[ Info: 0.11805805490577778 +[ Info: 0.12337760263694478 +[ Info: final loss: 0.12337760263694478 +[ Info: Stop triggered by EarlyStopping.GL(2.0) stopping criterion. + +```` + +Continuing iteration with a different stopping criterion: + +````@julia +IterationControl.train!( + mach, + Step(5), + NumberLimit(10), + Info(IterationControl.loss), +); +```` + +```` +[ Info: 0.1284682197583394 +[ Info: 0.1333389660968577 +[ Info: 0.13635253372234196 +[ Info: 0.14081266985093113 +[ Info: 0.14362910766700177 +[ Info: 0.1463502830758916 +[ Info: 0.14873864305645823 +[ Info: 0.15129491860311206 +[ Info: 0.154195294592051 +[ Info: 0.15754958515966186 +[ Info: final loss: 0.15754958515966186 +[ Info: Stop triggered by EarlyStopping.NumberLimit(10) stopping criterion. + +```` + +--- + +*This page was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).* + diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..fb40b13 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,5 @@ +The notebooks in this folder may work with Julia 1.6 but not later versions. + +Versions of the notebooks working with later Julia versions have been moved to +/docs/src/notebooks/ and are now rendered as part of the [main +documentation](https://juliaai.github.io/IterationControl.jl/stable/). diff --git a/examples/particle/particle.ipynb b/examples/particle/particle.ipynb index 7d6c827..5d0f835 100644 --- a/examples/particle/particle.ipynb +++ b/examples/particle/particle.ipynb @@ -1092,7 +1092,7 @@ "cell_type": "markdown", "source": [ "Switching the target just once more to the -1 position and\n", - "continuing training until the particle and target are withing 0.01:" + "continuing training until the particle and target are within 0.01:" ], "metadata": {} }, diff --git a/examples/particle/particle.jl b/examples/particle/particle.jl index 62f11ba..d4f4ce3 100644 --- a/examples/particle/particle.jl +++ b/examples/particle/particle.jl @@ -101,7 +101,7 @@ scatter!(particle_positions, label="particle", ms=6) loss(model) # Switching the target just once more to the -1 position and -# continuing training until the particle and target are withing 0.01: +# continuing training until the particle and target are within 0.01: data = [-1,] diff --git a/examples/particle/particle.md b/examples/particle/particle.md index 47aedb4..be9cc9b 100644 --- a/examples/particle/particle.md +++ b/examples/particle/particle.md @@ -118,7 +118,7 @@ loss(model) ``` Switching the target just once more to the -1 position and -continuing training until the particle and target are withing 0.01: +continuing training until the particle and target are within 0.01: ```@example particle data = [-1,] diff --git a/examples/square_rooter/square_rooter.ipynb b/examples/square_rooter/square_rooter.ipynb index 618aeae..d9ac572 100644 --- a/examples/square_rooter/square_rooter.ipynb +++ b/examples/square_rooter/square_rooter.ipynb @@ -230,7 +230,7 @@ "difference between `x` and the square of `root`), then we can lift\n", "that method to `IterationControl.loss` to enable control using\n", "loss-based stopping criteria, such as a loss threshold. In the\n", - "demonstation below, we also include a callback:" + "demonstration below, we also include a callback:" ], "metadata": {} }, diff --git a/examples/square_rooter/square_rooter.jl b/examples/square_rooter/square_rooter.jl index b0b7bb2..b15b1b8 100644 --- a/examples/square_rooter/square_rooter.jl +++ b/examples/square_rooter/square_rooter.jl @@ -70,7 +70,7 @@ IterationControl.train!(model, Step(2), NumberLimit(3), Info(m->m.root)); # difference between `x` and the square of `root`), then we can lift # that method to `IterationControl.loss` to enable control using # loss-based stopping criteria, such as a loss threshold. In the -# demonstation below, we also include a callback: +# demonstration below, we also include a callback: model = SquareRooter(4) train!(model, 1) diff --git a/examples/square_rooter/square_rooter.md b/examples/square_rooter/square_rooter.md index ef3b4f7..becec3a 100644 --- a/examples/square_rooter/square_rooter.md +++ b/examples/square_rooter/square_rooter.md @@ -124,7 +124,7 @@ If `model` admits a method returning a loss (for example, the difference between `x` and the square of `root`), then we can lift that method to `IterationControl.loss` to enable control using loss-based stopping criteria, such as a loss threshold. In the -demonstation below, we also include a callback: +demonstration below, we also include a callback: ```julia model = SquareRooter(4) diff --git a/ext/TablesExt.jl b/ext/TablesExt.jl new file mode 100644 index 0000000..78931a5 --- /dev/null +++ b/ext/TablesExt.jl @@ -0,0 +1,17 @@ +module TablesExt + +import IterationControl +import IterationControl.ControlTrace +import Tables + +Tables.istable(::Type{<:ControlTrace}) = true +Tables.columnaccess(::Type{<:ControlTrace}) = true +Tables.columns(trace::ControlTrace) = Tables.columns(trace.table) +Tables.columnindex(trace::ControlTrace, idx::Symbol) = Tables.columnindex(trace.table) +Tables.columnindex(trace::ControlTrace, idx::AbstractString) = + Tables.columnindex(trace, Symbol(idx)) +Tables.schema(trace::ControlTrace) = Tables.schema(trace.table) +Tables.getcolumn(trace::ControlTrace, I) = Tables.getcolumn(trace.table, I) +Tables.subset(trace::ControlTrace, I; kwargs...) = Tables.subset(trace.table, I; kwargs...) + +end # module diff --git a/src/IterationControl.jl b/src/IterationControl.jl index eb01d34..4bb16a8 100644 --- a/src/IterationControl.jl +++ b/src/IterationControl.jl @@ -1,7 +1,7 @@ module IterationControl import Base.* -import EarlyStopping: done, StoppingCriterion, StoppingCriterion +import EarlyStopping: done, public_state, StoppingCriterion, StoppingCriterion using InteractiveUtils using EarlyStopping using REPL @@ -40,6 +40,7 @@ include("stopping_controls.jl") include("composite_controls.jl") include("wrapped_controls.jl") include("controls.jl") +include("traces.jl") include("train.jl") include("square_rooter.jl") include("deprecated.jl") diff --git a/src/api.jl b/src/api.jl index 746144e..052b4ae 100644 --- a/src/api.jl +++ b/src/api.jl @@ -1,9 +1,6 @@ # ------------------------------------------------------------------- # # MODEL METHODS -# *models* are externally defined objects with certain functionality -# that is exposed by overloading these methods: - err_train(model) = ArgumentError("`IterationControl.train!(model, ::Int)` "* "not overloaded "* @@ -11,75 +8,174 @@ err_train(model) = err_ingest(model) = ArgumentError("Cannot use `Data` control here, as "* - "`IterationControl.ingest!(model)` "* + "`IterationControl.ingest!(model, datum)` "* "not overloaded "* "for `typeof(model) = $(typeof(model))`. ") -# ## COMPULSORY + + +# ## COMPULSORY METHOD TO LIFT MODEL TRAINING TO IterationControl # train `model` for another `Δn` iterations: -train!(model, Δn) = throw(err_train(model)) +train!(model, Δn::Int) = throw(err_train(model)) -# ## REQUIRED FOR SOME CONTROLS + +# ## OTHER METHODS THAT LIFT TO IterationControl A MODEL METHOD # extract a single numerical estimate of `models`'s performance such # as an out-of-sample loss; smaller is understood to be better: +""" + IterationControl.loss(model) + +Return the current loss associated with `model`, which will be some measure of the quality +of `model`, such as an out-of-sample statistical loss. The smaller the loss, the better +the inferred quality. + +This method must be overloaded for any `model` type wanting to support an iteration +`control` with `IterationControl.needs_loss(control) == true`. + +""" loss(model) = nothing # extract a vector of per-iteration "training" losses, accumulated in # the last `train!(model, Δn)` call; this will have length `Δn`: + +""" + IterationControl.training_losses(model) + +Return a vector of all, or recent, per-iteration "training" losses, since training of +`model` began. Typically, a training loss is an in-sample statistical measure of the +quality of `model`, as opposed to the (out-of-sample) loss returned by +`IterationControl.loss(model)`. The smaller the loss, the better the inferred quality. + +Training losses are returned in chronological order. + +This method must be overloaded for any `model` type wanting to support an iteration +`control` with `IterationControl.needs_training_losses(control) == true`. + +""" training_losses(model) = nothing -# ingest data: +""" + IterationControl.ingest!(model, datum) + +Have `model` ingest new training `datum`, without actually training `model`. + +This method must be overloaded for any `model` type wanting to support an iteration +`control` with `IterationControl.needs_ingest(control) == true`. + +""" ingest!(model, data) = throw(err_ingest(model)) -# ## OPTIONAL +""" + IterationControl.expose(model) + +Unwrap `model` to expose that part on which user-specified functions in iteration controls +should act. For example, in [`IterationControl.train!`](@ref)`(model, Step(), +Info(model->print(model))`, it is `IterationControl.expose(model)` that gets logged +to `Info`, not `model`. -# so that user-specified functions (in eg `Info`) are applied to some -# internal part of the model, rather than `model` directly: +""" expose(model) = model # switch for expose; not to be overloaded: -expose(model, raw) = expose(model, Val(raw)) +expose(model, use_wrapped) = expose(model, Val(use_wrapped)) expose(model, ::Val{true}) = model expose(model, ::Val{false}) = expose(model) -# ------------------------------------------------------------------- -# # CONTROL METHODS +# # INTERFACE IMPLEMENTED BY ITERATION CONTROLS # compulsory: `update!` # optional: `done`, `takedown`, `needs_training_losses` -# called after first training event; returns initialized control -# "state": +const DOC_UPDATE = """ + The returned state, `state`, is not part of the public API. To inspect a user-facing form + of `state`, as recorded in control traces, call [`IterationControl.public_state(control, + state)`](@ref). + + # New control implementations + """ + +""" + IterationControl.update!(control, model, verbosity, n) + +Return initialized internal state for `control`, when controlling `model`. Here `n` is the +current number of training iterations that have have been applied to `model` so far. (This +needn't be one, as a previously applied `Step(k)` control may have `k > 1`). + +$DOC_UPDATE Compulsory for all new `control` types. + +""" update!(control, model, verbosity, n) = nothing -# called after all subsequent training events; returns new "state": +""" + IterationControl.update!(control, model, verbosity, n, old_state) + +Return updated internal state for `control`, when controlling `model`. Here `n` is the +current number of training iterations that have have been applied to `model` so far. + +$DOC_UPDATE May mutate `state`, but typically does not. +A fallback returns `old_state`. +""" update!(control, model, verbosity, n, state) = state -# should we stop? -done(control, state) = false - -# What to do after `control`, or some other control, has triggered -# a stop. Returns user-inspectable outcomes associated with the -# control's applications (separate from logging). This should be a -# named tuple, except for composite controls which return a tuple of -# named tuples (see composite_controls.jl): -takedown(control, verbosity, state) = NamedTuple() - -# If it is nonsensical to apply `control` to any model for which -# `training_losses(model)` has not been overloaded and we want an -# error thrown when this is attempted, then overload this trait to -# take the value `true`. Otherwise `control` is applied anyway, and -# `training_losses`, if called, returns `nothing`. + +""" + IterationControl.done(control, state) + +Returns `true` if and only if the `control` wants to trigger a stop. Method belongs to +`EarlyStopping.jl`, a dependency of `IterationControl`. + +# New control implementations + +A fallback always returns `false`. +""" +EarlyStopping.done + +""" + takedown(control, verbosity, state) -> message + +With the understanding that all `control` updates are complete, log to `Info` or `Warn` a +relevant message, and return the message as a string. + +# New control implementations + +Typically the message is only logged to `Info` if non-empty and `verbosity > 0` or, if it +is a warning, logged to `Warn` if `verbosity > -1`. Irrespective of the `verbosity` level, +the message should be returned, but can be `""`, the fallback value. + +Ideally, the message should be brief. (The user can get detailed information about the +final state of the control by inspecting the corresponding [`ControlTrace`](@ref) object.) + +""" +takedown(control, verbosity, state) = "" + +""" + IterationControl.needs_training_losses(control) + +Returns `true` if `control` is only supported for a model for which +[`IterationControl.training_losses(model)`](@ref) is implemented. + +""" needs_training_losses(control) = false -# If it is nonsensical to apply `control` to any model for which -# `loss(model)` has not been overloaded and we want an error thrown -# when this is attempted, then overload this trait to take the value -# `true`. Otherwise `control` is applied anyway, and -# `loss`, if called, returns `nothing`. +""" + IterationControl.needs_loss(control) + +Returns `true` if `control` is only supported for a model for which +[`IterationControl.loss(model)`](@ref) is implemented. + +""" needs_loss(control) = false + +""" + IterationControl.needs_ingest(control) + +Returns `true` if `control` is only supported for a model for which +[`IterationControl.ingest!(model, datum)`](@ref) is implemented. + +""" +needs_ingest(control) = false diff --git a/src/composite_controls.jl b/src/composite_controls.jl index f674849..9086c6a 100644 --- a/src/composite_controls.jl +++ b/src/composite_controls.jl @@ -17,7 +17,7 @@ CompositeControl(a, b, c...) = CompositeControl(CompositeControl(a,b), c...) composite(controls...) Construct an iteration control that applies the specified `controls` -in sequence. +in sequence. Mainly intended for internal use by IterationControls.jl. """ composite(controls...) = CompositeControl(controls...) @@ -49,6 +49,17 @@ function Base.show(io::IO, c::CompositeControl) print(io, "CompositeControl($list)") end +# # PUBLIC_STATE + +# fallback for atomic state: +function public_state(c::CompositeControl, state) + controls = flat(c) + states = flat(state) + return map(zip(controls, states)) do (c, s) + public_state(c, s) + end +end + # # RECURSION TO DEFINE `done` @@ -65,17 +76,24 @@ done(c::CompositeControl, state) = _done(c, state, false) # # RECURSION TO DEFINE `takedown` # fallback for atomic controls: -function _takedown(control, v, state, old_takedown) - new_takedown = (control, takedown(control, v, state)) - return tuple(old_takedown..., new_takedown) -end +_takedown(control, verbosity, state, old_takedown) = + (old_takedown..., takedown(control, verbosity, state)) # composite: -_takedown(c::CompositeControl, v, state, old_takedown) = - _takedown(c.b, v, state.b, - _takedown(c.a, v, state.a, old_takedown)) - -takedown(c::CompositeControl, v, state) = _takedown(c, v, state, ()) +_takedown(c::CompositeControl, verbosity, state, old_takedown) = + _takedown( + c.b, + verbosity, + state.b, + _takedown( + c.a, + verbosity, + state.a, + old_takedown, + ), + ) + +takedown(c::CompositeControl, verbosity, state) = _takedown(c, verbosity, state, ()) # # TRAITS diff --git a/src/controls.jl b/src/controls.jl index aee2699..3d87a0a 100644 --- a/src/controls.jl +++ b/src/controls.jl @@ -5,7 +5,7 @@ struct Step end # constructor: -Step(; n=5) = Step(n) +Step(; n=1) = Step(n) @create_docs(Step, header="Step(; n=1)", @@ -21,12 +21,8 @@ function update!(c::Step, model, verbosity, n, state=(new_iterations = 0,)) state = (new_iterations = new_iterations + c.n,) return state end +public_state(c::Step, state) = (; state..., new_steps=div(state.new_iterations, c.n)) -@inline function takedown(c::Step, verbosity, state) - verbosity > 1 && - @info "A total of $(state.new_iterations) iterations added. " - return state -end # # Info @@ -50,9 +46,11 @@ Info(; f::Function=identity) = Info(f) "See also [`Warn`](@ref), [`Error`](@ref). ") function update!(c::Info, model, verbosity, args...) - verbosity > 0 && @info _log_eval(c.f, model) - return nothing + state = _log_eval(c.f, model) + verbosity > 0 && @info state + return state end +public_state(::Info, state) = (; called_back=state) # # Warn @@ -89,12 +87,14 @@ function update!(c::Warn, model, verbosity, n, state=(warnings=(),)) end return newstate end - +public_state(c::Warn, state) = (;) +_warnings(n) = n > 1 ? "warnings" : "warning" function takedown(c::Warn, verbosity, state) - warnings = join(state.warnings, "\n") - verbosity > 1 && !isempty(warnings) && - @warn "A `Warn` control issued these warnings:\n$warnings" - return state + n_warn = length(state.warnings) + message = isempty(state.warnings) ? "" : + "A `Warn` control issued $n_warn $(_warnings(n_warn))." + verbosity > -1 && !isempty(message) && @warn message + return message end @@ -119,15 +119,17 @@ Error(predicate; f="", exception=nothing) = Error(predicate, f, exception) "Here `m` "* "is the object being iterated.\n\n"* "Specify `exception=...` to throw an immediate "* - "execption, without "* + "exception, without "* "waiting to the end of the control cycle.\n\n"* "See also [`Info`](@ref), [`Warn`](@ref). ") -function update!(c::Error, - model, - verbosity, - n=1, - state=(done = false, error=())) +function update!( + c::Error, + model, + verbosity, + n=1, + state=(done = false, error=nothing), + ) if c.predicate(model) error = _log_eval(c.f, model) @error error @@ -136,10 +138,10 @@ function update!(c::Error, end return state end - +public_state(::Error, state) = (; error=state.error) done(c::Error, state) = state.done -takedown(c::Error, verbosity, state) = state +takedown(c::Error, verbosity, state) = string(state.error) # # CALLBACK @@ -152,42 +154,43 @@ struct Callback{F<:Function} end # constructor: -Callback(; f=identity, - stop_if_true=false, - stop_message=nothing, - raw=false) = Callback(f, stop_if_true, stop_message, raw) +Callback( + ; f=identity, + stop_if_true=false, + stop_message=nothing, + raw=false, +) = Callback(f, stop_if_true, stop_message, raw) Callback(f; kwargs...) = Callback(; f=f, kwargs...) -@create_docs(Callback, - header="Callback(f=_->nothing, stop_if_true=false, "* - "stop_message=nothing, raw=false)", - example="Callback(m->put!(v, my_loss_function(m))", - body="Call `f(IterationControl.expose(m))`, where "* - "`m` is the object being iterated, unless `raw=true`, in "* - "which case call `f(m)` (guaranteed if `expose` has not been "* - "overloaded.) "* - "If `stop_if_true` is `true`, then trigger an early stop "* - "if the value returned by `f` is `true`, logging the "* - "`stop_message` if specified. ") - -function update!(c::Callback, model, verbosity, n, state=(done=false, )) - r = c.f(expose(model, c.raw)) - done = (c.stop_if_true && r isa Bool && r) ? true : false - return (done=done,) -end - -done(c::Callback, state) = state.done +@create_docs( + Callback, + header="Callback(f=_->nothing, stop_if_true=false, "* + "stop_message=nothing, raw=false)", + example="Callback(m->put!(v, my_loss_function(m))", + body="Call `f(IterationControl.expose(m))`, where "* + "`m` is the object being iterated, unless `raw=true`, in "* + "which case call `f(m)` (guaranteed if `expose` has not been "* + "overloaded.) "* + "If `stop_if_true` is `true`, then trigger an early stop "* + "if the value returned by `f` is `true`, logging the "* + "`stop_message` if specified. ", +) + +update!(c::Callback, model, verbosity, n, state=nothing) = c.f(expose(model, c.raw)) +public_state(::Callback, state) = (; called_back=state) +done(c::Callback, state) = c.stop_if_true && state isa Bool && state function takedown(c::Callback, verbosity, state) - if state.done - message = c.stop_message === nothing ? + if done(c, state) + message = isnothing(c.stop_message) ? "Stop triggered by a `Callback` control. " : c.stop_message verbosity > 0 && @info message - return (done = true, log = message) + return message else - return (done = false, log = "") + message = "" end + return "" end @@ -210,14 +213,14 @@ Base.show(io::IO, d::Data{S}) where S = body="In each application of this control a new `item` from the "* "iterable, `data`, is retrieved (using `iterate`) and "* "`IterationControl.ingest!(m, item)` is called. Here "* - "`m` is the object being iterated. \n\n"* - "A control becomes passive once the `data` iterable is done. "* + "`m` is the object being iteratively trained. \n\n"* + "The control becomes passive once the `data` iterable is done. "* "To trigger "* "a stop *after one passive application of the control*, set "* "`stop_when_exhausted=true`. ") -const DATA_EXHAUSTED = "Data exhausted. " -const DATA_STOP ="Stop triggered because data exhausted ." +const DATA_EXHAUSTED = "Data exhausted." +const DATA_STOP ="Stop triggered because data exhausted." function update!(c::Data, model, verbosity, n) iter = c.data @@ -254,17 +257,20 @@ function update!(c::Data, model, verbosity, n, state) return (iter_state = iter_state, done = done) end +public_state(::Data, state) = (;) + done(c::Data, state) = state.done function takedown(c::Data, verbosity, state) if state.done verbosity > 0 && @info DATA_STOP - return (done = true, log = DATA_STOP) - else - return (done = false, log = "") + return DATA_STOP end + return "" end +needs_ingest(::Data) = true + # # WithLossDo @@ -275,13 +281,15 @@ struct WithLossDo{F<:Function} end # constructor: -WithLossDo(; f=x->@info("loss: $x"), - stop_if_true=false, - stop_message=nothing) = WithLossDo(f, stop_if_true, stop_message) +WithLossDo( + ; f=identity, + stop_if_true=false, + stop_message=nothing, + ) = WithLossDo(f, stop_if_true, stop_message) WithLossDo(f; kwargs...) = WithLossDo(; f=f, kwargs...) @create_docs(WithLossDo, - header="WithLossDo(f=x->@info(\"loss: \$x\"), "* + header="WithLossDo(f=identity, "* "stop_if_true=false, "* "stop_message=nothing)", example="WithLossDo(x->put!(my_losses, x))", @@ -293,31 +301,33 @@ WithLossDo(f; kwargs...) = WithLossDo(; f=f, kwargs...) needs_loss(::WithLossDo) = true -function update!(c::WithLossDo, - model, - verbosity, - n, - state=(loss=nothing, done=false)) +function update!( + c::WithLossDo, + model, + verbosity, + n, + state=(; loss=nothing, called_back=nothing), + ) loss = IterationControl.loss(model) loss === nothing && throw(ERR_NEEDS_LOSS) - r = c.f(loss) - done = (c.stop_if_true && r isa Bool && r) ? true : false - return (loss=loss, done=done) + called_back = c.f(loss) + return (; loss, called_back) end -done(c::WithLossDo, state) = state.done +function done(c::WithLossDo, state) + called_back = state.called_back + return c.stop_if_true && called_back isa Bool && called_back +end function takedown(c::WithLossDo, verbosity, state) - verbosity > 1 && @info "final loss: $(state.loss). " - if state.done + if done(c, state) message = c.stop_message === nothing ? "Stop triggered by a `WithLossDo` control. " : c.stop_message verbosity > 0 && @info message - return merge(state, (log = message,)) - else - return merge(state, (log = "",)) + return message end + return "" end @@ -330,15 +340,16 @@ struct WithTrainingLossesDo{F<:Function} end # constructor: -WithTrainingLossesDo(; f=v->@info("training: $v"), - stop_if_true=false, - stop_message=nothing) = - WithTrainingLossesDo(f, stop_if_true, stop_message) +WithTrainingLossesDo( + ; f=identity, + stop_if_true=false, + stop_message=nothing, +) = WithTrainingLossesDo(f, stop_if_true, stop_message) WithTrainingLossesDo(f; kwargs...) = WithTrainingLossesDo(; f=f, kwargs...) @create_docs(WithTrainingLossesDo, - header="WithTrainingLossesDo(f=v->@info(\"training: \$v\"), "* + header="WithTrainingLossesDo(f=identity, "* "stop_if_true=false, "* "stop_message=nothing)", example="WithTrainingLossesDo(v->put!(my_losses, last(v))", @@ -355,28 +366,27 @@ function update!(c::WithTrainingLossesDo, model, verbosity, n, - state=(latest_training_loss = nothing, done = false)) + state=(latest_training_loss = nothing, called_back=nothing)) losses = IterationControl.training_losses(model) losses === nothing && throw(ERR_NEEDS_TRAINING_LOSSES) - r = c.f(losses) - done = (c.stop_if_true && r isa Bool && r) ? true : false - return (latest_training_loss=losses[end], done=done) + called_back = c.f(losses) + return (; training_loss=losses[end], called_back) end -done(c::WithTrainingLossesDo, state) = state.done +function done(c::WithTrainingLossesDo, state) + called_back = state.called_back + return c.stop_if_true && called_back isa Bool && called_back +end function takedown(c::WithTrainingLossesDo, verbosity, state) - verbosity > 1 && - @info "final training loss: $(state.latest_training_loss). " - if state.done + if done(c, state) message = c.stop_message === nothing ? "Stop triggered by a `WithTrainingLossesDo` control. " : c.stop_message verbosity > 0 && @info message - return merge(state, (log = message,)) - else - return merge(state, (log = "",)) + return message end + return "" end @@ -389,45 +399,49 @@ struct WithNumberDo{F<:Function} end # constructor: -WithNumberDo(; f=n->@info("number: $n"), - stop_if_true=false, - stop_message=nothing) = WithNumberDo(f, stop_if_true, stop_message) +WithNumberDo( + ; f=identity, + stop_if_true=false, + stop_message=nothing, +) = WithNumberDo(f, stop_if_true, stop_message) WithNumberDo(f; kwargs...) = WithNumberDo(; f=f, kwargs...) @create_docs(WithNumberDo, - header="WithNumberDo(f=n->@info(\"number: \$n\"), "* + header="WithNumberDo(f=identity, "* "stop_if_true=false, "* "stop_message=nothing)", example="WithNumberDo(n->put!(my_channel, n))", body="Call `f(n + 1)`, where "* - "`n` is the number of complete control cycles. "* - "of the control (so, `n = 1, 2, 3, ...`, unless control is "* - "wrapped in a [`IterationControl.skip`](@ref))`.\n\n"* + "`n` is the number of already completed control cycles "* + "(so, `n = 1, 2, 3, ...`, unless control is "* + "wrapped in a [`IterationControl.skip`](@ref)).\n\n"* "If `stop_if_true` is `true`, then trigger an early stop "* "if the value returned by `f` is `true`, logging the "* "`stop_message` if specified. ") -function update!(c::WithNumberDo, - model, - verbosity, - n, - state=(done = false, n = n)) - r = c.f(n) - done = (c.stop_if_true && r isa Bool && r) ? true : false - return (done = done, n = n) +function update!( + c::WithNumberDo, + model, + verbosity, + n, + state=(number=n, called_back=nothing), + ) + called_back = c.f(n) + return (; number = n, called_back) end -done(c::WithNumberDo, state) = state.done +function done(c::WithNumberDo, state) + called_back = state.called_back + return c.stop_if_true && called_back isa Bool && called_back +end function takedown(c::WithNumberDo, verbosity, state) - verbosity > 1 && @info "final number: $(state.n). " - if state.done + if done(c, state) message = c.stop_message === nothing ? "Stop triggered by a `WithNumberDo` control. " : c.stop_message verbosity > 0 && @info message - return merge(state, (log = message,)) - else - return merge(state, (log = "",)) + return message end + return "" end diff --git a/src/square_rooter.jl b/src/square_rooter.jl index 9337d29..4024996 100644 --- a/src/square_rooter.jl +++ b/src/square_rooter.jl @@ -2,6 +2,8 @@ # Consider a model to compute Babylonian approximations to a square root: +module SquareRooterExample + mutable struct SquareRooter x::Float64 # input - number to be square rooted root::Float64 # current approximation of root @@ -9,7 +11,7 @@ mutable struct SquareRooter SquareRooter(x) = new(x, 1.0, Float64[]) end -function IterationControl.train!(m::SquareRooter, Δn::Int) +function train!(m::SquareRooter, Δn::Int) m.training_losses = Float64[] for i in 1:Δn next_guess = (m.root + m.x/m.root)/2 @@ -18,5 +20,14 @@ function IterationControl.train!(m::SquareRooter, Δn::Int) end end -IterationControl.loss(m::SquareRooter) = abs(m.root^2 - m.x) -IterationControl.training_losses(m::SquareRooter) = m.training_losses +loss(m::SquareRooter) = abs(m.root^2 - m.x) +training_losses(m::SquareRooter) = m.training_losses + +export SquareRooter, train!, loss, training_losses + +end + +import .SquareRooterExample: SquareRooter +train!(m::SquareRooter, Δn::Int) = SquareRooterExample.train!(m, Δn) +loss(m::SquareRooter) = SquareRooterExample.loss(m) +training_losses(m::SquareRooter) = SquareRooterExample.training_losses(m) diff --git a/src/stopping_controls.jl b/src/stopping_controls.jl index f6cab9d..d42956a 100644 --- a/src/stopping_controls.jl +++ b/src/stopping_controls.jl @@ -33,10 +33,13 @@ for ex in EARLY_STOPPING_WRAPPERS end |> eval end -function update!(c::StoppingCriterion, - model, - verbosity, - n, state=nothing) +function update!( + c::StoppingCriterion, + model, + verbosity, + n, + state=nothing, + ) _loss = loss(model) _training_losses = training_losses(model) if _training_losses !== nothing && !isempty(_training_losses) @@ -49,11 +52,7 @@ function update!(c::StoppingCriterion, end function takedown(c::StoppingCriterion, verbosity, state) - if done(c, state) - message = ES.message(c, state) - verbosity > 0 && @info message - return (done = true, log = message) - else - return (done = false, log = "") - end + message = done(c, state) ? ES.message(c, state) : "" + verbosity > 0 && !isempty(message) && @info message + return message end diff --git a/src/traces.jl b/src/traces.jl new file mode 100644 index 0000000..0dfdbe1 --- /dev/null +++ b/src/traces.jl @@ -0,0 +1,204 @@ +""" + columntable(rows) + + +**Private method.** + +Convert a tuple of named-tuples sharing the same keys into a named-tuple of vectors. + +""" +function columntable(rows) + names = keys(first(rows)) + columns = collect.(zip(values.(rows)...)) + NamedTuple{names}(columns) +end + +_last(columntable) = + NamedTuple{keys(columntable)}(last.(values(columntable))) + +# NOTE. because of `skip` controls, we have to allow for the possibility that +# `public_state(control, state) = `nothing`, so that construction of the table is delayed. + +""" + ControlTrace{C} + +Type of object recording a trace for a control of type `C`. + +All instances have the following public properties (accessed with dot syntax): + +- `control` + +- `table`: a named-tuple of vectors recording a history of the internal state of the + control as iteration proceeded + +- `triggered_stop`: `true` if `control` was responsible for stopping training + +- `message`: The message that was logged to `Info` or `Warn` when + [`IterationControl.train!`](@ref) completed (available even if a small `verbosity` level + suppressed the logging) + +Additional properties forward to the `table` property to access a particular component of +state. Note here "state" means public state (i.e, what is returned by +[`IterationControl.public_state(control, rawstate)`](@ref), where `rawstate` is the raw +internal state). + +A `ControlTrace` object implements the Tables.jl interface, so it can be converted to a +dataframe, for example: + +```julia-repl +using IterationControl +import IterationControl.SquareRooter +using DataFrames + +model = SquareRooter(9) + +julia> traces = IterationControl.train!( + model, + Step(), + Threshold(0.01), + WithLossDo(x->x<0.01), +) +[ Info: final loss: 0.0005493331704400362 +[ Info: final training loss: 0.02343785763332562 +[ Info: Stop triggered by Threshold(0.01) stopping criterion. +3-element Vector{Any}: + ControlTrace{<:Step}, ending (new_iterations = 4, new_steps = 4) + ControlTrace{<:Threshold}, ending (loss = 0.0005493331704400362, excess = -0.009450666829559964) + ControlTrace{<:WithLossDo}, ending (loss = 0.0005493331704400362, called_back = true) + +julia> trace = traces[end] +ControlTrace + control: WithLossDo{var"#2#3"}(var"#2#3"(), false, nothing) + loss: [ …, 2.5599999999999987, 0.1417301038062284, 0.0005493331704400362] + called_back: Bool[ …, 0, 0, 1] + table: (loss = …, called_back = …) + triggered_stop: false + +julia> trace.loss +4-element Vector{Float64}: + 16.0 + 2.5599999999999987 + 0.1417301038062284 + 0.0005493331704400362 + +julia> DataFrame(trace) +4×2 DataFrame + Row │ loss called_back + │ Float64 Bool +─────┼─────────────────────────── + 1 │ 16.0 false + 2 │ 2.56 false + 3 │ 0.14173 false + 4 │ 0.000549333 true +``` + +""" +mutable struct ControlTrace{C} + control::C + triggered_stop::Bool + table # named tuple of vectors or `nothing` + message::String + function ControlTrace(control::C, state) where C + triggered_stop = done(control, state) + public_state = IterationControl.public_state(control, state) + # if public state available, create table with one row: + table = isnothing(public_state) ? nothing : + columntable((public_state,)) + return new{C}(control, triggered_stop, table) + end +end + +Base.isempty(trace::ControlTrace) = isnothing(trace.table) +function reset!(trace::ControlTrace) + trace.message = "" + trace.triggered_stop = false + return trace +end +function reset!(trace::ControlTrace, control) + trace.control = control + reset!(trace) +end + +const TRACE_FIELDS = (:control, :table, :message, :triggered_stop) + +function Base.propertynames(trace::ControlTrace) + tablekeys = isnothing(trace.table) ? () : keys(trace.table) + return (tablekeys..., TRACE_FIELDS...) +end +function Base.getproperty(trace::ControlTrace, name::Symbol) + name in TRACE_FIELDS && return getfield(trace, name) + name in propertynames(trace) || throw(FieldError(typeof(trace), name)) + return getproperty(trace.table, name) +end + +function Base.push!(trace::ControlTrace, state; update_only::Bool=false) + public_state = IterationControl.public_state(trace.control, state) + isnothing(public_state) && return trace + if isnothing(trace.table) + trace.triggered_stop = done(trace.control, state) + # create a column table with one row: + trace.table = columntable((public_state,)) + return trace + end + for name in keys(public_state) + column = getproperty(trace.table, name) + if update_only + column[end] = getproperty(public_state, name) + else + push!(column, getproperty(public_state, name)) + end + end + trace.triggered_stop = done(trace.control, state) + return trace +end + +update!(trace::ControlTrace, state) = push!(trace, state; update_only=true) + +function takedown!(trace::ControlTrace, verbosity, state) + trace.message = takedown(trace.control, verbosity,state) + return trace +end + + +# Display helper +const OMITTED = "⟨omittted⟩" +_repr(x) = OMITTED +const Displayable = Union{Number,AbstractString,Char,Symbol} +function _repr(v::Vector{<:Displayable}) + length(v) ≤ 3 && return repr(v) + r = repr(last(v, 3)) + idx = findfirst(==('['), r) + return r[1:idx]*" …, "*r[idx+1:end] +end +_repr(T::DataType) = "<:$(nameof(T))" + +# Display, within containers, etc: +function Base.show(io::IO, trace::ControlTrace) + str = "ControlTrace{$(_repr(typeof(trace.control)))}" + if !isempty(trace) + last_row = _last(trace.table) + isempty(last_row) || (str *= ", ending $(_last(trace.table))") + end + print(io, str) +end + +# Display, standalone: +function Base.show(io::IO, ::MIME"text/plain", trace::ControlTrace) + names = propertynames(trace) + n = length(names) - 4 + names = names[1:n] # just the table column names + str = "ControlTrace" + isempty(trace) && (str *= " (empty)") + str *= "\n control: $(trace.control)" + for name in names + column = getproperty(trace.table, name) + str *= "\n $name: "*_repr(column) + end + table_str = "("*join(map(name->"$name = …", names), ", ")*")" + isnothing(trace.table) || (str *= "\n table: $table_str") + if isdefined(trace, :message) && !isempty(trace.message) + str *= "\n message: $(trace.message)" + end + str *= "\n triggered_stop: $(trace.triggered_stop)" + print(io, str) +end diff --git a/src/train.jl b/src/train.jl index 3b291b9..d66d79f 100644 --- a/src/train.jl +++ b/src/train.jl @@ -8,10 +8,131 @@ const ERR_NEEDS_TRAINING_LOSSES = ArgumentError( "Encountered a control that needs training losses but no training "* "losses found. ") -function train!(model, controls...; verbosity::Int=1) +const ERR_INCOMPATIBLE_TRACES = ArgumentError( + "The value of the keyword argument `traces` is incompatible with "* + "the specified controls. " +) + +iscompatible(traces, controls) = all(zip(traces, controls)) do (trace, control) + trace isa ControlTrace{typeof(control)} +end + +""" + train!(model, controls...; verbosity=1, compact=false, traces=[]) + +Train the iterative `model`, repeatedly applying the provided `controls` until one of the +controls triggers a stop. The first control must be `Step(k)`, for some integer `k` +specifying the number of training iterations between control cycles; `Step()` is the same +as `Step(1)`. + +Returns a vector of traces, one for each control. A trace records +the evolving internal state of each control, unless `compact=false`, in which case only +the current state is stored. + +```julia-repl +using IterationControl +model = IterationControl.SquareRooter(4) +controls = [Step(1), Threshold(1e-7), NumberLimit(100), InvalidValue()] + +julia> traces = IterationControl.train!(model, controls...) +[ Info: final loss: 8.881784197001252e-15 +[ Info: final training loss: 9.292229252721995e-8 +[ Info: Stop triggered by Threshold(1.0e-7) stopping criterion. + +julia> model.root +2.000000000000002 + +julia> traces[2] +ControlTrace + control: Threshold(1.0e-7) + loss: [ …, 0.002439396192741583, 3.716891878724482e-7, 8.881784197001252e-15] + excess: [ …, 0.002439296192741583, 2.716891878724482e-7, -9.99999911182158e-8] + table: (loss = …, excess = …) + message: Stop triggered by Threshold(1.0e-7) stopping criterion. + triggered_stop: true + +julia> traces[2].loss +5-element Vector{Float64}: + 2.25 + 0.20249999999999968 + 0.002439396192741583 + 3.716891878724482e-7 + 8.881784197001252e-15 +``` + +It is assumed that `IterationControl.train!(model, n::Int)` has been implemented (be sure +to include the `Int` annotation here!) and that this trains the model for `n` more +iterations. Specific controls may require implementations of +[`IterationControl.loss(model)`](@ref), [`IterationControl.training_losses(model)`](@ref), +or [`IterationControl.ingest!(model, datum)`](@ref). Refer to documentation for details. + +# Extended help + +# Appending to traces and live tracking + +For "live" access to the traces, while `train!` is running as an +asynchronous task, specify `traces=v`, where `v` is bound to some copy of `Any[]` to be +populated by the traces once the first control cycle completes, for subsequent mutation as +iteration proceeds: + +```julia-repl +model = IterationControl.SquareRooter(4) +controls = [Step(1), Threshold(0.01), Callback(_->sleep(0.5))] +traces = [] +Threads.@spawn IterationControl.train!(model, controls...; verbosity=0, traces=traces) +length(traces) # 0 +sleep(2) +length(traces) # 3 + +julia> traces +3-element Vector{Any}: + ControlTrace{<:Step}, ending (new_iterations = 3, new_steps = 3) + ControlTrace{<:Threshold}, ending (loss = 0.002439396192741583, excess = -0.007560603807258417) + ControlTrace{<:Callback}, ending (called_back = nothing,) + +julia> traces[2].loss +3-element Vector{Float64}: + 2.25 + 0.20249999999999968 + 0.002439396192741583 +``` + +Alternatively, one may specify `traces=v`, where `v` is the vector of traces returned by a +previous call to `train!` with the same control types. In that `v` is extended, and +matches the vector returned by the second call to `train!`. Continuing the above +example: + +```julia-repl +controls[2] = Threshold(1e-100) +IterationControl.train!(model, controls...; traces=traces) + +julia> traces[2].loss +6-element Vector{Float64}: + 2.25 + 0.20249999999999968 + 0.002439396192741583 + 3.716891878724482e-7 + 8.881784197001252e-15 + 0.0 + +julia> traces[1].new_iterations +6-element Vector{Int64}: + 1 + 2 + 3 + 1 + 2 + 3 +``` + + +""" +function train!(model, controls...; verbosity::Int=1, compact=false, traces=[]) isempty(controls) && throw(ERR_TRAIN) + traces == [] || iscompatible(traces, controls) || throw(ERR_INCOMPATIBLE_TRACES) + control = composite(controls...) # before training: @@ -20,13 +141,24 @@ function train!(model, controls...; verbosity::Int=1) # first training event: n = 1 # counts control cycles state = update!(control, model, verbosity, n) + first_traces = broadcast(ControlTrace, flat(control), flat(state)) + if isempty(traces) + append!(traces, first_traces) + else + broadcast(reset!, traces, controls) + if compact + broadcast(update!, traces, flat(state)) + else + broadcast(push!, traces, flat(state)) + end + end finished = done(control, state) # checks that model supports control: - if needs_loss(control) && loss(model) === nothing + if needs_loss(control) && isnothing(loss(model)) throw(ERR_NEEDS_LOSS) end - if needs_training_losses(control) && training_losses(model) === nothing + if needs_training_losses(control) && isnothing(training_losses(model)) throw(ERR_NEEDS_TRAINING_LOSSES) end @@ -34,6 +166,11 @@ function train!(model, controls...; verbosity::Int=1) while !finished n += 1 state = update!(control, model, verbosity, n, state) + if compact + broadcast(update!, traces, flat(state)) + else + broadcast(push!, traces, flat(state)) + end finished = done(control, state) end @@ -44,9 +181,12 @@ function train!(model, controls...; verbosity::Int=1) _loss isa Nothing || @info "final loss: $_loss" _training_losses isa Nothing || isempty(_training_losses) || @info "final training loss: $(_training_losses[end])" - verbosity > 1 && @info "total control cycles: $n" + verbosity > 1 && @info "total control cycles (steps): $n" end # finalization: - return takedown(control, verbosity, state) + broadcast(takedown!, traces, Ref(verbosity), flat(state)) + return collect(traces) end + + diff --git a/src/wrapped_controls.jl b/src/wrapped_controls.jl index 86d2e75..5515128 100644 --- a/src/wrapped_controls.jl +++ b/src/wrapped_controls.jl @@ -8,9 +8,8 @@ end """ IterationControl.louder(control, by=1) -Wrap `control` to make in more (or less) verbose. The same as -`control`, but as if the global `verbosity` were increased by the value -`by`. +Wrap `control` to make in more (or less) verbose. The same as `control`, but as if the +global `verbosity` were increased by the value `by`. """ louder(c; by=1) = Louder(c, by) @@ -21,12 +20,13 @@ update!(d::Louder, model, verbosity, args...) = update!(d.control, model, verbosity + d.by, args...) takedown(d::Louder, verbosity, state) = takedown(d.control, verbosity + d.by, state) - +public_state(d::Louder, state) = public_state(d.control, state) # # WithStateDo # helper: -shout(control, state) = @info "$(typeof(control)) state: $(flat(state))" +shout(control, public_state) = + (@info "`$(typeof(control))` state: $(public_state)"; public_state) struct WithStateDo{C,F} control::C @@ -34,17 +34,13 @@ struct WithStateDo{C,F} end """ - IterationControl.with_state_do(control, - f=x->@info "\$(typeof(control)) state: \$x") - -Wrap `control` to give access to it's internal state. Acts exactly -like `control` except that `f` is called on the internal state of -`control`. If `f` is not specified, the control type and state are -logged to `Info` at every update (useful for debugging new controls). + IterationControl.with_state_do(control,f=x->(@info "`\$(typeof(control))` state: \$x; x)") -**Warning.** The internal state of a control is not yet considered -part of the public interface and could change between in any pre 1.0 -release of IterationControl.jl. +Wrap `control` to give access to the public form of its internal state (which is also the +object accumulated by the corresponding [`ControlTrace`](@ref) returned by +[`IterationControl.train!`](@ref)). Acts exactly like `control` except that `f` is called +on current state. If `f` is not specified, the control type and state are logged to `Info` +at every update. Useful for debugging custom controls. """ with_state_do(c; f=state->shout(c, state)) = WithStateDo(c, f) @@ -55,20 +51,27 @@ for op in [:done, :takedown] end @inline function update!(d::WithStateDo, args...) state = update!(d.control, args...) - d.f(state) + d.f(public_state(d.control, state)) return state end +public_state(d::WithStateDo, state) = public_state(d.control, state) + # ## Skip -struct Skip{C,F<:Function} +struct Skip{C,F} control::C predicate::F + step::Int end -_pred(predicate) = predicate -_pred(predicate::Int) = t -> mod(t, predicate) == 0 +Base.show(io::IO, s::Skip) = s.step == 0 ? + print(io, "skip($(s.control), …)") : + print(io, "skip($(s.control), predicate=$(s.step))") +Base.show(io::IO, ::MIME"text/plain", s::Skip) = show(io, s) + +accept(predicate) = t -> mod(t, predicate) == 0 """ IterationControl.skip(control, predicate=1) @@ -78,12 +81,46 @@ An iteration control wrapper. If `predicate` is an **integer**, `k`: Apply `control` on every `k` calls to apply the wrapped control, starting with the `k`th call. -If `predicate` is a **function**: Apply `control` as usual when +If `predicate` is a **callable**: Apply `control` as usual when `predicate(n + 1)` is `true` but otherwise skip. Here `n` is the number of control cycles applied so far. +While the application of the underlying control gets skipped, the (unchanged) internal +state is nevertheless recorded to the corresponding [`ControlTrace`](@ref), but this trace +does not start recording until the first application of `control`: + +```julia-repl +using IterationControl +model = IterationControl.SquareRooter(4) + +julia> traces = IC.train!( + m, + Step(1), + IterationControl.skip(WithNumberDo(x->(@info(x); x)), predicate=2), + NumberLimit(5), +) +[ Info: 2 +[ Info: 4 +[ Info: final loss: 0.0 +[ Info: final training loss: 0.0 +[ Info: Stop triggered by NumberLimit(5) stopping criterion. +3-element Vector{Any}: + ControlTrace{<:Step}, ending (new_iterations = 5, new_steps = 5) + ControlTrace{<:Skip}, ending (number = 4, called_back = 4) + ControlTrace{<:NumberLimit}, ending (number = 5,) + +julia> traces[2].number +4-element Vector{Int64}: + 2 + 2 + 4 + 4 +``` """ -skip(control; predicate::Int=1) = Skip(control, _pred(predicate)) +function skip(control; predicate=1) + predicate isa Integer && return Skip(control, accept(predicate), predicate) + return Skip(control, predicate, 0) +end _state(s, model, verbosity, n, atomic_state...) = if s.predicate(n) new_atomic_state = @@ -114,11 +151,14 @@ function update!(s::Skip, model, verbosity, n, state) return state_candidate end +public_state(::Skip, ::NamedTuple{(:n,)}) = nothing +public_state(s::Skip, state) = public_state(s.control, state.atomic_state) + done(s::Skip, state) = done(s.control, state.atomic_state) -# can't be done if atomic state never intialized: +# can't be done if atomic state never initialized: done(s::Skip, state::NamedTuple{(:n,)}) = false takedown(s::Skip, verbosity, state) = takedown(s.control, verbosity, state.atomic_state) -takedown(::Skip, ::Any, ::NamedTuple{(:n,)}) = NamedTuple() +takedown(::Skip, ::Any, ::NamedTuple{(:n,)}) = "" diff --git a/test/TablesExt.jl b/test/TablesExt.jl new file mode 100644 index 0000000..8edeb18 --- /dev/null +++ b/test/TablesExt.jl @@ -0,0 +1,15 @@ +using Tables + +import IterationControl as IC +import IterationControl.SquareRooter + +@testset "Tables.jl interface for ControlTrace" begin + model = SquareRooter(16) + traces = IC.train!(model, Step(), NumberLimit(2), verbosity=-1) + @test Tables.rowtable(traces[1]) == [ + (new_iterations = 1, new_steps = 1), + (new_iterations = 2, new_steps = 2), + ] +end + +true diff --git a/test/_models_for_testing.jl b/test/_models_for_testing.jl index 719c77e..54ae512 100644 --- a/test/_models_for_testing.jl +++ b/test/_models_for_testing.jl @@ -2,6 +2,9 @@ # ## PARTICLE TRACKER (without methods lifted) +# The methods get lifted in /test/api.jl, after testing error messages for trying to +# proceed without the lifts. + # Consider an object that tracks a particle in one dimension, moving, # in discrete time, at a speed proportional to the distance away from # some moving target. The particle is initially at rest at the origin, @@ -55,8 +58,3 @@ end # ingest!(model, -0.9) # train!(model, 1) # @assert loss(model) ≈ 0.9 - -# model = Particle() -# ingest!(model, 1) -# train!(model, 2) -# @assert training_losses(model) ≈ [0.9, 0.81] diff --git a/test/api.jl b/test/api.jl index aea6ca6..0ad9be8 100644 --- a/test/api.jl +++ b/test/api.jl @@ -7,7 +7,7 @@ step = Step(1) @test_throws IC.err_train(model) IterationControl.train!(model, 1) # lifting train!: -IC.train!(model::Particle, n) = train!(model, n) +IC.train!(model::Particle, n::Int) = train!(model, n) @test_throws(IC.ERR_NEEDS_LOSS, IC.train!(model, step, withloss, NumberLimit(1))) diff --git a/test/composite_controls.jl b/test/composite_controls.jl index 5b2cf76..b7a79c4 100644 --- a/test/composite_controls.jl +++ b/test/composite_controls.jl @@ -33,9 +33,9 @@ end done_a = IC.done(a, state_a2) done_b = IC.done(b, state_b2) done_c = IC.done(c, state_c2) - report_a = IC.takedown(a, 0, state_a2) - report_b = IC.takedown(b, 0, state_b2) - report_c = IC.takedown(c, 0, state_c2) + message_a = IC.takedown(a, 0, state_a2) + message_b = IC.takedown(b, 0, state_b2) + message_c = IC.takedown(c, 0, state_c2) # composed: m = SquareRooter(4) @@ -45,8 +45,8 @@ end @test IC.flat(state_d2) == (state_a2, state_b2, state_c2) done_d = IC.done(d, state_d2) @test done_d == done_a || done_b || done_c - report_d = IC.takedown(d, 0, state_d2) - @test report_d == ((a, report_a), (b, report_b), (c, report_c)) + message_d = IC.takedown(d, 0, state_d2) + @test message_d == (message_a, message_b, message_c) end @testset "traits" begin diff --git a/test/controls.jl b/test/controls.jl index 8867e48..df64bf3 100644 --- a/test/controls.jl +++ b/test/controls.jl @@ -1,3 +1,8 @@ +using Dates +using IterationControl +import IterationControl as IC +import IterationControl.SquareRooter + @testset "Step" begin m = SquareRooter(4) IC.train!(m, 10) @@ -7,12 +12,13 @@ c = Step(n=2) state = IC.update!(c, m, 0, 1) @test state === (new_iterations = 2,) + @test IC.public_state(c, state) == (; new_iterations=2, new_steps=1) @test m.training_losses == all_training_losses[1:2] state = IC.update!(c, m, 0, 2, state) + @test IC.public_state(c, state) == (; new_iterations=4, new_steps=2) @test m.training_losses == all_training_losses[3:4] @test !IC.done(c, state) - @test_logs((:info, r"A total of "), - @test IC.takedown(c, 2, state) == (new_iterations = 4,)) + @test IC.takedown(c, 1, state) == "" end @testset "Info" begin @@ -22,10 +28,11 @@ end @test_logs (:info, 2.5) IC.update!(c, m, 2, 1) @test_logs (:info, 2.5) IC.update!(c, m, 1, 1) state = @test_logs IC.update!(c, m, 0, 1) - @test state === nothing + @test state === 2.5 + @test IC.public_state(c, state) == (; called_back=2.5) @test_logs (:info, 2.5) IC.update!(c, m, 2, 2, state) @test !IC.done(c, state) - @test IC.takedown(c, 10, state) == NamedTuple() + @test IC.takedown(c, 10, state) == "" end @testset "Warn" begin @@ -43,6 +50,7 @@ end @test_logs IC.update!(c, m, 0, 1) state = @test_logs IC.update!(c, m, -1, 1) @test state === (warnings=(),) + @test IC.public_state(c, state) == (;) m = SquareRooter(4) IC.train!(m, 1) @@ -51,12 +59,14 @@ end @test_logs (:warn, "") IC.update!(c, m, 0, 2, state) state = @test_logs IC.update!(c, m, -1, 2, state) @test state === (warnings=("", ""),) + @test IC.public_state(c, state) == (;) IC.train!(m, 1) @test_logs IC.update!(c, m, 1, 3, state) @test_logs IC.update!(c, m, 0, 3, state) state = @test_logs IC.update!(c, m, -1, 3, state) @test state === (warnings=("", ""),) + @test IC.public_state(c, state) == (;) m = SquareRooter(4) c = Warn(m -> m.root > 2.4, f = m->m.root) @@ -66,246 +76,219 @@ end @test_logs (:warn, 2.5) IC.update!(c, m, 0, 1) state = @test_logs IC.update!(c, m, -1, 1) @test state === (warnings=(2.5, ),) + @test IC.public_state(c, state) == (;) @test_logs (:warn, 2.5) IC.update!(c, m, 1, 2, state) @test_logs (:warn, 2.5) IC.update!(c, m, 0, 2, state) state = @test_logs IC.update!(c, m, -1, 2, state) @test state === (warnings=(2.5, 2.5),) + @test IC.public_state(c, state) == (;) @test !IC.done(c, state) - @test_logs((:warn, r"A `Warn`"), - @test IC.takedown(c, 2, state) == (warnings = (2.5, 2.5),)) - @test_logs @test IC.takedown(c, 1, state) == (warnings = (2.5, 2.5),) - + @test_logs( + (:warn, r"A `Warn`"), + (@test IC.takedown(c, 1, state) == "A `Warn` control issued 2 warnings."), + ) end @testset "Error" begin m = SquareRooter(4) c = Error(m -> m.root > 2.4) + # error encountered after first train: IC.train!(m, 1) - state = @test_logs (:error, "") IC.update!(c, m, 2, 1) + state = @test_logs (:error, "") IC.update!(c, m, 0, 1) @test state === (done=true, error="") + @test IC.public_state(c, state) == (; error="") + @test IC.done(c, state) + + # start again, with a model that won't throw errors after training: IC.train!(m, 1) state = @test_logs IC.update!(c, m, 2, 1) - @test state === (done=false, error=()) + @test state === (done=false, error=nothing) + @test IC.public_state(c, state) == (; error=nothing) m = SquareRooter(4) IC.train!(m, 1) - state = @test_logs (:error, "") IC.update!(c, m, 2, 1) - state = @test_logs (:error, "") IC.update!(c, m, 2, 2, state) + state = @test_logs (:error, "") IC.update!(c, m, 0, 1) + state = @test_logs (:error, "") IC.update!(c, m, 0, 2, state) @test state === (done=true, error="") + @test IC.public_state(c, state) == (; error="") m = SquareRooter(4) c = Error(m -> m.root > 2.4, f = m->m.root) IC.train!(m, 1) - state = @test_logs (:error, 2.5) IC.update!(c, m, 2, 1) + state = @test_logs (:error, 2.5) IC.update!(c, m, 0, 1) @test state === (done=true, error=2.5) + @test IC.public_state(c, state) == (; error=2.5) IC.train!(m, 1) - state = @test_logs IC.update!(c, m, 2, 1) - @test state === (done=false, error=()) + state = @test_logs IC.update!(c, m, 0, 1) + @test state === (done=false, error=nothing) + @test IC.public_state(c, state) == (; error=nothing) m = SquareRooter(4) IC.train!(m, 1) - state = @test_logs (:error, 2.5) IC.update!(c, m, 2, 1) - state = @test_logs (:error, 2.5) IC.update!(c, m, 2, 2, state) + state = @test_logs (:error, 2.5) IC.update!(c, m, 0, 1) + state = @test_logs (:error, 2.5) IC.update!(c, m, 0, 2, state) @test state === (done=true, error=2.5) @test IC.done(c, state) - @test IC.takedown(c, 10, state) == state + @test IC.takedown(c, 10, state) == "2.5" end @testset "Callback" begin - v = Float64[] - f(model) = (push!(v, IC.loss(model)); last(v) < 0.02) + function f(model) + push!(v, IC.loss(model)) + return last(v) < 0.02 + end c = Callback(f) m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test IC.public_state(c, state) == (; called_back=false) + @test !IC.done(c, state) @test v == [2.25, ] IC.train!(m, 2) state = IC.update!(c, m, 1, 2, state) - @test !state.done + @test IC.public_state(c, state) == (; called_back=true) + @test !IC.done(c, state) # because `c.stop_if_true==false` @test v ≈ [2.25, (3281/1640)^2 - 4] - @test IC.takedown(c, 0, state) == (done = false, log="") + @test IC.takedown(c, 0, state) == "" v = Float64[] - f2(model) = (push!(v, IC.loss(model)); last(v) < 0.02) - - c = Callback(f2, stop_if_true=true) + c = Callback(f, stop_if_true=true) m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test !IC.done(c, state) @test v == [2.25, ] IC.train!(m, 2) state = IC.update!(c, m, 1, 2, state) - @test state.done + @test IC.done(c, state) @test v ≈ [2.25, (3281/1640)^2 - 4] - @test IC.takedown(c, 0, state) == - (done = true, - log="Stop triggered by a `Callback` control. ") + @test_logs( + (:info, r"^Stop triggered by a `Callback"), + (@test IC.takedown(c, 1, state) == + "Stop triggered by a `Callback` control. "), + ) v = Float64[] - f3(model) = (push!(v, IC.loss(model)); last(v) < 0.02) - - c = Callback(f3, stop_if_true=true, stop_message="foo") + c = Callback(f, stop_if_true=true, stop_message="foo") m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test !IC.done(c, state) + @test IC.public_state(c, state) == (; called_back=false) @test v == [2.25, ] IC.train!(m, 2) state = IC.update!(c, m, 1, 2, state) - @test state.done + @test IC.done(c, state) @test v ≈ [2.25, (3281/1640)^2 - 4] - @test IC.takedown(c, 0, state) == - (done = true, - log="foo") - + @test_logs( + (:info, "foo"), + (@test IC.takedown(c, 1, state) == "foo"), + ) end @testset "WithLossDo" begin - v = Float64[] f(loss) = (push!(v, loss); last(v) < 0.02) c = WithLossDo(f) m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test IC.public_state(c, state) == + (loss=2.25, called_back=false) + @test !IC.done(c, state) @test v == [2.25, ] IC.train!(m, 2) state = IC.update!(c, m, 1, 2, state) - @test !state.done + @test !IC.done(c, state) # because `stop_if_true=false` + @test IC.public_state(c, state)[2] @test v ≈ [2.25, (3281/1640)^2 - 4] - @test IC.takedown(c, 1, state) == - (loss=v[end], done = false, log="") - @test_logs((:info, r"final loss"), - @test IC.takedown(c, 2, state) == - (loss=v[end], done = false, log="")) + @test IC.takedown(c, 1, state) == "" v = Float64[] - f2(loss) = (push!(v, loss); last(v) < 0.02) - c = WithLossDo(f2, stop_if_true=true) + c = WithLossDo(f, stop_if_true=true) m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test !IC.done(c, state) @test v == [2.25, ] IC.train!(m, 2) state = IC.update!(c, m, 1, 2, state) - @test state.done + @test IC.done(c, state) @test v ≈ [2.25, (3281/1640)^2 - 4] - @test IC.takedown(c, 0, state) == - (loss = v[end], - done = true, - log="Stop triggered by a `WithLossDo` control. ") - @test_logs((:info, r"Stop triggered"), - @test IC.takedown(c, 1, state) == - (loss = v[end], - done = true, - log="Stop triggered by a `WithLossDo` control. ")) - @test_logs((:info, r"final loss"), - (:info, r"Stop triggered"), - @test IC.takedown(c, 2, state) == - (loss = v[end], - done = true, - log="Stop triggered by a `WithLossDo` control. ")) + @test_logs( + (:info, r"^Stop triggered by a `WithLossDo"), + (@test IC.takedown(c, 1, state) == + "Stop triggered by a `WithLossDo` control. "), + ) v = Float64[] - f3(loss) = (push!(v, loss); last(v) < 0.02) - c = WithLossDo(f3, stop_if_true=true, stop_message="foo") + c = WithLossDo(f, stop_if_true=true, stop_message="foo") m = SquareRooter(4) - IC.train!(m, 1) - state = IC.update!(c, m, 1, 1) - @test !state.done - @test v == [2.25, ] - IC.train!(m, 2) + IC.train!(m, 3) state = IC.update!(c, m, 1, 2, state) - @test state.done - @test v ≈ [2.25, (3281/1640)^2 - 4] - @test IC.takedown(c, 0, state) == - (loss = v[end], - done = true, - log="foo") - + @test IC.done(c, state) + @test_logs( + (:info, r"foo"), + (@test IC.takedown(c, 1, state) == "foo"), + ) end @testset "WithTrainingLossesDo" begin - v = Float64[] f(training_loss) = (push!(v, last(training_loss)); last(v) < 0.5) c = WithTrainingLossesDo(f) m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done - @test v ≈ [1.5, ] + @test IC.public_state(c, state) == + (training_loss=1.5, called_back=false) + @test !IC.done(c, state) + @test v == [1.5, ] IC.train!(m, 1) state = IC.update!(c, m, 1, 2, state) - @test !state.done + @test !IC.done(c, state) # because `stop_if_true=false` + @test IC.public_state(c, state)[2] @test v ≈ [1.5, 0.45] - @test IC.takedown(c, 0, state) == - (latest_training_loss = v[end], done = false, log="") - @test_logs((:info, r"final train"), - @test IC.takedown(c, 2, state) == - (latest_training_loss=v[end], done = false, log="")) - + @test IC.takedown(c, 1, state) == "" v = Float64[] - f1(training_loss) = (push!(v, last(training_loss)); last(v) < 0.5) - c = WithTrainingLossesDo(f1, stop_if_true=true) + c = WithTrainingLossesDo(f, stop_if_true=true) m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test !IC.done(c, state) @test v == [1.5, ] IC.train!(m, 1) state = IC.update!(c, m, 1, 2, state) - @test state.done + @test IC.done(c, state) @test v ≈ [1.5, 0.45] - @test IC.takedown(c, 0, state) == - (latest_training_loss = v[end], - done = true, - log="Stop triggered by a `WithTrainingLossesDo` control. ") - @test_logs((:info, r"Stop "), - @test IC.takedown(c, 1, state) == - (latest_training_loss = v[end], - done = true, - log="Stop triggered by a `WithTrainingLossesDo` control. ")) - @test_logs((:info, r"final train"), - (:info, r"Stop"), - @test IC.takedown(c, 2, state) == - (latest_training_loss = v[end], - done = true, - log="Stop triggered by a `WithTrainingLossesDo` control. ")) + @test_logs( + (:info, r"^Stop triggered by a `WithTrainingLossesDo"), + (@test IC.takedown(c, 1, state) == + "Stop triggered by a `WithTrainingLossesDo` control. "), + ) v = Float64[] - f2(training_loss) = (push!(v, last(training_loss)); last(v) < 0.5) - c = WithTrainingLossesDo(f2, stop_if_true=true, stop_message="foo") + c = WithTrainingLossesDo(f, stop_if_true=true, stop_message="foo") m = SquareRooter(4) - IC.train!(m, 1) - state = IC.update!(c, m, 1, 1) - @test !state.done - @test v == [1.5, ] - IC.train!(m, 1) + IC.train!(m, 2) state = IC.update!(c, m, 1, 2, state) - @test state.done - @test v ≈ [1.5, 0.45] - @test IC.takedown(c, 0, state) == - (latest_training_loss = v[end], - done = true, - log="foo") + @test IC.done(c, state) + @test_logs( + (:info, r"foo"), + (@test IC.takedown(c, 1, state) == "foo"), + ) end - @testset "WithNumberDo" begin v = Int[] f(n) = (push!(v, n); last(n) > 1) @@ -313,60 +296,49 @@ end m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test IC.public_state(c, state) == (; number=1, called_back=false) + @test !IC.done(c, state) @test v == [1, ] IC.train!(m, 1) state = IC.update!(c, m, 1, 2, state) - @test !state.done + @test IC.public_state(c, state) == (; number=2, called_back=true) + @test !IC.done(c, state) # because `stop_if_true` is `false` @test v == [1, 2] - @test IC.takedown(c, 0, state) == (done = false, n = 2, log="") - @test_logs((:info, r"final number"), - @test IC.takedown(c, 2, state) == (done = false, n = 2, log="")) + @test_logs @test IC.takedown(c, 0, state) == "" v = Int[] - f2(n) = (push!(v, n); last(n) > 1) - c = WithNumberDo(f2, stop_if_true=true) + c = WithNumberDo(f, stop_if_true=true) m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test !IC.done(c, state) @test v == [1, ] IC.train!(m, 1) state = IC.update!(c, m, 1, 2, state) - @test state.done + @test IC.done(c, state) + @test IC.public_state(c, state) == (; number=2, called_back=true) @test v == [1, 2] - @test IC.takedown(c, 0, state) == - (done = true, - n= 2, - log="Stop triggered by a `WithNumberDo` control. ") - @test_logs((:info, r"Stop"), - @test IC.takedown(c, 1, state) == - (done = true, - n= 2, - log="Stop triggered by a `WithNumberDo` control. ")) - @test_logs((:info, r"final number"), - (:info, r"Stop"), - @test IC.takedown(c, 2, state) == - (done = true, - n= 2, - log="Stop triggered by a `WithNumberDo` control. ")) + @test_logs( + (:info, r"^Stop"), + (@test IC.takedown(c, 1, state) == + "Stop triggered by a `WithNumberDo` control. "), + ) v = Int[] - f3(n) = (push!(v, n); last(n) > 1) - c = WithNumberDo(f3, stop_if_true=true, stop_message="foo") + c = WithNumberDo(f, stop_if_true=true, stop_message="foo") m = SquareRooter(4) IC.train!(m, 1) state = IC.update!(c, m, 1, 1) - @test !state.done + @test !IC.done(c, state) @test v == [1, ] IC.train!(m, 1) state = IC.update!(c, m, 1, 2, state) - @test state.done + @test IC.done(c, state) @test v == [1, 2] - @test IC.takedown(c, 0, state) == - (done = true, - n = 2, - log="foo") + @test_logs( + (:info, r"foo"), + (@test IC.takedown(c, 1, state) == "foo"), + ) end @testset "Data" begin @@ -379,6 +351,7 @@ end state = IC.update!(c, model, 0, 1) IC.train!(model, 1) @test loss(model) ≈ 0.9 + @test IC.public_state(c, state) == (;) state = IC.update!(c, model, 0, 2, state) IC.train!(model, 1) @@ -393,53 +366,62 @@ end if option state = IC.update!(c, model, 0, 4, state) @test IC.done(c, state) - report = @test_logs (:info, IC.DATA_STOP) IC.takedown(c, 1, state) - @test report == (done = true, log = IC.DATA_STOP) + message = @test_logs (:info, IC.DATA_STOP) IC.takedown(c, 1, state) + @test message == IC.DATA_STOP else state = @test_logs IC.update!(c, model, 0, 4, state) @test !IC.done(c, state) - report = IC.takedown(c, 1, state) - @test report == (done = false, log = "") + message = @test_logs IC.takedown(c, 1, state) + @test message == "" end end end @testset "integration test" begin - data = repeat([-1, 1], outer=5); - + vector = repeat([-1, 1], outer=5); model = Particle(0.1) - losses = Float64[] - callback!(model) = push!(losses, model.position) - report = IC.train!(model, - Data(data), - Step(5), - Threshold(0.01), - TimeLimit(0.0005), - Info(loss), - Callback(callback!); - verbosity=-1) - @test !report[1][2].done - @test report[3][2].done + positions = Float64[] + callback!(model) = (push!(positions, model.position); model.position) + data, step, threshold, timelimit, info, callback = IC.train!( + model, + Data(vector), + Step(5), + Threshold(0.01), + TimeLimit(0.0005), + Info(IC.loss), + Callback(callback!); + verbosity=-1, + ) + @test data.message == "" + @test !data.triggered_stop + @test step.new_steps isa Vector{Int} + @test threshold.triggered_stop + @test timelimit.table.time[end] < now() + @test info.called_back isa Vector{Float64} + @test callback.called_back isa Vector{Float64} @test loss(model) < 0.01 model = Particle(0.1) - losses = Float64[] - noise = fill((:info, r""), 33) - report = @test_logs(noise..., - IC.train!(model, - Data(data, stop_when_exhausted=true), - Step(5), - WithNumberDo(), - WithLossDo(), - WithTrainingLossesDo(), - Threshold(0.01), - TimeLimit(0.0005), - Info(loss), - Callback(callback!), - verbosity=-1)) - @test length(losses) == length(data) + 1 - @test loss(model) > 0.01 + positions = Float64[] + # note: after data is exhausted, one gets one more control cycle. + noise = fill((:info, ), length(vector) + 4) + @test_logs(noise..., + IC.train!( + model, + Data(vector, stop_when_exhausted=true), + Step(5), + WithNumberDo(), + WithLossDo(), + WithTrainingLossesDo(), + Threshold(0.01), + TimeLimit(0.0005), + Info(IC.loss), + Callback(callback!), + verbosity=1), + ) + @test length(positions) == length(vector) + 1 + @test IC.loss(model) > 0.01 end @testset "constructors #55" begin diff --git a/test/runtests.jl b/test/runtests.jl index 3047054..d44e85c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,39 +6,26 @@ const IC = IterationControl include("_models_for_testing.jl") const SquareRooter = IC.SquareRooter -@testset "square_rooter.jl" begin - include("square_rooter.jl") +test_files = [ + "square_rooter.jl", + "utilities.jl", + "api.jl", + "controls.jl", + "stopping_controls.jl", + "composite_controls.jl", + "wrapped_controls.jl", + "train.jl", + "TablesExt.jl", + "deprecated.jl", +] + +files = isempty(ARGS) ? test_files : ARGS + +for file in files + quote + @testset $file begin + include($file) + end + end |> eval end -@testset "utilities" begin - include("utilities.jl") -end - -# this test must happen before test of controls.jl: -@testset "api" begin - include("api.jl") -end - -@testset "controls" begin - include("controls.jl") -end - -@testset "stopping_controls" begin - include("stopping_controls.jl") -end - -@testset "composite_controls" begin - include("composite_controls.jl") -end - -@testset "wrapped_controls" begin - include("wrapped_controls.jl") -end - -@testset "train!" begin - include("train.jl") -end - -@testset "deprecated" begin - include("deprecated.jl") -end diff --git a/test/stopping_controls.jl b/test/stopping_controls.jl index 8024860..5e42861 100644 --- a/test/stopping_controls.jl +++ b/test/stopping_controls.jl @@ -8,14 +8,14 @@ IC.train!(m, 3) state = IC.update!(c, m, 0, 1) @test state == 1 + @test IC.public_state(c, state) == (; number=1) @test !IC.done(c, state) IC.train!(m, 3) state = IC.update!(c, m, 0, 2, state) @test state == 2 @test IC.done(c, state) - report = @test_logs (:info, r"NumberLimit\(2\)") IC.takedown(c, 1, state) - @test report.done - @test report.log == + message = @test_logs (:info, r"NumberLimit\(2\)") IC.takedown(c, 1, state) + @test message == "Stop triggered by NumberLimit(2) stopping criterion. " # A stopping criterion than uses training losses: @@ -29,6 +29,8 @@ IC.train!(m, 3) state = IC.update!(c, m, 0, 1) + keys(IC.public_state(c, state)) == + (:loss, :PQ, :progress, :latest_training_loss, :num_training_losses) train_losses = m.training_losses @test reverse(state.training_losses) == train_losses # length 3 @test !IC.done(c, state) @@ -39,7 +41,6 @@ @test reverse(state.training_losses) == train_losses @test !IC.done(c, state) - report = IC.takedown(c, 1, state) - @test !report.done - @test report.log == "" + message = @test_logs IC.takedown(c, 1, state) + @test message == "" end diff --git a/test/traces.jl b/test/traces.jl new file mode 100644 index 0000000..1e3f115 --- /dev/null +++ b/test/traces.jl @@ -0,0 +1,142 @@ +using IterationControl +import IterationControl:_repr, SquareRooter +import IterationControl as IC + +@testset "helper for compact represtenation of columns" begin + @test _repr("junk") == IterationControl.OMITTED + @test _repr([[]]) == IterationControl.OMITTED + @test _repr([1,2,3, 4]) == "[ …, 2, 3, 4]" + @test _repr([1,2,3]) == "[1, 2, 3]" + @test _repr([1,2]) == "[1, 2]" + @test _repr(typeof(1)) == "<:Int64" + @test _repr(typeof([1,])) == "<:Array" +end + + +# # GENERATE A TRACE + +model = SquareRooter(4) +control = Threshold(0.1) +verbosity = 0 + +# train for one iteration: +IterationControl.train!(model, 1) + +# initialize state and trace: +state = IterationControl.update!(control, model, verbosity, 1) +trace = IterationControl.ControlTrace(control, state) + +# train another iteration: +IterationControl.train!(model, 1) + +# update state and trace: +state = IterationControl.update!(control, model, verbosity, 1, state) +push!(trace, state) + +@testset "property access" begin + @test propertynames(trace) == (:loss, :excess, IterationControl.TRACE_FIELDS...) + @test trace.control == control + @test trace.loss == [2.25, 0.20249999999999968] + @test trace.excess == [2.15, 0.10249999999999967] + @test !isdefined(trace, :message) + @test !trace.triggered_stop + @test trace.table == (; loss=trace.loss, excess=trace.excess) +end + +@testset "display" begin + @test sprint(show, MIME("text/plain"), trace) == + """ + ControlTrace + control: Threshold(0.1) + loss: [2.25, 0.20249999999999968] + excess: [2.15, 0.10249999999999967] + table: (loss = …, excess = …) + triggered_stop: false + """ |> rstrip + @test sprint(show, trace) == + "ControlTrace{<:Threshold}, ending (loss = 0.20249999999999968, "* + "excess = 0.10249999999999967)" +end + +# one more train and update will trigger a stop: +IterationControl.train!(model, 1) +state = IterationControl.update!(control, model, verbosity, 1, state) +push!(trace, state) + +IC.takedown!(trace, verbosity, state) + +@testset "display" begin + @test sprint(show, MIME("text/plain"), trace) == + """ + ControlTrace + control: Threshold(0.1) + loss: [2.25, 0.20249999999999968, 0.002439396192741583] + excess: [2.15, 0.10249999999999967, -0.09756060380725842] + table: (loss = …, excess = …) + message: Stop triggered by Threshold(0.1) stopping criterion. + triggered_stop: true + """ |> rstrip + @test sprint(show, trace) == + "ControlTrace{<:Threshold}, ending (loss = 0.002439396192741583, "* + "excess = -0.09756060380725842)" +end + +@testset "reset!" begin + t = deepcopy(trace.table) + IC.reset!(trace, Threshold(10.0)) + @test trace.control == Threshold(10.0) + @test !trace.triggered_stop + @test trace.message == "" + @test trace.table == t +end + +@testset "delayed addition of trace table (skip control)" begin + model = SquareRooter(4) + control = IC.skip(NumberLimit(3), predicate=2) + verbosity = 0 + + # control cycle #1 + + # train one iteration: + IterationControl.train!(model, 1) + + # initialize state and trace: + state = IterationControl.update!(control, model, verbosity, 1) + trace = IterationControl.ControlTrace(control, state) + @test isempty(trace) # because of first skip + + # control cycle #2 + + # training iteration: + IterationControl.train!(model, 1) + + # update state and trace: + state = IterationControl.update!(control, model, verbosity, 2, state) + push!(trace, state) + @test trace.table.number == [1,] + + # control cycle #3 + + # train another iteration: + IterationControl.train!(model, 1) + + # update state and trace: + state = IterationControl.update!(control, model, verbosity, 3, state) + push!(trace, state) + @test trace.table.number == [1, 1] + + # control cycle #4 + + # train another iteration: + IterationControl.train!(model, 1) + + # update state and trace: + state = IterationControl.update!(control, model, verbosity, 4, state) + push!(trace, state) + @test trace.table.number == [1, 1, 2] + + IC.takedown!(trace, verbosity, state) + @test !trace.triggered_stop +end + +true diff --git a/test/train.jl b/test/train.jl index 43cac34..ee0c4a5 100644 --- a/test/train.jl +++ b/test/train.jl @@ -1,14 +1,24 @@ +using IterationControl +import IterationControl as IC + @testset "basic integration" begin - m = SquareRooter(4) - report = IC.train!(m, Step(2), InvalidValue(), NumberLimit(3); verbosity=0); - @test report[1] == (Step(2), (new_iterations = 6,)) - @test report[2] == (InvalidValue(), (done=false, log="")) - report[3] == (NumberLimit(3), - (done=true, - log="Stop triggered by NumberLimit(3) "* - "stopping criterion. ")) - - m = SquareRooter(4) + m = IC.SquareRooter(4) + controls = [Step(2), InvalidValue(), NumberLimit(3)] + traces = [] + trace1, trace2, trace3 = + IC.train!(m, controls...; verbosity=0, traces); + + @test traces == Any[trace1, trace2, trace3] + @test trace1.table == (new_iterations = [2, 4, 6], new_steps = [1, 2, 3]) + @test !trace1.triggered_stop + + @test trace2.table == (; encountered = [nothing, nothing, nothing]) + @test !trace2.triggered_stop + + @test trace3.table == (; number = [1, 2, 3]) + @test trace3.triggered_stop + + m = IC.SquareRooter(4) @test_logs((:info, r"final loss"), (:info, r"final training loss"), (:info, r"Stop triggered by Num"), @@ -19,73 +29,114 @@ (:info, r"Stepping model for 2 more iterations"), (:info, r"final loss"), (:info, r"final training loss"), - (:info, r"total control cycles\: 3"), - (:info, r"A total of 6 iterations added"), + (:info, r"total control cycles \(steps\): 3"), (:info, r"Stop triggered by NumberLimit"), IC.train!(m, Step(2), InvalidValue(), NumberLimit(3); - verbosity=2)); + verbosity=2), + ); + + # compact traces: + m = IC.SquareRooter(4) + controls = [Step(2), InvalidValue(), NumberLimit(3)] + trace1, trace2, trace3 = + IC.train!(m, controls...; verbosity=0, compact=true); + @test trace1.new_iterations == [6,] + @test trace2.encountered == [nothing,] + @test trace3.number == [3,] + + controls[2] = Threshold() + @test_throws IC.ERR_INCOMPATIBLE_TRACES IC.train!(m, controls; traces) +end + +@testset "`traces` as kwarg" begin + model = IterationControl.SquareRooter(4) + controls = [Step(1), Threshold(0.01), Callback(_->sleep(0.1))] + traces = [] + Threads.@spawn IterationControl.train!(model, controls...; verbosity=0, traces=traces) + @test length(traces) == 0 + sleep(1) + @test length(traces) == 3 + losses = deepcopy(traces[2].loss) + @test length(losses) == 3 + @test losses[end] < 0.01 && losses[end] > 0.001 + + # change a control and continue on with `traces=traces`: + controls[2] = Threshold(1e-100) + IterationControl.train!(model, controls...; verbosity=0, traces=traces) + newlosses = traces[2].loss + @test newlosses[1:3] == losses + @test newlosses[end] < 1e-100 + @test traces[1].new_iterations == [1, 2, 3, 1, 2, 3] end struct WrappedSquareRooter - s::SquareRooter + s::IC.SquareRooter end +IC.train!(w::WrappedSquareRooter, n::Int) = IC.train!(w.s, n) +IC.loss(w::WrappedSquareRooter) = IC.loss(w.s) +IC.expose(w::WrappedSquareRooter) = w.s + @testset "expose and Callback(..., raw=true)" begin - IC.train!(w::WrappedSquareRooter, n) = IC.train!(w.s, n) - IC.loss(w::WrappedSquareRooter) = IC.loss(w.s) - IC.train!(w::WrappedSquareRooter, n) = IC.train!(w.s, n) - IC.expose(w::WrappedSquareRooter) = w.s - wrapper = WrappedSquareRooter(SquareRooter(4)) + wrapper = WrappedSquareRooter(IC.SquareRooter(4)) roots = Any[] wrappers = Any[] - g(w) = push!(wrappers, deepcopy(w)) + function g(w) + clone = deepcopy(w) + push!(wrappers, clone) + return clone + end - c1 = Callback(s->push!(roots, s.root)) + c1 = Callback(s->(push!(roots, s.root); s.root)) c2 = Callback(g, raw=true) - IC.train!(wrapper, Step(1), c1, c2, NumberLimit(2), verbosity=0) + traces = IC.train!(wrapper, Step(1), c1, c2, NumberLimit(2), verbosity=0) @test map(w -> w.s.root, wrappers) ≈ roots + @test traces[2].called_back == [2.5, 2.05] end @testset "skip integration" begin - m = SquareRooter(4) - numbers = Int[] - IC.train!(m, - Step(1), - IterationControl.skip( - WithNumberDo(x->push!(numbers, x)), predicate=3), - NumberLimit(10), verbosity=0) - @test numbers == [3, 6, 9] + m = IC.SquareRooter(4) + traces = IC.train!( + m, + Step(1), + IterationControl.skip(WithNumberDo(), predicate=3), + NumberLimit(10), + verbosity=0, + ) + @test traces[2].number == [3, 3, 3, 6, 6, 6, 9, 9] end -@testset "integration test related to #38" begin +@testset "integration test for skip" begin model = IterationControl.SquareRooter(4) - @test_logs((:info, r"number"), - (:info, r"number"), - (:info, r"final loss"), - (:info, r"final training loss"), - (:info, r"Stop triggered by"), - IC.train!(model, - Step(1), - Threshold(2.1), - WithNumberDo(), - IterationControl.skip(WithLossDo(), predicate=3))) + @test_logs( + (:info, r"final loss"), + (:info, r"final training loss"), + (:info, r"Stop triggered by"), + IC.train!( + model, + Step(1), + Threshold(2.1), + WithNumberDo(), + IterationControl.skip(WithLossDo(), predicate=3), + ), + ) end @testset "integration test with PQ" begin # get complete training losses: - model = SquareRooter(1e40) + model = IC.SquareRooter(1e40) IC.train!(model, 70) tlosses = IC.training_losses(model) # get complete losses: - model = SquareRooter(1e40) + model = IC.SquareRooter(1e40) losses = map(1:70) do _ IC.train!(model, 1) IC.loss(model) @@ -93,28 +144,28 @@ end # train with PQ criterion: K = 5 - model = SquareRooter(1e40) + model = IC.SquareRooter(1e40) pq = PQ(k=K) - states = Any[] - f(s) = push!(states, s) - controls = [Step(1), NumberLimit(70), IC.with_state_do(pq, f=f)] - IC.train!(model, controls..., verbosity=0) + controls = [Step(1), NumberLimit(70), PQ(k=K)] + _, _, trace = IC.train!(model, controls..., verbosity=0) # check internal state of PQ makes sense, as far as `loss` and # `training_losses` are concerned: - @test map(s->s.loss, states) == losses - @test map(enumerate(states)) do (j, s) + loss0, _, _, training_loss, num_training_losses = trace.table + @test loss0 == losses + @test map(eachindex(loss0)) do j L = min(K, j) # what length of PQ training_losses should be - t= s.loss == losses[j] && - s.training_losses == reverse(tlosses[j-L+1:j]) + num_training_losses[j] == L && + loss0[j] == losses[j] && + training_loss[j] == tlosses[j] end |> all end # https://github.com/JuliaAI/MLJIteration.jl/issues/36 @testset "integration test related to MLJIteration.jl #36" begin - model = SquareRooter(NaN) - report = + model = IC.SquareRooter(NaN) + step_trace, invalid_trace, _ = IC.train!(model, Step(1), InvalidValue(), NumberLimit(3), verbosity=0) - @test report[1][2].new_iterations == 1 - @test occursin("Stopping", report[2][2].log) + @test step_trace.new_iterations == [1,] + @test occursin("Stopping", invalid_trace.message) end diff --git a/test/wrapped_controls.jl b/test/wrapped_controls.jl index bc168a8..a018024 100644 --- a/test/wrapped_controls.jl +++ b/test/wrapped_controls.jl @@ -1,3 +1,7 @@ +using IterationControl +import IterationControl as IC +import IterationControl.SquareRooter + @testset "louder" begin m = SquareRooter(4) control = Warn(_->true, f="42") @@ -7,21 +11,37 @@ state = @test_logs IC.update!(c, m, -1, 1) IC.train!(m, 2) state = @test_logs (:warn, r"42") IC.update!(c, m, 0, 2, state) - @test_logs IC.takedown(c, 1, state) - @test_logs (:warn, r"A ") IC.takedown(c, 2, state) + @test_logs( + (:warn, r"A "), + (@test IC.takedown(c, 1, state) == + "A `Warn` control issued 2 warnings."), + ) + @test_logs( + (:warn, r"A "), + (@test IC.takedown(c, 0, state) == + "A `Warn` control issued 2 warnings."), + ) + @test_logs( + (@test IC.takedown(c, -1, state) == + "A `Warn` control issued 2 warnings."), + ) c = IC.louder(control, by=1) IC.train!(m, 1) state = @test_logs IC.update!(c, m, -2, 1) IC.train!(m, 2) state = @test_logs (:warn, r"42") IC.update!(c, m, -1, 2, state) - @test_logs IC.takedown(c, 0, state) - @test_logs (:warn, r"A ") IC.takedown(c, 1, state) + @test_logs( + (:warn, r"A "), + (@test IC.takedown(c, -1, state) == + "A `Warn` control issued 2 warnings."), + ) + @test_logs IC.takedown(c, -2, state) end @testset "with_state_do" begin m = SquareRooter(4) - test_controls = [Step(2), InvalidValue(), GL(), Callback()] + test_controls = [Step(2), InvalidValue(), Callback()] _info = fill((:info, r""), 2*length(test_controls)) @test_logs(_info..., @@ -31,40 +51,53 @@ end @test state == IC.update!(d, m, 1, 1) state_c = IC.update!(c, m, 0, 2, state) state_d = IC.update!(d, m, 0, 2, state) - @test state_c == state_c + @test state_c == state_d @test IC.done(c, state_c) == IC.done(d, state_d) end) # integration test: m = SquareRooter(4) v = Any[] - f(state) = push!(v, state.new_iterations) - IC.train!(m, IC.with_state_do(Step(2), f=f), NumberLimit(10)) - @test v == 2:2:20 + f(public_state) = push!(v, public_state) + step, numberlimit = + IC.train!(m, IC.with_state_do(Step(2), f=f), NumberLimit(10), verbosity=0) + @test v[end] == (new_iterations=20, new_steps=10) + @test length(v) == 10 end @testset "skip" begin + + # constructor and display: + s = IC.skip(Threshold(0.1), predicate=2) + @test s.predicate(2) + @test !s.predicate(3) + @test sprint(show, s) == "skip(Threshold(0.1), predicate=2)" + s = IC.skip(Threshold(0.1), predicate=IC.accept(2)) + @test s.predicate(2) + @test !s.predicate(3) + @test sprint(show, s) == "skip(Threshold(0.1), …)" + @test sprint(show, MIME("text/plain"), s) == "skip(Threshold(0.1), …)" + m = SquareRooter(4) - test_controls = [Step(2), InvalidValue(), GL(), Callback()] - _info = fill((:info, r""), 2) + test_controls = [Step(2), InvalidValue(), Callback(), Threshold(0.1)] - @test_logs(_info..., - for c in test_controls - s = IC.skip(c, predicate=2) - @test !s.predicate(1) - @test s.predicate(2) - @test !s.predicate(3) - atomic_state = IC.update!(c, m, 1, 1) - state = IC.update!(s, m, 1, 1) - @test state == (n = 1, ) - state = IC.update!(s, m, 1, 2, state) - @test state == (atomic_state = atomic_state, n = 2) - state = IC.update!(s, m, 1, 3, state) - @test state == (atomic_state = atomic_state, n = 3) - atomic_state = IC.update!(c, m, 1, 2, atomic_state) - state = IC.update!(s, m, 1, 4, state) - @test state == (atomic_state = atomic_state, n = 4) - @test IC.done(c, atomic_state) == IC.done(s, state) - @test IC.takedown(c, 0, atomic_state) == IC.takedown(s, 0, state) - end) + for c in test_controls + s = IC.skip(c, predicate=2) + @test !s.predicate(1) + @test s.predicate(2) + @test !s.predicate(3) + atomic_state = IC.update!(c, m, 0, 1) + state = IC.update!(s, m, 0, 1) + @test state == (n = 1, ) + @test isnothing(IC.public_state(s, state)) + state = IC.update!(s, m, 0, 2, state) + @test state == (atomic_state = atomic_state, n = 2) + state = IC.update!(s, m, 0, 3, state) + @test state == (atomic_state = atomic_state, n = 3) + atomic_state = IC.update!(c, m, 0, 2, atomic_state) + state = IC.update!(s, m, 0, 4, state) + @test state == (atomic_state = atomic_state, n = 4) + @test IC.done(c, atomic_state) == IC.done(s, state) + @test IC.takedown(c, 0, atomic_state) == IC.takedown(s, 0, state) + end end diff --git a/typos.toml b/typos.toml new file mode 100644 index 0000000..c738aa4 --- /dev/null +++ b/typos.toml @@ -0,0 +1,12 @@ +[files] +# we exclude markdown in the notebooks, as they are auto-generated: +extend-exclude = ["*.svg", "*.drawio", "*.html", "*.pdf", "docs/src/notebooks/**/*.md"] + +[default.extend-words] +# Don't correct "mape" to "map" +mape = "mape" +yhat = "yhat" +Jupyter = "Jupyter" +Missings = "Missings" + +