From bc3af2e533368a47878f85bb10cfd4c22e49c4e9 Mon Sep 17 00:00:00 2001 From: jadamcrain Date: Thu, 15 Oct 2026 11:31:42 -0700 Subject: [PATCH 1/2] Exclude procedures through the configuration, reported as not run A property per procedure, test.exclude. = , leaves a procedure out of a run: it's recorded as excluded without touching the device or the plugin, and the run carries on. The report shows it as not run, with the reason, counts it apart from skips, and warns that conformance hasn't been demonstrated for it. --- README.md | 2 + docs/automation.md | 26 +++- docs/manual-testing.md | 29 +++- src/main/resources/report.css | 27 +++- .../stepfunc/dnp3c/conformance/Harness.scala | 7 +- .../io/stepfunc/dnp3c/conformance/Main.scala | 64 ++++++-- .../stepfunc/dnp3c/conformance/Revision.scala | 59 +++++++- .../dnp3c/conformance/api/Options.scala | 32 ++-- .../dnp3c/conformance/api/RunSummary.scala | 13 ++ .../dnp3c/conformance/api/TestOption.scala | 31 ++++ .../conformance/api/options/TestOptions.scala | 15 +- .../conformance/application/Console.scala | 2 +- .../application/OptionFileWriter.scala | 12 +- .../conformance/application/SavedResult.scala | 7 + .../conformance/application/TestRunner.scala | 59 ++++---- .../application/report/Classes.scala | 1 + .../application/report/Report.scala | 35 ++++- .../dnp3c/conformance/HarnessTest.scala | 39 +++++ .../stepfunc/dnp3c/conformance/MainTest.scala | 72 ++++++++- .../conformance/api/OptionsTestSuite.scala | 26 ++++ .../application/SavedResultTest.scala | 14 ++ .../application/TestRunnerTest.scala | 137 +++++++++++++++++- 22 files changed, 610 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 90b601a..5d39688 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,8 @@ dnp3c -rev 2.7 -help # list the commands and test se Results, including `report.html`, are written to the revision's folder of the directory set by `test.output.directory`, e.g. `results/2.7/L2/`. The configuration must set `outstation.subset.level`, the DNP3 subset level the device is configured for: it has no default. See [Testing a device by hand](docs/manual-testing.md) for the whole workflow. +A procedure the device can't pass yet, e.g. for a known gap, can be excluded in the configuration, with the reason: `test.exclude.8.26.2 = Assign Class isn't supported yet`. It isn't run, and the report shows it as not run, with the reason, apart from skipped procedures, which don't apply to the device. See [Exclude procedures](docs/manual-testing.md#exclude-procedures). + ## Library The library jar, `io.stepfunc:dnp3c`, is published to Maven Central. See [Automating the tests](docs/automation.md) for writing an integration plugin and running it in CI. diff --git a/docs/automation.md b/docs/automation.md index 9f6b604..25a982b 100644 --- a/docs/automation.md +++ b/docs/automation.md @@ -40,7 +40,7 @@ object Main { - `Options.load` parses and checks the whole file, and throws a `ConfigurationException` listing every problem. - The revision decides which procedures exist and what their ids mean. `Revisions.list` holds every revision dnp3c has, and `Revisions.find("2.7")` looks one up by id. There's no default. - The configuration must set `outstation.subset.level`, the subset level the device is configured for, to a level the revision tests: level 2 for 2.7. `run` and `writeReport` throw a `ConfigurationException` before doing anything if it doesn't. `run` also does if the configuration lacks a value the revision, or a procedure that will run, needs: e.g. a Device Profile value that a test checks the device's responses against. -- `run` takes a procedure or section id of the revision, e.g. `"8.6.5.4"` or `"8.6"`, or `"all"`, and throws an `IllegalArgumentException` for an unknown one before anything runs. It runs the procedures in order and stops at the first that fails. It returns a `RunSummary` with each procedure's outcome (`Passed`, `Failed(message)` or `Skipped(reason)`) and the procedures it didn't reach. +- `run` takes a procedure or section id of the revision, e.g. `"8.6.5.4"` or `"8.6"`, or `"all"`, and throws an `IllegalArgumentException` for an unknown one before anything runs. It runs the procedures in order and stops at the first that fails. It returns a `RunSummary` with each procedure's outcome (`Passed`, `Failed(message)`, `Skipped(reason)` or `Excluded(reason)`) and the procedures it didn't reach. - Results go in the folder for the revision and subset level in the output directory, e.g. `results/2.7/L2/`, so runs of different revisions or levels never mix. - `writeReport` writes `report.html` in that folder from every result there, from this run and earlier ones, and returns its path. `Harness.writeReport(options, revision)` does the same without a plugin. @@ -50,7 +50,27 @@ For each procedure, the harness: 2. connects to the DUT over TCP and runs the steps, calling the plugin for each step it can't perform itself; 3. calls `endProcedure`, closes the connection, and writes the procedure's results. -A procedure that won't run is skipped before any of that: one that isn't implemented, doesn't apply at the subset level, or needs a feature the device lacks. It gets a log and a result saying why, but the harness doesn't connect to the DUT or call the plugin, not even `startProcedure` or `endProcedure`. +A procedure that won't run is skipped before any of that: one that isn't implemented, doesn't apply at the subset level, or needs a feature the device lacks. So is one the configuration excludes. It gets a log and a result saying why, but the harness doesn't connect to the DUT, capture or call the plugin, not even `startProcedure` or `endProcedure`. + +### Known gaps + +A device with a known gap would stop every run at the procedure that tests it. Exclude the procedure in the configuration, with the reason, and the run carries on past it: + +``` +test.exclude.8.26.2 = Assign Class isn't supported yet (stepfunc/dnp3#480) +``` + +[Testing a device by hand](manual-testing.md#exclude-procedures) has the rules: one property per procedure of the revision, each with a reason, checked before anything runs, and a procedure that's skipped stays skipped. Exclude a procedure for a feature the device has; turn off a `supports.*` option only for one it lacks. + +An excluded procedure's outcome is `Excluded(reason)`, which is new: a match over `ProcedureOutcome` must handle it. Unlike `Skipped`, it means the procedure applies to the device but didn't run, so the device hasn't been shown to conform to it. The report shows it as not run, with the reason, and says that conformance hasn't been demonstrated. + +`RunSummary.succeeded` says whether the run completed without a failure, not whether the device conforms: a run with exclusions succeeds. `summary.exclusions` lists the procedures it excluded, in order. For a CI job that must fail until every gap is closed: + +```scala +if (!summary.succeeded || summary.exclusions.nonEmpty || !reported) sys.exit(1) +``` + +An exclusion replaces a procedure's earlier result only when a run reaches it, as running it does: run the procedure, its section or `all`. `writeReport` never changes results. The plugin runs in the same JVM as dnp3c. It can control a device elsewhere, e.g. through a test fixture, an API or a simulator, or it can host the device itself. A software outstation can be started in `startProcedure` and stopped in `endProcedure`, so each procedure starts from a known state. @@ -75,7 +95,7 @@ To get the whole command-line tool, with its commands and options, but with your io.stepfunc.dnp3c.conformance.Main.run(args, new MyDevicePlugin()) ``` -`Main.run` exits the JVM with 0 if every procedure passed or was skipped, and nonzero otherwise. +`Main.run` exits the JVM with 0 if every procedure passed, was skipped or was excluded, and nonzero otherwise. ## Dependency diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 77e1a52..7620b0b 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -62,11 +62,28 @@ The procedures are organized in sections: 6 (link layer), 7 (transport layer) an ``` dnp3c -rev 2.7 -config device.ini -describe 8.6 # the procedures in a section dnp3c -rev 2.7 -config device.ini -describe 8.6.5.4 # the steps of one procedure -dnp3c -rev 2.7 -config device.ini -disabled # optional procedures your configuration turns off +dnp3c -rev 2.7 -config device.ini -disabled # procedures your configuration skips or excludes ``` Procedures for features the DUT doesn't support are optional. If a `supports.*` option turns one off, it's recorded as skipped instead of run. +## Exclude procedures + +A procedure that applies to the DUT, but that you can't run yet, e.g. for a known gap in the DUT that's being fixed, can be left out of runs by excluding it in the configuration, with the reason: + +``` +test.exclude.8.26.2 = Assign Class isn't supported yet (stepfunc/dnp3#480) +``` + +- **One property per procedure,** `test.exclude.`, and each needs a reason. Only procedures can be excluded, not sections or `all`. Every exclusion must name a procedure of the revision given with `-rev`, even one outside what you run: `-run`, `-describe` and `-disabled` check them all, and fail on any that doesn't. +- **An excluded procedure isn't run.** dnp3c doesn't connect to the DUT, capture or prompt for it, and the run carries on to the next procedure. It's recorded as excluded, with a log saying so, replacing the procedure's earlier result, log and capture. +- **Excluded isn't skipped.** A skipped procedure doesn't apply to the DUT or its configuration, and the DUT can still be conformant. An excluded one does apply, so the DUT hasn't been shown to conform to it. The report shows it as not run, with the reason, and counts it apart from passed, failed and skipped procedures. A run that excludes procedures still succeeds, but it ends by saying how many it excluded. +- **A procedure that's skipped stays skipped,** e.g. one that doesn't apply at the DUT's subset level, even if it's excluded too. +- **Options describe the DUT; exclusions record gaps in the testing.** Don't turn a `supports.*` option off to leave out a procedure for a feature the DUT has: that changes what other procedures expect of it. +- **Only a run replaces results.** Removing an exclusion doesn't change the report until the procedure runs again, and adding one doesn't replace an earlier result until a run reaches the procedure: run it, its section or `all`. `-report` never changes results. + +`-describe` shows an excluded procedure's reason instead of its steps, and marks the excluded procedures in a section. `-disabled` lists them with their reasons. + ## Run `-run` takes the id of any procedure or section, or `all`: @@ -86,7 +103,7 @@ Some questions come after a procedure's steps, where stopping for your answer wo If you miss a time limit, the procedure fails with a message saying so; rerun it. A limit that's too short to work to by hand is usually a timeout on the DUT that you can lengthen, along with its option here. -A run stops at the first procedure that fails, and the procedures after it don't run. The exit code is 0 if every procedure passed or was skipped, and nonzero otherwise. +A run stops at the first procedure that fails, and the procedures after it don't run. The exit code is 0 if every procedure passed, was skipped or was excluded, and nonzero otherwise. ## Results @@ -100,13 +117,13 @@ results/ report.css 8.6.5.4/ log.html every step, frame and check, with the outcome - results.properties the result (pass, fail or skip), the revision and subset level, and why it failed or was skipped + results.properties the result (pass, fail, skip or excluded), the revision and subset level, and why it failed, was skipped or was excluded dump.pcap the packet capture, if capture is on ``` A procedure that won't run, because the DUT lacks a feature it needs, it doesn't apply at the subset level, or it isn't implemented, is recorded as skipped without connecting to the DUT. -The report is rebuilt after every run, including runs that fail. It names the revision and the subset level, and shows every procedure: the ones you've run with their result (and, for one that was skipped, why, or for one that passed, anything it noted, such as a device that complies with IEEE 1815-2012 but not 1815-202x), and the rest as not run. To rebuild it without running anything, use `dnp3c -rev 2.7 -config device.ini -report`. +The report is rebuilt after every run, including runs that fail. It names the revision and the subset level, and shows every procedure: the ones you've run with their result (and, for one that was skipped or excluded, why, or for one that passed, anything it noted, such as a device that complies with IEEE 1815-2012 but not 1815-202x), and the rest as not run, with no saved result. Excluded procedures are shown as "NOT RUN — EXCLUDED", and if there are any, the report says at the top that conformance hasn't been demonstrated for them. To rebuild it without running anything, use `dnp3c -rev 2.7 -config device.ini -report`. The report counts only results saved under its revision and level. It lists anything else it finds at the top and leaves it out: a result saved under another revision or level, or a procedure whose last run didn't finish (it has a log but no result). @@ -161,8 +178,8 @@ https://stepfunc.io -config Path to configuration file -describe Describe the steps to be executed in the specified procedure - -disabled List all optional test procedures that are disabled by - configuration + -disabled List the test procedures the configuration skips or + excludes -help Print this help information -options List all of the available configuration file options -report Generate an HTML report containing all the results diff --git a/src/main/resources/report.css b/src/main/resources/report.css index 7ebc2fa..16f5ab6 100644 --- a/src/main/resources/report.css +++ b/src/main/resources/report.css @@ -16,6 +16,8 @@ --fail-bg: #fbe4e6; --skip: #8a6400; --skip-bg: #fff3cd; + --excluded: #6a1b9a; + --excluded-bg: #f3e5f5; --mono: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", monospace; } @@ -100,6 +102,7 @@ h1 { .total.pass { color: var(--pass); } .total.fail { color: var(--fail); } .total.skip { color: var(--skip); } +.total.excluded { color: var(--excluded); } .total.no-result { color: var(--sf-grey); } main { @@ -197,7 +200,7 @@ tr.incomplete > td:first-child { } /* the result column, shown as a badge */ -td.pass a, td.fail a, td.skip a { +td.pass a, td.fail a, td.skip a, td.excluded a { display: inline-block; min-width: 52px; padding: 0 10px; @@ -211,9 +214,10 @@ td.pass a, td.fail a, td.skip a { td.pass a { color: var(--pass); background-color: var(--pass-bg); } td.fail a { color: var(--fail); background-color: var(--fail-bg); } td.skip a { color: var(--skip); background-color: var(--skip-bg); } +td.excluded a { color: var(--excluded); background-color: var(--excluded-bg); white-space: nowrap; } /* a procedure's id (underlined, as links are) and its result badge open its log */ -td.pass a:hover, td.fail a:hover, td.skip a:hover { +td.pass a:hover, td.fail a:hover, td.skip a:hover, td.excluded a:hover { text-decoration: underline; } @@ -229,6 +233,25 @@ p.notice { border-radius: 4px; } +/* procedures the configuration excluded, which the device hasn't been shown to conform to */ +p.exclusions { + color: var(--excluded); + background-color: var(--excluded-bg); + padding: 6px 10px; + border-radius: 4px; +} + +p.exclusions strong { + font-weight: 600; +} + +/* why a procedure was excluded, under its title */ +span.exclusion { + color: var(--excluded); + font-size: 12px; + font-weight: 600; +} + /* what a procedure noted about a pass, under its title */ span.note { color: var(--pass); diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/Harness.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/Harness.scala index 6d8e9f7..3652ccf 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/Harness.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/Harness.scala @@ -34,13 +34,14 @@ final class Harness(options: Options, plugin: IntegrationPlugin, revision: Revis * Run the procedures of a procedure or section, or "all" of them, in order, stopping at the first that fails. * * A procedure fails when one of its steps or validations does, including a plugin action or verification that - * throws. + * throws. One the configuration excludes (`test.exclude.`) isn't run: it's recorded as excluded, without + * touching the device or the plugin, and the run carries on. * * @param id a procedure or section of the revision, e.g. "8.6.5.4" or "8.6", or "all" * @throws IllegalArgumentException if the revision has no procedure or section with the id, before anything runs * @throws api.ConfigurationException if the configuration has no subset level, or one the revision doesn't test, or - * lacks what the revision, or a procedure that will run, needs configured, - * before anything runs + * excludes something that isn't a procedure of the revision, or lacks what the + * revision, or a procedure that will run, needs configured, before anything runs * @throws HarnessExecutionException if the harness or the plugin fails around a procedure, e.g. when * `startProcedure` throws or the capture can't be opened; it holds the * results so far. An interruption propagates as it is. diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/Main.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/Main.scala index 9d6b66e..1aa56bb 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/Main.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/Main.scala @@ -128,8 +128,12 @@ object Main { // and a subset level the revision tests; a run checks the rest of the configuration too, before it starts, while // the commands that only describe or report show what's wrong with it instead val level = - try revision.subsetLevel(options) - catch { + try { + val level = revision.subsetLevel(options) + // the exclusions -disabled and -describe show must name procedures of the revision + if (config.listDisabled || config.describe.nonEmpty) revision.checkExclusions(options) + level + } catch { case ex: ConfigurationException => Colors.error.printLine(ErrorText.of(ex)) return -1 @@ -143,13 +147,14 @@ object Main { config.describe match { case Some(id) => revision.elements.get(id) match { - case Some(section: TestSection) => print(section, Int.MaxValue, 0) + case Some(section: TestSection) => print(section, Int.MaxValue, 0, exclusions(options, revision, level)) case Some(procedure: TestProcedure) => - revision.skip(procedure, options, level) match { - case Some(Skip.NotApplicable(levels)) => + revision.decide(procedure, options, level) match { + case Decision.Skip(Skip.NotApplicable(levels)) => println( s"Not applicable at subset level $level: ${procedure.id} applies to subset level(s) " + levels.mkString(", ")) + case Decision.Exclude(reason) => println(s"NOT RUN — EXCLUDED ${procedure.id}: $reason") case _ => // the description says if it isn't implemented, or which options disable it; building the steps checks // the options the procedure uses, e.g. 8.23's device attributes @@ -188,8 +193,13 @@ object Main { try { val summary = harness.run(id) if (!summary.succeeded) Colors.error.printLine("Abort due to test failure") + exclusionWarning(summary).foreach(Colors.warn.printLine) summary.succeeded } catch { + case ex: HarnessExecutionException => + Colors.error.printLine(ErrorText.of(ex)) + exclusionWarning(ex.partial).foreach(Colors.warn.printLine) + false case NonFatal(ex) => Colors.error.printLine(ErrorText.of(ex)) false @@ -204,6 +214,20 @@ object Main { } } + // how a run's end tells of the procedures it excluded, which it didn't run + private[conformance] def exclusionWarning(summary: RunSummary): Option[String] = + Option.when(summary.exclusions.nonEmpty)( + s"${summary.exclusions.size} procedure(s) excluded and not run. Conformance has not been demonstrated for them.") + + // the procedures the configuration excludes that aren't skipped anyway, and why + private def exclusions(options: OptionSource, revision: Revision, level: Int): Map[String, String] = + revision.flatMapProcedures { p => + revision.decide(p, options, level) match { + case Decision.Exclude(reason) => Some(p.id -> reason) + case _ => None + } + }.toMap + private def writeReport(write: => Path): Boolean = try { Colors.text.printLine("Wrote the report: " + write.toAbsolutePath.normalize) @@ -262,22 +286,25 @@ object Main { s"Unknown revision: $id. The revisions are: ${Revisions.list.map(_.id).mkString(", ")}" def listDisabled(options: OptionSource, revision: Revision, level: Int): Unit = { - Colors.text.printLine("Listing optional test procedures disabled by configuration: ") + Colors.text.printLine("Listing the test procedures the configuration skips or excludes: ") Colors.text.printLine("") disabledLines(options, revision, level).foreach(Colors.text.printLine) } // each procedure the configuration keeps from running, once: those that don't apply at the level, then those that - // options disable. One that isn't implemented isn't the configuration's doing. + // options disable, then those it excludes. One that isn't implemented isn't the configuration's doing. private[conformance] def disabledLines(options: OptionSource, revision: Revision, level: Int): List[String] = { - val skipped = revision.flatMapProcedures(p => revision.skip(p, options, level).map(p -> _)).toList - val notApplicable = skipped.collect { case (p, Skip.NotApplicable(_)) => + val decided = revision.flatMapProcedures(p => Some(p -> revision.decide(p, options, level))).toList + val notApplicable = decided.collect { case (p, Decision.Skip(Skip.NotApplicable(_))) => List("Procedure: " + p.id, s" Not applicable at subset level $level") } - val disabled = skipped.collect { case (p, Skip.Disabled(unmet)) => + val disabled = decided.collect { case (p, Decision.Skip(Skip.Disabled(unmet))) => ("Procedure: " + p.id) :: unmet.values.map((option, value) => " Option: " + option.id + " == " + value) } - (notApplicable ++ disabled).flatten + val excluded = decided.collect { case (p, Decision.Exclude(reason)) => + List("Procedure: " + p.id, " NOT RUN — EXCLUDED: " + reason) + } + (notApplicable ++ disabled ++ excluded).flatten } def printNetworkAdapters(): Unit = { @@ -297,8 +324,11 @@ object Main { def printOption(option: TestOption): Unit = { Colors.success.printLine("// " + option.description) - if (option.required) Colors.text.printLine(option.id + " = (required)") - else Colors.text.printLine(option.id + " = " + option.defaultAsString) + option match { + case o: ExclusionsOption => Colors.text.printLine("# " + o.example) + case _ if option.required => Colors.text.printLine(option.id + " = (required)") + case _ => Colors.text.printLine(option.id + " = " + option.defaultAsString) + } } def printGroup(group: OptionGroup): Unit = { @@ -337,10 +367,12 @@ object Main { } } - def print(element: TestElement, depth: Int, numSpaces: Int): Unit = { + // with the procedures the configuration excludes marked + def print(element: TestElement, depth: Int, numSpaces: Int, excluded: Map[String, String] = Map.empty): Unit = { val spaces = " " * numSpaces - def line = (spaces + element.id).padTo(30, " ").mkString + element.description + def line = (spaces + element.id).padTo(30, " ").mkString + element.description + + excluded.get(element.id).fold("")(reason => s" [NOT RUN — EXCLUDED: $reason]") val colors: Colors = element match { case x: TestProcedure if x.incomplete => Colors.incomplete @@ -350,7 +382,7 @@ object Main { colors.printLine(line) if (depth > 0) { - element.subElements.foreach(e => print(e, depth - 1, numSpaces + 2)) + element.subElements.foreach(e => print(e, depth - 1, numSpaces + 2, excluded)) } } diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/Revision.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/Revision.scala index 73c91b2..d3db011 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/Revision.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/Revision.scala @@ -81,8 +81,9 @@ final class Revision private[conformance] ( } /** - * Why a procedure won't run on a device configured for a subset level, if it won't. The runner, the configuration - * check and `-disabled` all decide with it. + * Why a procedure won't run on a device configured for a subset level, if it won't, because it doesn't apply to the + * device or its configuration. The runner, the configuration check and `-disabled` all decide with it, before any + * exclusion (see [[decide]]). */ def skip(procedure: TestProcedure, options: OptionSource, level: Int): Option[Skip] = if (procedure.incomplete) Some(Skip.NotImplemented) @@ -93,9 +94,41 @@ final class Revision private[conformance] ( } /** - * Check that the configuration is one this revision can test: it gives a subset level the revision tests, and it has - * what the revision needs configured, whatever procedures run, and what each procedure selected needs, if it will run - * (see [[skip]]). + * Whether a procedure runs, is skipped, or is excluded by the configuration. A skip comes first: a procedure that + * doesn't apply stays skipped, even if it's excluded too. + */ + private[conformance] def decide(procedure: TestProcedure, options: OptionSource, level: Int): Decision = + skip(procedure, options, level) match { + case Some(s) => Decision.Skip(s) + case None => options.get(TestOptions.exclude).get(procedure.id).fold(Decision.Run)(Decision.Exclude(_)) + } + + // every exclusion must name a procedure of this revision, whatever procedures a run selects + private def exclusionProblems(options: OptionSource): List[ConfigProblem] = + options.get(TestOptions.exclude).toList.sorted.flatMap { (procedure, reason) => + val key = s"${TestOptions.exclude.id}.$procedure" + elements.get(procedure) match { + case Some(_: TestProcedure) => None + case Some(_) => + Some(ConfigProblem(key, reason, "a section: exclude its procedures, each with its own property")) + case None => Some(ConfigProblem(key, reason, s"revision $id has no procedure $procedure")) + } + } + + /** + * Check that every exclusion in the configuration names a procedure of this revision + * + * @throws ConfigurationException listing every one that doesn't + */ + private[conformance] def checkExclusions(options: OptionSource): Unit = { + val problems = exclusionProblems(options) + if (problems.nonEmpty) throw new ConfigurationException("configuration", problems) + } + + /** + * Check that the configuration is one this revision can test: it gives a subset level the revision tests, its + * exclusions name procedures of the revision, and it has what the revision needs configured, whatever procedures + * run, and what each procedure selected needs, if it will run (see [[decide]]). * * @param selected the procedures a run will run; all of them unless given * @return the subset level @@ -104,9 +137,10 @@ final class Revision private[conformance] ( def validate(options: OptionSource, selected: Iterable[TestProcedure] = flatMapProcedures(Some(_))): Int = { val level = subsetLevel(options) // what the procedures that will run need, each problem once - val procedures = selected.filter(skip(_, options, level).isEmpty) + val procedures = selected.filter(decide(_, options, level) == Decision.Run) val problems = - (configurationProblems(options) ++ procedures.flatMap(_.definition.configurationProblems(options))).distinct + (exclusionProblems(options) ++ configurationProblems(options) ++ + procedures.flatMap(_.definition.configurationProblems(options))).distinct if (problems.nonEmpty) throw new ConfigurationException("configuration", problems) level } @@ -157,6 +191,17 @@ enum Skip { case Disabled(unmet: Unmet) } +/** whether a procedure runs, see [[Revision.decide]] */ +private[conformance] enum Decision { + case Run + + /** it doesn't apply to the device or its configuration */ + case Skip(skip: io.stepfunc.dnp3c.conformance.Skip) + + /** it applies, but the configuration leaves it out, for this reason */ + case Exclude(reason: String) +} + object Revisions { /** DNP3 IED Certification Procedure, Subset Level 2, version 2.7 */ diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/api/Options.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/api/Options.scala index 6472eca..70de88e 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/api/Options.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/api/Options.scala @@ -7,6 +7,8 @@ import java.nio.charset.StandardCharsets import java.nio.file.{ Files, Path } import java.util.Properties +import io.stepfunc.dnp3c.conformance.api.options.TestOptions + import scala.jdk.CollectionConverters._ /** @@ -64,21 +66,31 @@ object Options { private def parse(source: String, map: Map[String, String]): Options = { val byId = registered.map(o => o.id -> o).toMap - val parsed: List[Either[ConfigProblem, (TypedTestOption[?], Any)]] = map.toList.sortBy(_._1).map { - case (key, value) => - byId.get(key) match { - case None => Left(ConfigProblem(key, value, "unknown option")) - case Some(option) => - try Right(option -> option.parse(value)) - catch { case ex: Exception => Left(ConfigProblem(key, value, ErrorText.of(ex))) } - } + // each exclusion is a property of its own under test.exclude, which the option collects + val exclude = TestOptions.exclude + val (exclusions, others) = map.partition((key, _) => key.startsWith(exclude.id + ".")) + + val parsed: List[Either[ConfigProblem, (TypedTestOption[?], Any)]] = others.toList.map { case (key, value) => + byId.get(key) match { + case None => Left(ConfigProblem(key, value, "unknown option")) + case Some(option) => + try Right(option -> option.parse(value)) + catch { case ex: Exception => Left(ConfigProblem(key, value, ErrorText.of(ex))) } + } + } + + val excluded: List[Either[ConfigProblem, (String, String)]] = exclusions.toList.map { case (key, value) => + try Right(exclude.entry(key.stripPrefix(exclude.id + "."), value)) + catch { case ex: Exception => Left(ConfigProblem(key, value, ErrorText.of(ex))) } } - val problems = parsed.collect { case Left(problem) => problem } + val problems = (parsed ++ excluded).collect { case Left(problem) => problem }.sortBy(_.key) if (problems.nonEmpty) throw new ConfigurationException(source, problems) val defaults: Map[TypedTestOption[?], Any] = registered.map(o => o -> o.default).toMap - new Options(defaults ++ parsed.collect { case Right(value) => value }) + new Options( + defaults ++ parsed.collect { case Right(value) => value } + + (exclude -> excluded.collect { case Right(entry) => entry }.toMap)) } } diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/api/RunSummary.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/api/RunSummary.scala index d5e7f01..c50aecd 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/api/RunSummary.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/api/RunSummary.scala @@ -12,6 +12,12 @@ object ProcedureOutcome { case object Passed extends ProcedureOutcome case class Failed(message: String) extends ProcedureOutcome case class Skipped(reason: String) extends ProcedureOutcome + + /** + * The configuration left it out (`test.exclude.`), for this reason. Unlike a skipped procedure, it applies to + * the device, so the device hasn't been shown to conform to it. + */ + case class Excluded(reason: String) extends ProcedureOutcome } /** @@ -27,7 +33,14 @@ case class ProcedureResult(id: String, outcome: ProcedureOutcome, notes: List[St */ case class RunSummary(results: List[ProcedureResult], unreached: List[String]) { + /** + * Whether the run completed without a failure: it reached every procedure, and each passed, was skipped or was + * excluded. It says the run succeeded, not that the device conforms: see [[exclusions]]. + */ def succeeded: Boolean = unreached.isEmpty && !results.exists(_.outcome.isInstanceOf[ProcedureOutcome.Failed]) + + /** the procedures the configuration excluded, in order, which the device hasn't been shown to conform to */ + def exclusions: List[ProcedureResult] = results.filter(_.outcome.isInstanceOf[ProcedureOutcome.Excluded]) } /** diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/api/TestOption.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/api/TestOption.scala index a3db44b..3b123d4 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/api/TestOption.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/api/TestOption.scala @@ -50,6 +50,16 @@ trait IndexListOption extends TypedTestOption[List[Index]] trait RequiredIntegerOption extends TypedTestOption[Option[Int]] { final override def required: Boolean = true } +// procedures to leave out of a run, by id, each with the reason: a family of properties, one per procedure, under the +// option's id, e.g. "test.exclude.8.26.2 = " +trait ExclusionsOption extends TypedTestOption[Map[String, String]] { + + // a property of the family, for the configurations written and listed, commented out + def example: String + + // the procedure id and reason of a property of the family, given the rest of its key after the option's id and "." + def entry(procedure: String, value: String): (String, String) +} object OptionTypes { @@ -92,6 +102,8 @@ object OptionTypes { RequiredMinimumOptionImpl(group, name, desc, min) def hexByteInSet(group: OptionGroup, name: String, default: Byte, desc: String, allowed: Set[Byte]): HexByteOption = HexByteInSetImpl(group, name, default, desc, allowed) + def exclusions(group: OptionGroup, name: String, description: String, example: String): ExclusionsOption = + ExclusionsOptionImpl(group, name, description, example) // one of the allowed words, in any case def choice(group: OptionGroup, name: String, default: String, desc: String, allowed: List[String]): StringOption = ChoiceOptionImpl(group, name, default, desc, allowed) @@ -101,6 +113,25 @@ object OptionTypes { override def parse(s: String): String = s.trim } + private case class ExclusionsOptionImpl(group: OptionGroup, name: String, description: String, example: String) + extends ExclusionsOption { + + override def default: Map[String, String] = Map.empty + + override def defaultAsString: String = "" + + // the family has no value of its own + override def parse(s: String): Map[String, String] = + throw new Exception(s"exclude one procedure per property, with the reason: $id. = ") + + override def entry(procedure: String, value: String): (String, String) = { + if (procedure.isEmpty) throw new Exception(s"name the procedure: $id. = ") + val reason = value.trim + if (reason.isEmpty) throw new Exception(s"give the reason $procedure is excluded") + procedure -> reason + } + } + private case class ChoiceOptionImpl( group: OptionGroup, name: String, diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/api/options/TestOptions.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/api/options/TestOptions.scala index 618241a..d67d92b 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/api/options/TestOptions.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/api/options/TestOptions.scala @@ -3,7 +3,9 @@ package io.stepfunc.dnp3c.conformance.api.options -import io.stepfunc.dnp3c.conformance.api.{ IntegerOption, OptionGroup, OptionTypes, StringOption, TestOption } +import io.stepfunc.dnp3c.conformance.api.{ + ExclusionsOption, IntegerOption, OptionGroup, OptionTypes, StringOption, TestOption +} object TestOptions extends OptionGroup { @@ -25,8 +27,17 @@ object TestOptions extends OptionGroup { None ) + val exclude: ExclusionsOption = OptionTypes.exclusions( + this, + "exclude", + "Procedures to leave out of a run, one property each, with the reason: test.exclude. = . " + + "An excluded procedure applies to the DUT but isn't run, and is reported as not run, so the DUT isn't shown to " + + "conform to it. A procedure for a feature the DUT lacks is skipped by its supports.* option instead", + "test.exclude.8.26.2 = Assign Class isn't supported yet" + ) + def prefix: String = "test" def description: String = "Options for the test harness itself" - def options: List[TestOption] = List(outputDirectory, deviceUnderTest, manualActionWindowMs) + def options: List[TestOption] = List(outputDirectory, deviceUnderTest, manualActionWindowMs, exclude) } diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/application/Console.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/application/Console.scala index 827bb6d..b65a12b 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/application/Console.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/application/Console.scala @@ -65,7 +65,7 @@ object Console { private val describe = OptionDefinition("describe", true, "Describe the steps to be executed in the specified procedure") private val disabled = - OptionDefinition("disabled", false, "List all optional test procedures that are disabled by configuration") + OptionDefinition("disabled", false, "List the test procedures the configuration skips or excludes") private val report = OptionDefinition("report", false, "Generate an HTML report containing all the results") private val options = OptionDefinition("options", false, "List all of the available configuration file options") private val adapters = diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/application/OptionFileWriter.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/application/OptionFileWriter.scala index 82b9f21..62c73ca 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/application/OptionFileWriter.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/application/OptionFileWriter.scala @@ -5,7 +5,7 @@ package io.stepfunc.dnp3c.conformance.application import java.io.File -import io.stepfunc.dnp3c.conformance.api.{ OptionGroup, TestOption } +import io.stepfunc.dnp3c.conformance.api.{ ExclusionsOption, OptionGroup, TestOption } object OptionFileWriter { @@ -22,9 +22,13 @@ object OptionFileWriter { def writeOption(option: TestOption): Unit = { writer.println("# " + option.description) - // a required option has no default to write: it's left for you to set - if (option.required) writer.println("# " + option.id + " = ") - else writer.println(option.id + " = " + option.defaultAsString) + option match { + // exclusions have no default, just a property per procedure: an example shows how to write one + case o: ExclusionsOption => writer.println("# " + o.example) + // a required option has no default to write: it's left for you to set + case _ if option.required => writer.println("# " + option.id + " = ") + case _ => writer.println(option.id + " = " + option.defaultAsString) + } writer.println() } diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/application/SavedResult.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/application/SavedResult.scala index cd8e5d6..96da8f9 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/application/SavedResult.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/application/SavedResult.scala @@ -31,6 +31,7 @@ object SavedResult { private val pass = "pass" private val skip = "skip" private val fail = "fail" + private val excluded = "excluded" private val resultKey = "result" private val revisionKey = "revision" @@ -48,6 +49,9 @@ object SavedResult { case `pass` => ProcedureOutcome.Passed case `fail` => ProcedureOutcome.Failed(reason) case `skip` => ProcedureOutcome.Skipped(reason) + case `excluded` => + if (reason.isBlank) throw new Exception(s"An excluded result has no reason: $file") + ProcedureOutcome.Excluded(reason) case value => throw new Exception(s"Unknown result: $value") } SavedResult( @@ -77,6 +81,9 @@ object SavedResult { case ProcedureOutcome.Skipped(reason) => props.setProperty(resultKey, skip) props.setProperty(reasonKey, reason) + case ProcedureOutcome.Excluded(reason) => + props.setProperty(resultKey, excluded) + props.setProperty(reasonKey, reason) } saved.revision.foreach(props.setProperty(revisionKey, _)) saved.level.foreach(l => props.setProperty(levelKey, l.toString)) diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/application/TestRunner.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/application/TestRunner.scala index 12dcc7d..283109e 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/application/TestRunner.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/application/TestRunner.scala @@ -6,7 +6,7 @@ package io.stepfunc.dnp3c.conformance.application import java.io.Closeable import java.nio.file.{ Files, Path } -import io.stepfunc.dnp3c.conformance.{ Revision, Skip } +import io.stepfunc.dnp3c.conformance.{ Decision, Revision, Skip } import io.stepfunc.dnp3c.conformance.application.pcap.PacketCapture import io.stepfunc.dnp3c.conformance.api._ import io.stepfunc.dnp3c.conformance.api.options.{ @@ -120,23 +120,15 @@ object TestRunner { // the harness or the plugin failed, after the procedure reached the outcome, if any private class ProcedureAborted(val outcome: Option[ProcedureOutcome], val error: Throwable) extends Exception(error) - /** - * Why a procedure won't run, if it won't: it isn't implemented, it doesn't apply at the subset level, or the device - * lacks a feature it needs. Decided before anything is opened, so a skipped procedure never touches the device or the - * plugin. - */ - private def skipReason( - revision: Revision, - level: Int, - procedure: TestProcedure, - options: OptionSource): Option[String] = - revision.skip(procedure, options, level).map { - case Skip.NotImplemented => "not implemented" - case Skip.NotApplicable(levels) => - s"not applicable: it applies to subset level(s) ${levels.mkString(", ")}, and the device is configured for " + - s"level $level" - case Skip.Disabled(unmet) => s"disabled because ${unmet.message}" - } + // why a procedure is skipped: it isn't implemented, it doesn't apply at the subset level, or the device lacks a + // feature it needs + private def skipReason(skip: Skip, level: Int): String = skip match { + case Skip.NotImplemented => "not implemented" + case Skip.NotApplicable(levels) => + s"not applicable: it applies to subset level(s) ${levels.mkString(", ")}, and the device is configured for " + + s"level $level" + case Skip.Disabled(unmet) => s"disabled because ${unmet.message}" + } private def runProcedure( revision: Revision, @@ -156,10 +148,18 @@ object TestRunner { SavedResult.write(SavedResult(result, Some(revision.id), Some(level), kept), resultsFile) } - skipReason(revision, level, procedure, options) match { - case Some(reason) => - return ProcedureResult(procedure.id, skip(procedure, path, title, reason, options, r => save(r))) - case None => + // decided before anything is opened, so a procedure that's skipped or excluded never touches the device or the + // plugin + def record(outcome: ProcedureOutcome, message: String): ProcedureResult = + ProcedureResult(procedure.id, notRun(path, title, outcome, message, options, r => save(r))) + + revision.decide(procedure, options, level) match { + case Decision.Skip(skip) => + val reason = skipReason(skip, level) + return record(ProcedureOutcome.Skipped(reason), s"Skipped procedure: ${procedure.id}, $reason") + case Decision.Exclude(reason) => + return record(ProcedureOutcome.Excluded(reason), s"NOT RUN — EXCLUDED ${procedure.id}: $reason") + case Decision.Run => } // run once the procedure ends, however it ends, the last added first @@ -217,23 +217,24 @@ object TestRunner { ProcedureResult(procedure.id, result, notes) } - // records a procedure that won't run: its log says why, and nothing is opened - private def skip( - procedure: TestProcedure, + // records a procedure that's skipped or excluded: its log says why, and nothing is opened + private def notRun( path: Path, title: String, - reason: String, + outcome: ProcedureOutcome, + message: String, options: OptionSource, save: ProcedureOutcome => Unit): ProcedureOutcome = { - val outcome = ProcedureOutcome.Skipped(reason) try { Files.createDirectories(path) - // results from an earlier run must not look like this run's, even if writing this one fails + // results from an earlier run must not look like this run's, even if writing this one fails: the result first Files.deleteIfExists(path.resolve(ResultFiles.result)) + Files.deleteIfExists(path.resolve(ResultFiles.log)) Files.deleteIfExists(path.resolve(ResultFiles.capture)) val htmlLogger = new ReportLogger(path.resolve(ResultFiles.log).toFile, title, options.get(TestOptions.deviceUnderTest)) - try Logger.aggregate(ConsoleLogger, htmlLogger).info(s"Skipped procedure: ${procedure.id}, $reason") + val logger = Logger.aggregate(ConsoleLogger, htmlLogger) + try if (outcome.isInstanceOf[ProcedureOutcome.Excluded]) logger.warn(message) else logger.info(message) finally htmlLogger.write() save(outcome) } catch { case NonFatal(ex) => throw new ProcedureAborted(Some(outcome), ex) } diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Classes.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Classes.scala index 2b97d74..69847f8 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Classes.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Classes.scala @@ -14,6 +14,7 @@ object Classes { val pass = Clazz("pass") val fail = Clazz("fail") val skip = Clazz("skip") + val excluded = Clazz("excluded") val noResult = Clazz("no-result") val section = Clazz("section") val procedure = Clazz("procedure") diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Report.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Report.scala index b247706..31268f9 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Report.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Report.scala @@ -34,21 +34,24 @@ case class Report(revision: Revision, level: Int, sections: Iterable[Report.Sect def link(text: String) = "" + text + "" // a procedure that has run has a log, which its id and its result both open def id = if (p.incomplete || p.outcome.isEmpty) p.id else link(p.id) + // a procedure that didn't run has no traffic def pcapLink = if (p.hasPcaps && !p.incomplete && - p.outcome.exists(o => !o.isInstanceOf[ProcedureOutcome.Skipped])) { + p.outcome.exists(o => + !o.isInstanceOf[ProcedureOutcome.Skipped] && !o.isInstanceOf[ProcedureOutcome.Excluded])) { "pcap" } else { "----" } - def getText(): String = if (p.incomplete) "----" + def getText(): String = if (p.incomplete) "Not implemented" else p.outcome match { case Some(ProcedureOutcome.Passed) => link("PASS") case Some(ProcedureOutcome.Failed(_)) => link("FAIL") case Some(ProcedureOutcome.Skipped(_)) => link("SKIP") - case None => "not run" + case Some(ProcedureOutcome.Excluded(_)) => link("NOT RUN — EXCLUDED") + case None => "Not run — no saved result" } def getClass(): Clazz = if (p.incomplete) Classes.unimplemented @@ -56,13 +59,16 @@ case class Report(revision: Revision, level: Int, sections: Iterable[Report.Sect case Some(ProcedureOutcome.Passed) => Classes.pass case Some(ProcedureOutcome.Failed(_)) => Classes.fail case Some(ProcedureOutcome.Skipped(_)) => Classes.skip + case Some(ProcedureOutcome.Excluded(_)) => Classes.excluded case None => Classes.noResult } - // why the procedure was skipped, or what it noted about a pass, under its title + // why the procedure was skipped or excluded, or what it noted about a pass, under its title def skipReason: String = p.outcome match { case Some(ProcedureOutcome.Skipped(reason)) if reason.nonEmpty => "
Skipped: " + Tags.escape(reason) + "" + case Some(ProcedureOutcome.Excluded(reason)) => + "
Excluded: " + Tags.escape(reason) + "" case Some(ProcedureOutcome.Passed) => p.notes.map(n => "
Note: " + Tags.escape(n) + "").mkString case _ => "" @@ -88,18 +94,32 @@ case class Report(revision: Revision, level: Int, sections: Iterable[Report.Sect case p: Report.Procedure => List(p) } + private def implemented: List[Report.Procedure] = sections.flatMap(procedures).filterNot(_.incomplete).toList + + private def excluded: Int = implemented.count(_.outcome.exists(_.isInstanceOf[ProcedureOutcome.Excluded])) + // e.g. "146 passed", counting implemented procedures by result private def totals: Iterable[(Clazz, String)] = { - val implemented = sections.flatMap(procedures).filterNot(_.incomplete).toList def count(counted: Option[ProcedureOutcome] => Boolean) = implemented.count(p => counted(p.outcome)) List( (Classes.pass, s"${count(_.contains(ProcedureOutcome.Passed))} passed"), (Classes.fail, s"${count(_.exists(_.isInstanceOf[ProcedureOutcome.Failed]))} failed"), (Classes.skip, s"${count(_.exists(_.isInstanceOf[ProcedureOutcome.Skipped]))} skipped"), - (Classes.noResult, s"${count(_.isEmpty)} not run") + (Classes.excluded, s"$excluded excluded"), + (Classes.noResult, s"${count(_.isEmpty)} with no saved result") ) } + // a report with exclusions mustn't read as a clean result + private def exclusionWarning(implicit i: Indentation): Iterable[String] = + if (excluded == 0) Nil + else + Tags.p(Attributes.classes(Clazz("exclusions")))(List( + s"$excluded procedure(s) excluded and not run. Conformance has not been demonstrated for them.", + "
A skipped procedure doesn't apply to the device or its configuration, and the device can still conform. " + + "An excluded procedure does apply, but the configuration left it out of the run (test.exclude)." + )) + def toHTML(deviceUnderTest: String): Iterable[String] = { def headings(implicit i: Indentation): Iterable[String] = List("id", "result", "capture", "description").flatMap { @@ -122,7 +142,8 @@ case class Report(revision: Revision, level: Int, sections: Iterable[Report.Sect Tags.p(List(s"Generated $generated")), totals.flatMap { case (clazz, text) => Tags.span(Attributes.classes(Clazz("total"), clazz))(List(text)) } ) { - notices.flatMap(n => Tags.p(Attributes.classes(Clazz("notice")))(List(Tags.escape(n)))) ++ + exclusionWarning ++ + notices.flatMap(n => Tags.p(Attributes.classes(Clazz("notice")))(List(Tags.escape(n)))) ++ Tags.p(Attributes.classes(Clazz("hint")))(List("Click a procedure's id or result to open its log.")) ++ Tags.table(Attributes.classes(Clazz("results"))) { Tags.tr(headings) ++ diff --git a/src/test/scala/io/stepfunc/dnp3c/conformance/HarnessTest.scala b/src/test/scala/io/stepfunc/dnp3c/conformance/HarnessTest.scala index 7b83682..913167e 100644 --- a/src/test/scala/io/stepfunc/dnp3c/conformance/HarnessTest.scala +++ b/src/test/scala/io/stepfunc/dnp3c/conformance/HarnessTest.scala @@ -95,6 +95,45 @@ class HarnessTest extends AnyFunSuite with Matchers { "Note: compliant with 1815-2012 only") } + test("the report shows each exclusion's reason, counts exclusions apart from skips, and warns of them") { + val directory = output + val folder = directory.resolve("2.7").resolve("L2") + def result(id: String, outcome: ProcedureOutcome): Unit = { + Files.createDirectories(folder.resolve(id)) + SavedResult.write(SavedResult(outcome, Some("2.7"), Some(2)), folder.resolve(id).resolve("results.properties")) + Files.writeString(folder.resolve(id).resolve("log.html"), "") + Files.writeString(folder.resolve(id).resolve("dump.pcap"), "") + } + result("8.1.2", ProcedureOutcome.Passed) + result("8.3.2", ProcedureOutcome.Skipped("disabled because supports.x == false")) + result("8.4.1.2.1", ProcedureOutcome.Excluded("Assign Class & co aren't supported")) + result("8.4.1.2.2", ProcedureOutcome.Excluded("a second gap")) + val report = Files.readString(Harness.writeReport(options(directory), revision)) + + report should include("Excluded: Assign <b>Class</b> & co aren't supported") + report should not include "Class" + report should include("Excluded: a second gap") + report should include("""NOT RUN — EXCLUDED""") + for (total <- List("1 passed", "0 failed", "1 skipped", "2 excluded", "with no saved result")) + report should include(total) + report should include("2 procedure(s) excluded and not run. Conformance has not been demonstrated for them.") + // no traffic for a procedure that didn't run + report should include("./8.1.2/dump.pcap") + report should not include "./8.3.2/dump.pcap" + report should not include "./8.4.1.2.1/dump.pcap" + // skips as they were, and a procedure without a result, or that isn't implemented, says so + report should include("""SKIP""") + report should include("Skipped: disabled because supports.x == false") + report should include("Not run — no saved result") + report should include("Not implemented") + } + + test("a report without exclusions doesn't warn of them") { + val report = Files.readString(Harness.writeReport(options(output), revision)) + report should include("0 excluded") + report should not include "excluded and not run" + } + test("tells about a procedure whose run left a log but no result") { val directory = output val folder = directory.resolve("2.7").resolve("L2").resolve("8.1.2") diff --git a/src/test/scala/io/stepfunc/dnp3c/conformance/MainTest.scala b/src/test/scala/io/stepfunc/dnp3c/conformance/MainTest.scala index 5213049..26265ec 100644 --- a/src/test/scala/io/stepfunc/dnp3c/conformance/MainTest.scala +++ b/src/test/scala/io/stepfunc/dnp3c/conformance/MainTest.scala @@ -6,8 +6,11 @@ package io.stepfunc.dnp3c.conformance import java.io.File import java.nio.file.Files -import io.stepfunc.dnp3c.conformance.api.{ BooleanOption, OptionGroups, Options } -import io.stepfunc.dnp3c.conformance.api.options.{ OutstationOptions, SupportOptions } +import io.stepfunc.dnp3c.conformance.api.{ + BooleanOption, OptionGroups, Options, ProcedureOutcome, ProcedureResult, RunSummary +} +import io.stepfunc.dnp3c.conformance.api.options.{ OutstationOptions, SupportOptions, TestOptions } +import io.stepfunc.dnp3c.conformance.application.SavedResult import io.stepfunc.dnp3c.conformance.application.Console import io.stepfunc.dnp3c.dnp3.DefaultIntegrationPlugin import org.junit.runner.RunWith @@ -23,14 +26,16 @@ class MainTest extends AnyFunSuite with Matchers { writeConfigFile: Option[File], config: Option[File], report: Boolean = false, - revision: Option[String] = Some("2.7")) = + revision: Option[String] = Some("2.7"), + run: Option[String] = None, + listDisabled: Boolean = false) = Console.Configuration( revision = revision, listRevisions = false, - run = None, + run = run, help = false, describe = describe, - listDisabled = false, + listDisabled = listDisabled, report = report, options = false, adapters = false, @@ -137,4 +142,61 @@ class MainTest extends AnyFunSuite with Matchers { withClue(s"${revision.id} L$level $value: ")(ids.diff(ids.distinct) should equal(Nil)) } } + + test("-disabled lists an exclusion with its reason, and a procedure that's skipped anyway as skipped") { + val options = Options.from(Map( + OutstationOptions.subsetLevel.id -> "2", + s"${TestOptions.exclude.id}.8.26.2" -> "Assign Class isn't supported yet", + s"${TestOptions.exclude.id}.8.26.3" -> "nor no-class assignment" + )) + val lines = Main.disabledLines(options, Revisions.v3_1r1, 2) + lines.sliding(2).toList should + contain(List("Procedure: 8.26.2", " NOT RUN — EXCLUDED: Assign Class isn't supported yet")) + lines(lines.indexOf("Procedure: 8.26.3") + 1) should equal(" Not applicable at subset level 2") + lines.mkString should not include "nor no-class assignment" + } + + private def withExclusion(id: String, reason: String = "Assign Class isn't supported yet") = + level2 + s"${TestOptions.exclude.id}.$id = $reason\n" + + private def command31r1(contents: String, describe: Option[String] = None, disabled: Boolean = false): Int = + withConfig(contents)(config => + Main.execute( + command(describe, None, Some(config), revision = Some("3.1r1"), listDisabled = disabled), + new DefaultIntegrationPlugin())) + + test("-describe and -disabled show an exclusion, which must name a procedure of the revision") { + command31r1(withExclusion("8.26.2"), describe = Some("8.26.2")) should equal(0) + command31r1(withExclusion("8.26.2"), describe = Some("8.26")) should equal(0) + command31r1(withExclusion("8.26.2"), disabled = true) should equal(0) + for (id <- List("9.9", "8.26", "all")) { + command31r1(withExclusion(id), describe = Some("8.1.2")) should not equal 0 + command31r1(withExclusion(id), disabled = true) should not equal 0 + } + // and give a reason + command31r1(withExclusion("8.26.2", reason = ""), disabled = true) should not equal 0 + command31r1(level2 + s"${TestOptions.exclude.id} = 8.26.2\n", disabled = true) should not equal 0 + } + + test("a run that excludes a procedure records it, and succeeds") { + val output = Files.createTempDirectory("dnp3c") + val contents = withExclusion("8.26.2") + + s"${TestOptions.outputDirectory.id} = ${output.toString.replace("\\", "/")}\n" + withConfig(contents)(config => + Main.execute( + command(None, None, Some(config), revision = Some("3.1r1"), run = Some("8.26")), + new DefaultIntegrationPlugin())) should equal(0) + val folder = output.resolve("3.1r1").resolve("L2") + SavedResult.read(folder.resolve("8.26.2").resolve("results.properties").toFile).outcome should equal( + ProcedureOutcome.Excluded("Assign Class isn't supported yet")) + Files.readString(folder.resolve("report.html")) should include("Excluded: Assign Class isn't supported yet") + } + + test("a run's end warns of its exclusions, even when a later procedure failed") { + val excluded = ProcedureResult("8.26.2", ProcedureOutcome.Excluded("gap")) + Main.exclusionWarning(RunSummary(List(ProcedureResult("8.1.2", ProcedureOutcome.Passed)), Nil)) should equal(None) + Main.exclusionWarning( + RunSummary(List(excluded, ProcedureResult("8.26.3", ProcedureOutcome.Failed("x"))), List("9.1"))) should equal( + Some("1 procedure(s) excluded and not run. Conformance has not been demonstrated for them.")) + } } diff --git a/src/test/scala/io/stepfunc/dnp3c/conformance/api/OptionsTestSuite.scala b/src/test/scala/io/stepfunc/dnp3c/conformance/api/OptionsTestSuite.scala index 05db539..1ef63cf 100644 --- a/src/test/scala/io/stepfunc/dnp3c/conformance/api/OptionsTestSuite.scala +++ b/src/test/scala/io/stepfunc/dnp3c/conformance/api/OptionsTestSuite.scala @@ -128,6 +128,8 @@ class OptionsTestSuite extends AnyFunSuite with Matchers { test("every default parses back to itself") { for (option <- OptionGroups.list.flatMap(_.options).filterNot(_.required)) option match { + // exclusions have no value of their own, only a property per procedure + case _: ExclusionsOption => case o: TypedTestOption[?] => withClue(o.id)(o.parse(o.defaultAsString) should equal(o.default)) } } @@ -147,6 +149,30 @@ class OptionsTestSuite extends AnyFunSuite with Matchers { // the subset level is written commented out, for you to set Files.readString(path) should include("# outstation.subset.level = ") Options.load(path).get(options.OutstationOptions.subsetLevel) should equal(None) + // and so is an example of an exclusion + Files.readString(path) should include(s"# ${TestOptions.exclude.id}.8.26.2 = ") + Options.load(path).get(TestOptions.exclude) should equal(Map()) + } + + test("each exclusion is a property of its own, naming the procedure and giving the reason") { + val exclusions = Options.load(file( + "test.exclude.8.26.2 = Assign Class isn't supported yet (stepfunc/dnp3#480) ", + "test.exclude.8.6.5b = Relé: 50% done, see #12 — later", + s"${TestOptions.deviceUnderTest.id} = RTU" + )).get(TestOptions.exclude) + exclusions should equal(Map( + "8.26.2" -> "Assign Class isn't supported yet (stepfunc/dnp3#480)", + "8.6.5b" -> "Relé: 50% done, see #12 — later")) + Options.from(Map()).get(TestOptions.exclude) should equal(Map()) + } + + test("an exclusion needs a procedure and a reason, and there's no list of them") { + problems("test.exclude.8.26.2 =", "test.exclude.8.1.2 = ", "test.exclude. = gap", "test.exclude = 8.26.2") + .map(_.key) should equal(List("test.exclude", "test.exclude.", "test.exclude.8.1.2", "test.exclude.8.26.2")) + problems("test.exclude = 8.26.2").map(_.message).head should include("test.exclude. = ") + // keys like it are still unknown + problems("test.excluded.8.26.2 = gap", "test.exclude.8.26.2 = gap").map(_.key) should equal( + List("test.excluded.8.26.2")) } test("writing the default configuration file fails when the write does") { diff --git a/src/test/scala/io/stepfunc/dnp3c/conformance/application/SavedResultTest.scala b/src/test/scala/io/stepfunc/dnp3c/conformance/application/SavedResultTest.scala index be44614..e85a37e 100644 --- a/src/test/scala/io/stepfunc/dnp3c/conformance/application/SavedResultTest.scala +++ b/src/test/scala/io/stepfunc/dnp3c/conformance/application/SavedResultTest.scala @@ -35,4 +35,18 @@ class SavedResultTest extends AnyFunSuite with Matchers { SavedResult.read(results.toFile) should equal(SavedResult(ProcedureOutcome.Passed, None, None)) List(results, directory).foreach(Files.delete) } + + test("an excluded result keeps its reason, and can't be read without one") { + val directory = Files.createTempDirectory("dnp3c") + val results = directory.resolve("results.properties") + val saved = SavedResult(ProcedureOutcome.Excluded("Assign Class isn't supported: #480"), Some("3.1r1"), Some(2)) + SavedResult.write(saved, results) + Files.readString(results) should include("result=excluded") + SavedResult.read(results.toFile) should equal(saved) + for (reason <- List("", "reason=\n", "reason= \n")) { + Files.writeString(results, "result=excluded\nrevision=3.1r1\nsubsetLevel=2\n" + reason) + an[Exception] should be thrownBy SavedResult.read(results.toFile) + } + List(results, directory).foreach(Files.delete) + } } diff --git a/src/test/scala/io/stepfunc/dnp3c/conformance/application/TestRunnerTest.scala b/src/test/scala/io/stepfunc/dnp3c/conformance/application/TestRunnerTest.scala index 6175009..8cbb9d6 100644 --- a/src/test/scala/io/stepfunc/dnp3c/conformance/application/TestRunnerTest.scala +++ b/src/test/scala/io/stepfunc/dnp3c/conformance/application/TestRunnerTest.scala @@ -7,9 +7,9 @@ import java.io.Closeable import java.net.InetSocketAddress import java.nio.file.{ Files, Path } -import io.stepfunc.dnp3c.conformance.{ Main, Revision, Revisions } +import io.stepfunc.dnp3c.conformance.{ Decision, Main, Revision, Revisions, Skip } import io.stepfunc.dnp3c.conformance.api._ -import io.stepfunc.dnp3c.conformance.api.ProcedureOutcome.{ Failed, Passed, Skipped } +import io.stepfunc.dnp3c.conformance.api.ProcedureOutcome.{ Excluded, Failed, Passed, Skipped } import io.stepfunc.dnp3c.conformance.api.options.{ ChannelOptions, NetworkOptions, OutstationOptions, SupportOptions, TestOptions } @@ -173,8 +173,36 @@ class TestRunnerTest extends AnyFunSuite with Matchers { def result(id: String): Option[ProcedureOutcome] = saved(id).map(_.outcome) def logged(id: String): Boolean = Files.exists(folder.resolve(id).resolve("log.html")) + + // at level 2, excluding procedures by id, each with its reason + def excluding(exclusions: (String, String)*): Options = Options.from( + Map(TestOptions.outputDirectory.id -> directory.toString, OutstationOptions.subsetLevel.id -> "2") ++ + exclusions.map((id, reason) => s"${TestOptions.exclude.id}.$id" -> reason)) + + // runs an element of a revision of its own, "t", whose results go in tFolder + def in(revision: Revision, element: TestElement, options: Options): RunSummary = + TestRunner.run(revision, element, options, new Plugin(fail), new Environment(fail)) + + val tFolder: Path = directory.resolve("t").resolve("L2") } + // a revision "t" of one section, "section", which tests levels 1 and 2 + private def revisionOf(procedures: TestProcedure*): Revision = + new Revision("t", "test", "2026-01-01", List(1, 2), List(section(procedures*))) + + // like procedure, but it records when its steps are built + private def configured(name: String, fail: Failures): TestProcedure = + TestProcedure( + name, + name, + new ProcedureDefinition { + override protected def steps(options: OptionSource): Iterable[LabeledStep] = { + fail(s"configure $name") + procedure(name, fail).definition.configure(options).steps + } + } + ) + test("runs a procedure between the plugin's callbacks, and cleans up in reverse") { val run = new Run() run(procedure("a", run.fail)) should equal(RunSummary(List(ProcedureResult("a", Passed)), Nil)) @@ -270,12 +298,113 @@ class TestRunnerTest extends AnyFunSuite with Matchers { SavedResult.write( SavedResult(ProcedureOutcome.Passed, Some("2.7"), Some(2)), folder.resolve("results.properties")) - // the log can't be written: log.html is a folder - Files.createDirectories(folder.resolve("log.html")) + // the log can't be written: log.html is a folder, which can't be deleted as it isn't empty + Files.createDirectories(folder.resolve("log.html").resolve("x")) an[HarnessExecutionException] should be thrownBy run(procedure("a", run.fail, List(SupportOptions.selfAddress))) run.result("a") should equal(None) } + test("an excluded procedure isn't configured or run, opens nothing and calls no plugin method, and the run goes on") { + val run = new Run() + val revision = revisionOf(configured("a", run.fail), configured("b", run.fail), configured("c", run.fail)) + val summary = run.in(revision, revision.all, run.excluding("b" -> "Assign Class isn't supported yet")) + summary should equal(RunSummary( + List( + ProcedureResult("a", Passed), + ProcedureResult("b", Excluded("Assign Class isn't supported yet")), + ProcedureResult("c", Passed)), + Nil)) + // the run succeeded, but the device isn't shown to conform to b + summary.succeeded should equal(true) + summary.exclusions.map(_.id) should equal(List("b")) + run.events.list.filter(_.endsWith(" b")) should equal(Nil) + run.events.list.count(_ == "start") should equal(2) + run.events.list.count(_ == "capture open") should equal(2) + + val saved = SavedResult.read(run.tFolder.resolve("b").resolve("results.properties").toFile) + saved should equal(SavedResult(Excluded("Assign Class isn't supported yet"), Some("t"), Some(2))) + Files.readString(run.tFolder.resolve("b").resolve("log.html")) should include( + "NOT RUN — EXCLUDED b: Assign Class isn't supported yet") + } + + test("a failure after an exclusion still stops the run") { + val run = new Run("step b") + val revision = revisionOf(procedure("a", run.fail), procedure("b", run.fail), procedure("c", run.fail)) + val summary = run.in(revision, revision.all, run.excluding("a" -> "gap")) + summary should equal( + RunSummary(List(ProcedureResult("a", Excluded("gap")), ProcedureResult("b", Failed("step b failed"))), List("c"))) + summary.succeeded should equal(false) + summary.exclusions.map(_.id) should equal(List("a")) + } + + test("an exclusion replaces the procedure's earlier result, log and capture") { + for (earlier <- List(Passed, Failed("step a failed"))) { + val run = new Run() + val revision = revisionOf(procedure("a", run.fail)) + val folder = run.tFolder.resolve("a") + Files.createDirectories(folder) + SavedResult.write(SavedResult(earlier, Some("t"), Some(2), List("a note")), folder.resolve("results.properties")) + Files.writeString(folder.resolve("log.html"), "the earlier log") + Files.writeString(folder.resolve("dump.pcap"), "the earlier capture") + run.in(revision, revision.all, run.excluding("a" -> "gap")) + SavedResult.read(folder.resolve("results.properties").toFile) should equal( + SavedResult(Excluded("gap"), Some("t"), Some(2))) + Files.readString(folder.resolve("log.html")) should not include "the earlier log" + Files.exists(folder.resolve("dump.pcap")) should equal(false) + } + } + + test("an exclusion that fails to be recorded aborts the run, and leaves no earlier pass") { + val run = new Run() + val revision = revisionOf(procedure("a", run.fail), procedure("b", run.fail)) + val folder = run.tFolder.resolve("b") + Files.createDirectories(folder) + SavedResult.write(SavedResult(Passed, Some("t"), Some(2)), folder.resolve("results.properties")) + // the log can't be written: log.html is a folder, which can't be deleted as it isn't empty + Files.createDirectories(folder.resolve("log.html").resolve("x")) + val ex = the[HarnessExecutionException] thrownBy run.in(revision, revision.all, run.excluding("b" -> "gap")) + ex.procedureId should equal("b") + ex.partial should equal(RunSummary(List(ProcedureResult("a", Passed), ProcedureResult("b", Excluded("gap"))), Nil)) + Files.exists(folder.resolve("results.properties")) should equal(false) + } + + test("an exclusion must name a procedure of the revision, whatever the run selects") { + val run = new Run() + val revision = revisionOf(procedure("a", run.fail), procedure("b", run.fail)) + for (id <- List("9.9", "section", "all")) { + val ex = the[ConfigurationException] thrownBy run.in(revision, revision.elements("a"), run.excluding(id -> "gap")) + ex.problems.map(_.key) should equal(List(s"${TestOptions.exclude.id}.$id")) + (the[ConfigurationException] thrownBy revision.checkExclusions(run.excluding(id -> "gap"))).problems should + equal(ex.problems) + } + run.events.list should equal(Nil) + // one outside the selection is fine + run.in(revision, revision.elements("a"), run.excluding("b" -> "gap")).results should equal( + List(ProcedureResult("a", Passed))) + } + + test("a procedure that's skipped stays skipped when it's excluded too, and only those that run need configuring") { + val run = new Run() + def procedure(name: String, levels: Option[Set[Int]] = None) = + TestProcedure( + name, + name, + new ProcedureDefinition { + override protected def steps(options: OptionSource): Iterable[LabeledStep] = Nil + override def configurationProblems(options: OptionSource): List[ConfigProblem] = + List(ConfigProblem(name, "", "needed")) + }, + levels + ) + val revision = revisionOf(procedure("runs"), procedure("excluded"), procedure("level", Some(Set(1)))) + val options = run.excluding("excluded" -> "gap", "level" -> "not at level 2 anyway") + def decision(id: String) = revision.decide(revision.elements(id).asInstanceOf[TestProcedure], options, 2) + decision("runs") should equal(Decision.Run) + decision("excluded") should equal(Decision.Exclude("gap")) + decision("level") should equal(Decision.Skip(Skip.NotApplicable(List(1)))) + (the[ConfigurationException] thrownBy revision.validate(options)).problems.map(_.key) should equal(List("runs")) + } + test("a procedure that doesn't apply at the device's subset level is skipped, saying so") { val run = new Run() val revision = From f7d1e829338b8525496e0337a6bbd0af9b3847a9 Mon Sep 17 00:00:00 2001 From: jadamcrain Date: Thu, 15 Oct 2026 11:39:36 -0700 Subject: [PATCH 2/2] Word the exclusion warning by count, and test what the CLI prints --- .../io/stepfunc/dnp3c/conformance/Main.scala | 4 +- .../application/report/Report.scala | 7 +- .../dnp3c/protocol/logging/Colors.scala | 17 ++- .../dnp3c/conformance/HarnessTest.scala | 2 +- .../stepfunc/dnp3c/conformance/MainTest.scala | 109 +++++++++++++----- .../application/TestRunnerTest.scala | 14 ++- 6 files changed, 111 insertions(+), 42 deletions(-) diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/Main.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/Main.scala index 1aa56bb..4f9396a 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/Main.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/Main.scala @@ -9,6 +9,7 @@ import java.nio.file.Path import io.stepfunc.dnp3c.conformance.api._ import io.stepfunc.dnp3c.conformance.application.pcap.PacketCapture import io.stepfunc.dnp3c.conformance.application.{ Console, Logo, OptionFileWriter } +import io.stepfunc.dnp3c.conformance.application.report.Report import io.stepfunc.dnp3c.dnp3.{ DefaultIntegrationPlugin, IntegrationPlugin } import io.stepfunc.dnp3c.protocol.logging.Colors import org.apache.commons.cli.ParseException @@ -216,8 +217,7 @@ object Main { // how a run's end tells of the procedures it excluded, which it didn't run private[conformance] def exclusionWarning(summary: RunSummary): Option[String] = - Option.when(summary.exclusions.nonEmpty)( - s"${summary.exclusions.size} procedure(s) excluded and not run. Conformance has not been demonstrated for them.") + Option.when(summary.exclusions.nonEmpty)(Report.exclusionWarning(summary.exclusions.size)) // the procedures the configuration excludes that aren't skipped anyway, and why private def exclusions(options: OptionSource, revision: Revision, level: Int): Map[String, String] = diff --git a/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Report.scala b/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Report.scala index 31268f9..d2757f5 100644 --- a/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Report.scala +++ b/src/main/scala/io/stepfunc/dnp3c/conformance/application/report/Report.scala @@ -115,7 +115,7 @@ case class Report(revision: Revision, level: Int, sections: Iterable[Report.Sect if (excluded == 0) Nil else Tags.p(Attributes.classes(Clazz("exclusions")))(List( - s"$excluded procedure(s) excluded and not run. Conformance has not been demonstrated for them.", + s"${Tags.escape(Report.exclusionWarning(excluded))}", "
A skipped procedure doesn't apply to the device or its configuration, and the device can still conform. " + "An excluded procedure does apply, but the configuration left it out of the run (test.exclude)." )) @@ -156,6 +156,11 @@ case class Report(revision: Revision, level: Int, sections: Iterable[Report.Sect object Report { + // what the report, and the end of a run, say of the procedures excluded + def exclusionWarning(count: Int): String = + if (count == 1) "1 procedure was excluded and not run. Conformance has not been demonstrated for it." + else s"$count procedures were excluded and not run. Conformance has not been demonstrated for them." + sealed trait Element case class Procedure( diff --git a/src/main/scala/io/stepfunc/dnp3c/protocol/logging/Colors.scala b/src/main/scala/io/stepfunc/dnp3c/protocol/logging/Colors.scala index 580d0a5..945f408 100644 --- a/src/main/scala/io/stepfunc/dnp3c/protocol/logging/Colors.scala +++ b/src/main/scala/io/stepfunc/dnp3c/protocol/logging/Colors.scala @@ -3,8 +3,12 @@ package io.stepfunc.dnp3c.protocol.logging +import java.io.PrintStream + import org.fusesource.jansi.{ Ansi, AnsiConsole } +import scala.util.DynamicVariable + object Colors { private val width = 120 @@ -19,6 +23,13 @@ object Colors { val incomplete = Colors(Ansi.Color.BLUE, Ansi.Color.WHITE) val plugin = Colors(Ansi.Color.YELLOW, Ansi.Color.BLACK) + // the ANSI console, unless a test reads what's printed + private val stream = new DynamicVariable[Option[PrintStream]](None) + private def out: PrintStream = stream.value.getOrElse(AnsiConsole.out()) + + // prints to a stream instead of the console while f runs, for tests + private[dnp3c] def withOut[T](out: PrintStream)(f: => T): T = stream.withValue(Some(out))(f) + } case class Colors(fg: Ansi.Color, bg: Ansi.Color) { @@ -30,10 +41,10 @@ case class Colors(fg: Ansi.Color, bg: Ansi.Color) { private def build(msg: String): Ansi = Ansi.ansi().bg(bg).fg(fg).a(msg).reset() def printLine(prefix: String, other: Colors, msg: String): Unit = { - AnsiConsole.out().print(build(prefix)) // print the prefix - AnsiConsole.out().println(other.build(pad(msg, Colors.width - prefix.length))) // followed by the message + Colors.out.print(build(prefix)) // print the prefix + Colors.out.println(other.build(pad(msg, Colors.width - prefix.length))) // followed by the message } - def printLine(msg: String): Unit = AnsiConsole.out().println(build(pad(msg, Colors.width))) + def printLine(msg: String): Unit = Colors.out.println(build(pad(msg, Colors.width))) } diff --git a/src/test/scala/io/stepfunc/dnp3c/conformance/HarnessTest.scala b/src/test/scala/io/stepfunc/dnp3c/conformance/HarnessTest.scala index 913167e..3f910b5 100644 --- a/src/test/scala/io/stepfunc/dnp3c/conformance/HarnessTest.scala +++ b/src/test/scala/io/stepfunc/dnp3c/conformance/HarnessTest.scala @@ -116,7 +116,7 @@ class HarnessTest extends AnyFunSuite with Matchers { report should include("""NOT RUN — EXCLUDED""") for (total <- List("1 passed", "0 failed", "1 skipped", "2 excluded", "with no saved result")) report should include(total) - report should include("2 procedure(s) excluded and not run. Conformance has not been demonstrated for them.") + report should include("2 procedures were excluded and not run. Conformance has not been demonstrated for them.") // no traffic for a procedure that didn't run report should include("./8.1.2/dump.pcap") report should not include "./8.3.2/dump.pcap" diff --git a/src/test/scala/io/stepfunc/dnp3c/conformance/MainTest.scala b/src/test/scala/io/stepfunc/dnp3c/conformance/MainTest.scala index 26265ec..7d1d07f 100644 --- a/src/test/scala/io/stepfunc/dnp3c/conformance/MainTest.scala +++ b/src/test/scala/io/stepfunc/dnp3c/conformance/MainTest.scala @@ -3,16 +3,16 @@ package io.stepfunc.dnp3c.conformance -import java.io.File -import java.nio.file.Files +import java.io.{ ByteArrayOutputStream, File, PrintStream } +import java.nio.charset.StandardCharsets +import java.nio.file.{ Files, Path } -import io.stepfunc.dnp3c.conformance.api.{ - BooleanOption, OptionGroups, Options, ProcedureOutcome, ProcedureResult, RunSummary -} +import io.stepfunc.dnp3c.conformance.api.{ BooleanOption, OptionGroups, Options, ProcedureOutcome } import io.stepfunc.dnp3c.conformance.api.options.{ OutstationOptions, SupportOptions, TestOptions } import io.stepfunc.dnp3c.conformance.application.SavedResult import io.stepfunc.dnp3c.conformance.application.Console -import io.stepfunc.dnp3c.dnp3.DefaultIntegrationPlugin +import io.stepfunc.dnp3c.dnp3.{ DefaultIntegrationPlugin, IntegrationPlugin, PluginReporter } +import io.stepfunc.dnp3c.protocol.logging.Colors import org.junit.runner.RunWith import org.scalatestplus.junit.JUnitRunner import org.scalatest.matchers.should.Matchers @@ -159,44 +159,89 @@ class MainTest extends AnyFunSuite with Matchers { private def withExclusion(id: String, reason: String = "Assign Class isn't supported yet") = level2 + s"${TestOptions.exclude.id}.$id = $reason\n" - private def command31r1(contents: String, describe: Option[String] = None, disabled: Boolean = false): Int = - withConfig(contents)(config => + // the exit code, and what the command printed, to the console and to standard output + private def output(command: => Int): (Int, String) = { + val bytes = new ByteArrayOutputStream() + val stream = new PrintStream(bytes, true, StandardCharsets.UTF_8) + val code = Colors.withOut(stream)(scala.Console.withOut(stream)(command)) + (code, bytes.toString(StandardCharsets.UTF_8)) + } + + private def command31r1( + contents: String, + describe: Option[String] = None, + disabled: Boolean = false, + run: Option[String] = None, + plugin: IntegrationPlugin = new DefaultIntegrationPlugin()): (Int, String) = + output(withConfig(contents)(config => Main.execute( - command(describe, None, Some(config), revision = Some("3.1r1"), listDisabled = disabled), - new DefaultIntegrationPlugin())) + command(describe, None, Some(config), revision = Some("3.1r1"), run = run, listDisabled = disabled), + plugin))) + + test("-disabled and -describe show an exclusion's reason, without the procedure's steps") { + val (disabled, listed) = command31r1(withExclusion("8.26.2"), disabled = true) + disabled should equal(0) + listed should include("Procedure: 8.26.2") + listed should include("NOT RUN — EXCLUDED: Assign Class isn't supported yet") + + val (described, procedure) = command31r1(withExclusion("8.26.2"), describe = Some("8.26.2")) + described should equal(0) + procedure should include("NOT RUN — EXCLUDED 8.26.2: Assign Class isn't supported yet") + procedure should not include "step" + + val (section, tree) = command31r1(withExclusion("8.26.2"), describe = Some("8.26")) + section should equal(0) + tree should include("Test Procedures [NOT RUN — EXCLUDED: Assign Class isn't supported yet]") + tree should include("No-Class Assignment Test Procedures") + } - test("-describe and -disabled show an exclusion, which must name a procedure of the revision") { - command31r1(withExclusion("8.26.2"), describe = Some("8.26.2")) should equal(0) - command31r1(withExclusion("8.26.2"), describe = Some("8.26")) should equal(0) - command31r1(withExclusion("8.26.2"), disabled = true) should equal(0) + test("an exclusion must name a procedure of the revision, and give a reason") { for (id <- List("9.9", "8.26", "all")) { - command31r1(withExclusion(id), describe = Some("8.1.2")) should not equal 0 - command31r1(withExclusion(id), disabled = true) should not equal 0 + command31r1(withExclusion(id), describe = Some("8.1.2"))._1 should not equal 0 + command31r1(withExclusion(id), disabled = true)._1 should not equal 0 } - // and give a reason - command31r1(withExclusion("8.26.2", reason = ""), disabled = true) should not equal 0 - command31r1(level2 + s"${TestOptions.exclude.id} = 8.26.2\n", disabled = true) should not equal 0 + command31r1(withExclusion("8.26.2", reason = ""), disabled = true)._1 should not equal 0 + command31r1(level2 + s"${TestOptions.exclude.id} = 8.26.2\n", disabled = true)._1 should not equal 0 } - test("a run that excludes a procedure records it, and succeeds") { + // writes the results in a folder of their own + private def withOutput(contents: String): (String, Path) = { val output = Files.createTempDirectory("dnp3c") - val contents = withExclusion("8.26.2") + - s"${TestOptions.outputDirectory.id} = ${output.toString.replace("\\", "/")}\n" - withConfig(contents)(config => - Main.execute( - command(None, None, Some(config), revision = Some("3.1r1"), run = Some("8.26")), - new DefaultIntegrationPlugin())) should equal(0) + (contents + s"${TestOptions.outputDirectory.id} = ${output.toString.replace("\\", "/")}\n", output) + } + + private val oneExcluded = "1 procedure was excluded and not run. Conformance has not been demonstrated for it." + + test("a run that excludes a procedure records it, says so, and succeeds") { + val (contents, output) = withOutput(withExclusion("8.26.2")) + val (code, printed) = command31r1(contents, run = Some("8.26")) + code should equal(0) + printed should include("NOT RUN — EXCLUDED 8.26.2: Assign Class isn't supported yet") + printed should include(oneExcluded) val folder = output.resolve("3.1r1").resolve("L2") SavedResult.read(folder.resolve("8.26.2").resolve("results.properties").toFile).outcome should equal( ProcedureOutcome.Excluded("Assign Class isn't supported yet")) Files.readString(folder.resolve("report.html")) should include("Excluded: Assign Class isn't supported yet") } - test("a run's end warns of its exclusions, even when a later procedure failed") { - val excluded = ProcedureResult("8.26.2", ProcedureOutcome.Excluded("gap")) - Main.exclusionWarning(RunSummary(List(ProcedureResult("8.1.2", ProcedureOutcome.Passed)), Nil)) should equal(None) - Main.exclusionWarning( - RunSummary(List(excluded, ProcedureResult("8.26.3", ProcedureOutcome.Failed("x"))), List("9.1"))) should equal( - Some("1 procedure(s) excluded and not run. Conformance has not been demonstrated for them.")) + test("a run's end warns of its exclusions, even when a later procedure fails") { + // 6.3.2, after 6.1.2, starts by cycling the DUT's power, which this plugin can't + val plugin = new DefaultIntegrationPlugin { + override def cyclePower(reporter: PluginReporter): Unit = throw new Exception("can't cycle power") + } + val (contents, _) = withOutput(withExclusion("6.1.2", "no link layer access") + "tcp.device.port = 1\n") + val (code, printed) = command31r1(contents, run = Some("6"), plugin = plugin) + code should not equal 0 + printed should include("NOT RUN — EXCLUDED 6.1.2: no link layer access") + printed should include("can't cycle power") + printed.indexOf(oneExcluded) should be > printed.indexOf("Abort due to test failure") + + // or when the plugin fails around one + val failing = new DefaultIntegrationPlugin { + override def startProcedure(reporter: PluginReporter): Unit = throw new Exception("can't start") + } + val (aborted, told) = command31r1(withOutput(withExclusion("6.1.2"))._1, run = Some("6"), plugin = failing) + aborted should not equal 0 + told.indexOf(oneExcluded) should be > told.indexOf("can't start") } } diff --git a/src/test/scala/io/stepfunc/dnp3c/conformance/application/TestRunnerTest.scala b/src/test/scala/io/stepfunc/dnp3c/conformance/application/TestRunnerTest.scala index 8cbb9d6..9631a68 100644 --- a/src/test/scala/io/stepfunc/dnp3c/conformance/application/TestRunnerTest.scala +++ b/src/test/scala/io/stepfunc/dnp3c/conformance/application/TestRunnerTest.scala @@ -317,9 +317,17 @@ class TestRunnerTest extends AnyFunSuite with Matchers { // the run succeeded, but the device isn't shown to conform to b summary.succeeded should equal(true) summary.exclusions.map(_.id) should equal(List("b")) - run.events.list.filter(_.endsWith(" b")) should equal(Nil) - run.events.list.count(_ == "start") should equal(2) - run.events.list.count(_ == "capture open") should equal(2) + // nothing at all for b + def ran(name: String) = List( + "capture open", + "driver open", + "start", + s"configure $name", + s"step $name", + "end", + "driver close", + "capture close") + run.events.list should equal(ran("a") ++ ran("c")) val saved = SavedResult.read(run.tFolder.resolve("b").resolve("results.properties").toFile) saved should equal(SavedResult(Excluded("Assign Class isn't supported yet"), Some("t"), Some(2)))