From 923b57faf4f688bbe8b34a4469fcad83923c9f5e Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 28 Oct 2026 00:57:53 -0700 Subject: [PATCH 1/2] test(workflow-operator): share the sklearn spec template The 51 per-model sklearn descriptor specs each repeated the same five tests. SklearnModelOpDescBehaviors now registers them, and each spec passes its own literal model name and sklearn import. Test names and assertions are unchanged; the specs with extra tests keep them. --- .../SklearnAdaptiveBoostingOpDescSpec.scala | 61 +------- .../sklearn/SklearnBaggingOpDescSpec.scala | 61 +------- ...SklearnBernoulliNaiveBayesOpDescSpec.scala | 66 ++------ ...klearnComplementNaiveBayesOpDescSpec.scala | 64 +------- .../SklearnDecisionTreeOpDescSpec.scala | 62 +------- .../SklearnDummyClassifierOpDescSpec.scala | 61 +------- .../sklearn/SklearnExtraTreeOpDescSpec.scala | 61 +------- .../sklearn/SklearnExtraTreesOpDescSpec.scala | 61 +------- .../SklearnGaussianNaiveBayesOpDescSpec.scala | 63 +------- .../SklearnGradientBoostingOpDescSpec.scala | 61 +------- .../sklearn/SklearnKNNOpDescSpec.scala | 61 +------- .../sklearn/SklearnLinearSVMOpDescSpec.scala | 61 +------- ...klearnLogisticRegressionCVOpDescSpec.scala | 61 +------- .../SklearnLogisticRegressionOpDescSpec.scala | 61 +------- .../sklearn/SklearnModelOpDescBehaviors.scala | 145 ++++++++++++++++++ ...klearnMultiLayerPerceptronOpDescSpec.scala | 61 +------- ...learnMultinomialNaiveBayesOpDescSpec.scala | 64 +------- .../SklearnNearestCentroidOpDescSpec.scala | 61 +------- .../SklearnPassiveAggressiveOpDescSpec.scala | 61 +------- .../sklearn/SklearnPerceptronOpDescSpec.scala | 61 +------- ...earnProbabilityCalibrationOpDescSpec.scala | 61 +------- .../SklearnRandomForestOpDescSpec.scala | 61 +------- .../sklearn/SklearnRidgeCVOpDescSpec.scala | 61 +------- .../sklearn/SklearnRidgeOpDescSpec.scala | 61 +------- .../sklearn/SklearnSDGOpDescSpec.scala | 61 +------- .../sklearn/SklearnSVMOpDescSpec.scala | 57 +------ ...rnTrainingAdaptiveBoostingOpDescSpec.scala | 64 ++------ .../SklearnTrainingBaggingOpDescSpec.scala | 62 +------- ...rainingBernoulliNaiveBayesOpDescSpec.scala | 66 ++------ ...ainingComplementNaiveBayesOpDescSpec.scala | 64 ++------ ...klearnTrainingDecisionTreeOpDescSpec.scala | 62 +------- ...arnTrainingDummyClassifierOpDescSpec.scala | 64 ++------ .../SklearnTrainingExtraTreeOpDescSpec.scala | 62 +------- .../SklearnTrainingExtraTreesOpDescSpec.scala | 62 +------- ...TrainingGaussianNaiveBayesOpDescSpec.scala | 63 ++------ ...rnTrainingGradientBoostingOpDescSpec.scala | 64 ++------ .../SklearnTrainingKNNOpDescSpec.scala | 62 +------- ...rnTrainingLinearRegressionOpDescSpec.scala | 64 ++------ .../SklearnTrainingLinearSVMOpDescSpec.scala | 62 +------- ...ainingLogisticRegressionCVOpDescSpec.scala | 64 ++------ ...TrainingLogisticRegressionOpDescSpec.scala | 64 ++------ ...ainingMultiLayerPerceptronOpDescSpec.scala | 64 ++------ ...iningMultinomialNaiveBayesOpDescSpec.scala | 64 ++------ ...arnTrainingNearestCentroidOpDescSpec.scala | 64 ++------ ...nTrainingPassiveAggressiveOpDescSpec.scala | 64 ++------ .../SklearnTrainingPerceptronOpDescSpec.scala | 62 +------- ...ningProbabilityCalibrationOpDescSpec.scala | 64 ++------ ...klearnTrainingRandomForestOpDescSpec.scala | 62 +------- .../SklearnTrainingRidgeCVOpDescSpec.scala | 62 +------- .../SklearnTrainingRidgeOpDescSpec.scala | 62 +------- .../SklearnTrainingSDGOpDescSpec.scala | 62 +------- .../SklearnTrainingSVMOpDescSpec.scala | 62 +------- 52 files changed, 510 insertions(+), 2811 deletions(-) create mode 100644 common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescBehaviors.scala diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala index 084fa014f3c..726c2048b2c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnAdaptiveBoostingOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnAdaptiveBoostingOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnAdaptiveBoostingOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnAdaptiveBoostingOpDesc).operatorInfo - info.userFriendlyName shouldBe "Adaptive Boosting" - info.operatorDescription shouldBe "Sklearn Adaptive Boosting Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnAdaptiveBoostingOpDesc" should "default its config fields" in { - val d = new SklearnAdaptiveBoostingOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnAdaptiveBoostingOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnAdaptiveBoostingOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnAdaptiveBoostingOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnAdaptiveBoostingOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import AdaBoostClassifier") - code should include("make_pipeline") - code should include("Adaptive Boosting") - } - - "SklearnAdaptiveBoostingOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnAdaptiveBoostingOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnAdaptiveBoosting\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnAdaptiveBoostingOpDesc] - val r = restored.asInstanceOf[SklearnAdaptiveBoostingOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnAdaptiveBoostingOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnAdaptiveBoostingOpDesc, + "Adaptive Boosting", + "from sklearn.ensemble import AdaBoostClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala index bc11cade128..f2557231e74 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBaggingOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnBaggingOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnBaggingOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnBaggingOpDesc).operatorInfo - info.userFriendlyName shouldBe "Bagging" - info.operatorDescription shouldBe "Sklearn Bagging Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnBaggingOpDesc" should "default its config fields" in { - val d = new SklearnBaggingOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnBaggingOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnBaggingOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnBaggingOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnBaggingOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import BaggingClassifier") - code should include("make_pipeline") - code should include("Bagging") - } - - "SklearnBaggingOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnBaggingOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnBagging\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnBaggingOpDesc] - val r = restored.asInstanceOf[SklearnBaggingOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnBaggingOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnBaggingOpDesc, + "Bagging", + "from sklearn.ensemble import BaggingClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala index 96ac3eca767..d3e594e8f40 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnBernoulliNaiveBayesOpDescSpec.scala @@ -19,55 +19,21 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { +class SklearnBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { - "SklearnBernoulliNaiveBayesOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnBernoulliNaiveBayesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Bernoulli Naive Bayes" - info.operatorDescription shouldBe "Sklearn Bernoulli Naive Bayes Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnBernoulliNaiveBayesOpDesc" should "default its config fields" in { - val d = new SklearnBernoulliNaiveBayesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnBernoulliNaiveBayesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnBernoulliNaiveBayesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnBernoulliNaiveBayesOpDesc.generatePythonCode" should - "import and instantiate the BernoulliNB estimator" in { - val d = new SklearnBernoulliNaiveBayesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.naive_bayes import BernoulliNB") - code should include("make_pipeline") - code should include("Bernoulli Naive Bayes") - } + sklearnClassifier( + new SklearnBernoulliNaiveBayesOpDesc, + "Bernoulli Naive Bayes", + "from sklearn.naive_bayes import BernoulliNB", + estimatorTestName = "import and instantiate the BernoulliNB estimator" + ) // The same table statement serves both ports, so training and scoring skip a // row with a missing value alike. - it should "drop rows with missing values before fitting and before scoring" in { + "SklearnBernoulliNaiveBayesOpDesc.generatePythonCode" should + "drop rows with missing values before fitting and before scoring" in { val d = new SklearnBernoulliNaiveBayesOpDesc d.target = "y" d.generatePythonCode() should include("table.dropna()") @@ -97,18 +63,4 @@ class SklearnBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { """print("Skipped", len(in2df) - len(_test), "of", len(in2df), "rows with missing values")""" ) } - - "SklearnBernoulliNaiveBayesOpDesc" should - "round-trip its config fields through the polymorphic base" in { - val d = new SklearnBernoulliNaiveBayesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnBernoulliNaiveBayes\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnBernoulliNaiveBayesOpDesc] - val r = restored.asInstanceOf[SklearnBernoulliNaiveBayesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala index 90b67bdb1ed..0bc62c280d0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnComplementNaiveBayesOpDescSpec.scala @@ -19,63 +19,13 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnComplementNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnComplementNaiveBayesOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnComplementNaiveBayesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Complement Naive Bayes" - info.operatorDescription shouldBe "Sklearn Complement Naive Bayes Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnComplementNaiveBayesOpDesc" should "default its config fields" in { - val d = new SklearnComplementNaiveBayesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnComplementNaiveBayesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnComplementNaiveBayesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnComplementNaiveBayesOpDesc.generatePythonCode" should - "import and instantiate the ComplementNB estimator" in { - val d = new SklearnComplementNaiveBayesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.naive_bayes import ComplementNB") - code should include("make_pipeline") - code should include("Complement Naive Bayes") - } - - "SklearnComplementNaiveBayesOpDesc" should - "round-trip its config fields through the polymorphic base" in { - val d = new SklearnComplementNaiveBayesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnComplementNaiveBayes\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnComplementNaiveBayesOpDesc] - val r = restored.asInstanceOf[SklearnComplementNaiveBayesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnComplementNaiveBayesOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnComplementNaiveBayesOpDesc, + "Complement Naive Bayes", + "from sklearn.naive_bayes import ComplementNB", + estimatorTestName = "import and instantiate the ComplementNB estimator" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala index a2d42832dd4..ad8774364ad 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDecisionTreeOpDescSpec.scala @@ -19,37 +19,19 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnDecisionTreeOpDescSpec extends AnyFlatSpec with Matchers { +class SklearnDecisionTreeOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { - "SklearnDecisionTreeOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnDecisionTreeOpDesc).operatorInfo - info.userFriendlyName shouldBe "Decision Tree" - info.operatorDescription shouldBe "Sklearn Decision Tree Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnDecisionTreeOpDesc" should "default its config fields" in { - val d = new SklearnDecisionTreeOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } + sklearnClassifier( + new SklearnDecisionTreeOpDesc, + "Decision Tree", + "from sklearn.tree import DecisionTreeClassifier" + ) // A split compares, and a comparison can ask whether the value is there before it // asks how large it is, so the row is worth keeping and the count is worth saying. - it should "keep a row with a blank feature and say how many it kept" in { + "SklearnDecisionTreeOpDesc" should "keep a row with a blank feature and say how many it kept" in { val d = new SklearnDecisionTreeOpDesc d.target = "y" val code = d.generatePythonCode() @@ -57,34 +39,4 @@ class SklearnDecisionTreeOpDescSpec extends AnyFlatSpec with Matchers { code should include("dropna(subset=[") code should include("rows_with_gaps") } - - "SklearnDecisionTreeOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnDecisionTreeOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnDecisionTreeOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnDecisionTreeOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.tree import DecisionTreeClassifier") - code should include("make_pipeline") - code should include("Decision Tree") - } - - "SklearnDecisionTreeOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnDecisionTreeOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnDecisionTree\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnDecisionTreeOpDesc] - val r = restored.asInstanceOf[SklearnDecisionTreeOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala index c670069ae79..13c67083f98 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnDummyClassifierOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnDummyClassifierOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnDummyClassifierOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnDummyClassifierOpDesc).operatorInfo - info.userFriendlyName shouldBe "Dummy Classifier" - info.operatorDescription shouldBe "Sklearn Dummy Classifier Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnDummyClassifierOpDesc" should "default its config fields" in { - val d = new SklearnDummyClassifierOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnDummyClassifierOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnDummyClassifierOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnDummyClassifierOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnDummyClassifierOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.dummy import DummyClassifier") - code should include("make_pipeline") - code should include("Dummy Classifier") - } - - "SklearnDummyClassifierOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnDummyClassifierOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnDummyClassifier\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnDummyClassifierOpDesc] - val r = restored.asInstanceOf[SklearnDummyClassifierOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnDummyClassifierOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnDummyClassifierOpDesc, + "Dummy Classifier", + "from sklearn.dummy import DummyClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala index 6c2f424b171..7ce234c7f6f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreeOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnExtraTreeOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnExtraTreeOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnExtraTreeOpDesc).operatorInfo - info.userFriendlyName shouldBe "Extra Tree" - info.operatorDescription shouldBe "Sklearn Extra Tree Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnExtraTreeOpDesc" should "default its config fields" in { - val d = new SklearnExtraTreeOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnExtraTreeOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnExtraTreeOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnExtraTreeOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnExtraTreeOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.tree import ExtraTreeClassifier") - code should include("make_pipeline") - code should include("Extra Tree") - } - - "SklearnExtraTreeOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnExtraTreeOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnExtraTree\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnExtraTreeOpDesc] - val r = restored.asInstanceOf[SklearnExtraTreeOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnExtraTreeOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnExtraTreeOpDesc, + "Extra Tree", + "from sklearn.tree import ExtraTreeClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala index 33de266f3c7..7cc9893a9ae 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnExtraTreesOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnExtraTreesOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnExtraTreesOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnExtraTreesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Extra Trees" - info.operatorDescription shouldBe "Sklearn Extra Trees Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnExtraTreesOpDesc" should "default its config fields" in { - val d = new SklearnExtraTreesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnExtraTreesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnExtraTreesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnExtraTreesOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnExtraTreesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import ExtraTreesClassifier") - code should include("make_pipeline") - code should include("Extra Trees") - } - - "SklearnExtraTreesOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnExtraTreesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnExtraTrees\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnExtraTreesOpDesc] - val r = restored.asInstanceOf[SklearnExtraTreesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnExtraTreesOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnExtraTreesOpDesc, + "Extra Trees", + "from sklearn.ensemble import ExtraTreesClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala index f6020e0a501..6b6fc92f49e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGaussianNaiveBayesOpDescSpec.scala @@ -19,65 +19,16 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { +class SklearnGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { - "SklearnGaussianNaiveBayesOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnGaussianNaiveBayesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Gaussian Naive Bayes" - info.operatorDescription shouldBe "Sklearn Gaussian Naive Bayes Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnGaussianNaiveBayesOpDesc" should "default its config fields" in { - val d = new SklearnGaussianNaiveBayesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnGaussianNaiveBayesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnGaussianNaiveBayesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnGaussianNaiveBayesOpDesc.generatePythonCode" should - "import and instantiate the GaussianNB estimator" in { - val d = new SklearnGaussianNaiveBayesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.naive_bayes import GaussianNB") - code should include("make_pipeline") - code should include("Gaussian Naive Bayes") - } - - "SklearnGaussianNaiveBayesOpDesc" should - "round-trip its config fields through the polymorphic base" in { - val d = new SklearnGaussianNaiveBayesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnGaussianNaiveBayes\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnGaussianNaiveBayesOpDesc] - val r = restored.asInstanceOf[SklearnGaussianNaiveBayesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } + sklearnClassifier( + new SklearnGaussianNaiveBayesOpDesc, + "Gaussian Naive Bayes", + "from sklearn.naive_bayes import GaussianNB", + estimatorTestName = "import and instantiate the GaussianNB estimator" + ) "SklearnGaussianNaiveBayesOpDesc.getOutputSchemas" should "reject Count Vectorizer, naming the estimator and what accepts it instead" in { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala index 67b9de06c80..d0042f4675b 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnGradientBoostingOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnGradientBoostingOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnGradientBoostingOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnGradientBoostingOpDesc).operatorInfo - info.userFriendlyName shouldBe "Gradient Boosting" - info.operatorDescription shouldBe "Sklearn Gradient Boosting Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnGradientBoostingOpDesc" should "default its config fields" in { - val d = new SklearnGradientBoostingOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnGradientBoostingOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnGradientBoostingOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnGradientBoostingOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnGradientBoostingOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import GradientBoostingClassifier") - code should include("make_pipeline") - code should include("Gradient Boosting") - } - - "SklearnGradientBoostingOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnGradientBoostingOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnGradientBoosting\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnGradientBoostingOpDesc] - val r = restored.asInstanceOf[SklearnGradientBoostingOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnGradientBoostingOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnGradientBoostingOpDesc, + "Gradient Boosting", + "from sklearn.ensemble import GradientBoostingClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala index 256a298af94..26ebe81f9f0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnKNNOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnKNNOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnKNNOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnKNNOpDesc).operatorInfo - info.userFriendlyName shouldBe "K-nearest Neighbors" - info.operatorDescription shouldBe "Sklearn K-nearest Neighbors Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnKNNOpDesc" should "default its config fields" in { - val d = new SklearnKNNOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnKNNOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnKNNOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnKNNOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnKNNOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.neighbors import KNeighborsClassifier") - code should include("make_pipeline") - code should include("K-nearest Neighbors") - } - - "SklearnKNNOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnKNNOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnKNN\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnKNNOpDesc] - val r = restored.asInstanceOf[SklearnKNNOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnKNNOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnKNNOpDesc, + "K-nearest Neighbors", + "from sklearn.neighbors import KNeighborsClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala index f09c1f7ed99..ead536b525e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLinearSVMOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnLinearSVMOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnLinearSVMOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnLinearSVMOpDesc).operatorInfo - info.userFriendlyName shouldBe "Linear Support Vector Machine" - info.operatorDescription shouldBe "Sklearn Linear Support Vector Machine Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnLinearSVMOpDesc" should "default its config fields" in { - val d = new SklearnLinearSVMOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnLinearSVMOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnLinearSVMOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnLinearSVMOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnLinearSVMOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.svm import LinearSVC") - code should include("make_pipeline") - code should include("Linear Support Vector Machine") - } - - "SklearnLinearSVMOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnLinearSVMOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnLinearSVM\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnLinearSVMOpDesc] - val r = restored.asInstanceOf[SklearnLinearSVMOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnLinearSVMOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnLinearSVMOpDesc, + "Linear Support Vector Machine", + "from sklearn.svm import LinearSVC" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala index 6bdcb8b99aa..603a59de197 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionCVOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnLogisticRegressionCVOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnLogisticRegressionCVOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnLogisticRegressionCVOpDesc).operatorInfo - info.userFriendlyName shouldBe "Logistic Regression Cross Validation" - info.operatorDescription shouldBe "Sklearn Logistic Regression Cross Validation Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnLogisticRegressionCVOpDesc" should "default its config fields" in { - val d = new SklearnLogisticRegressionCVOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnLogisticRegressionCVOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnLogisticRegressionCVOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnLogisticRegressionCVOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnLogisticRegressionCVOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import LogisticRegressionCV") - code should include("make_pipeline") - code should include("Logistic Regression Cross Validation") - } - - "SklearnLogisticRegressionCVOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnLogisticRegressionCVOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnLogisticRegressionCV\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnLogisticRegressionCVOpDesc] - val r = restored.asInstanceOf[SklearnLogisticRegressionCVOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnLogisticRegressionCVOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnLogisticRegressionCVOpDesc, + "Logistic Regression Cross Validation", + "from sklearn.linear_model import LogisticRegressionCV" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala index 6f6b0e10b58..922fef21a19 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnLogisticRegressionOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnLogisticRegressionOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnLogisticRegressionOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnLogisticRegressionOpDesc).operatorInfo - info.userFriendlyName shouldBe "Logistic Regression" - info.operatorDescription shouldBe "Sklearn Logistic Regression Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnLogisticRegressionOpDesc" should "default its config fields" in { - val d = new SklearnLogisticRegressionOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnLogisticRegressionOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnLogisticRegressionOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnLogisticRegressionOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnLogisticRegressionOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import LogisticRegression") - code should include("make_pipeline") - code should include("Logistic Regression") - } - - "SklearnLogisticRegressionOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnLogisticRegressionOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnLogisticRegression\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnLogisticRegressionOpDesc] - val r = restored.asInstanceOf[SklearnLogisticRegressionOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnLogisticRegressionOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnLogisticRegressionOpDesc, + "Logistic Regression", + "from sklearn.linear_model import LogisticRegression" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescBehaviors.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescBehaviors.scala new file mode 100644 index 00000000000..102a5f8c51e --- /dev/null +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescBehaviors.scala @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.texera.amber.operator.sklearn + +import org.apache.texera.amber.core.tuple.AttributeType +import org.apache.texera.amber.operator.LogicalOp +import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers + +import scala.reflect.ClassTag + +/** + * The five tests every per-model sklearn descriptor spec shares. Each estimator + * ships a classifier descriptor (training and testing ports, Sklearn group) and a + * training-only descriptor (one training port, Sklearn Training group), and both + * pin the same contracts: operatorInfo, the config fields' defaults, the + * model_name/model output schema, the estimator import in the generated code, and + * a JSON round-trip through the polymorphic `LogicalOp` base. + * + * Each spec passes its model name and sklearn import as literals rather than + * reading them back from the descriptor, so a renamed model or a changed import + * still fails that model's own spec. The JSON `operatorType` is the class name + * without its `OpDesc` suffix, the name every sklearn descriptor registers under. + * Test names keep the `[.] should ...` form, so a model's + * spec reports the same names as before. + * + * `newDesc` is by-name: every test builds its own descriptor. + */ +trait SklearnModelOpDescBehaviors extends AnyFlatSpecLike with Matchers { + + /** Registers the shared tests for a classifier descriptor. */ + protected def sklearnClassifier[T <: SklearnModelOpDesc: ClassTag]( + newDesc: => T, + modelName: String, + estimatorImport: String, + estimatorTestName: String = "import the configured sklearn estimator" + ): Unit = + sklearnModel( + newDesc, + modelName, + estimatorImport, + estimatorTestName, + operatorInfoTestName = + "advertise the model name, Sklearn group, and the training/testing port shape", + group = OperatorGroupConstants.SKLEARN_GROUP, + inputPorts = List("training", "testing") + ) + + /** Registers the shared tests for a training-only descriptor. */ + protected def sklearnTrainer[T <: SklearnModelOpDesc: ClassTag]( + newDesc: => T, + modelName: String, + estimatorImport: String + ): Unit = + sklearnModel( + newDesc, + modelName, + estimatorImport, + estimatorTestName = "import the configured sklearn estimator", + operatorInfoTestName = + "advertise the model name, Sklearn Training group, and the single training port", + group = OperatorGroupConstants.SKLEARN_TRAINING_GROUP, + inputPorts = List("training") + ) + + private def sklearnModel[T <: SklearnModelOpDesc: ClassTag]( + newDesc: => T, + modelName: String, + estimatorImport: String, + estimatorTestName: String, + operatorInfoTestName: String, + group: String, + inputPorts: List[String] + ): Unit = { + val descriptor = implicitly[ClassTag[T]].runtimeClass.getSimpleName + val operatorType = descriptor.stripSuffix("OpDesc") + + s"$descriptor.operatorInfo" should operatorInfoTestName in { + val info = newDesc.operatorInfo + info.userFriendlyName shouldBe modelName + info.operatorDescription shouldBe s"Sklearn $modelName Operator" + info.operatorGroupName shouldBe group + info.inputPorts.map(_.displayName) shouldBe inputPorts + info.outputPorts should have length 1 + info.outputPorts.head.blocking shouldBe true + } + + descriptor should "default its config fields" in { + val d = newDesc + d.countVectorizer shouldBe false + d.tfidfTransformer shouldBe false + d.target shouldBe null + d.text shouldBe empty + } + + s"$descriptor.getOutputSchemas" should + "emit the model_name/model schema keyed by the declared output port" in { + val d = newDesc + val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) + schema.getAttribute("model_name").getType shouldBe AttributeType.STRING + schema.getAttribute("model").getType shouldBe AttributeType.BINARY + } + + s"$descriptor.generatePythonCode" should estimatorTestName in { + val d = newDesc + d.target = "y" + val code = d.generatePythonCode() + code should include(estimatorImport) + code should include("make_pipeline") + code should include(modelName) + } + + descriptor should "round-trip its config fields through the polymorphic base" in { + val d = newDesc + d.target = "label" + d.countVectorizer = true + val json = objectMapper.writeValueAsString(d) + json should include("\"operatorType\":\"" + operatorType + "\"") + val restored = objectMapper.readValue(json, classOf[LogicalOp]) + restored shouldBe a[T] + val r = restored.asInstanceOf[SklearnModelOpDesc] + r.target shouldBe "label" + r.countVectorizer shouldBe true + } + } +} diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala index 85a54a28828..b326b695f51 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultiLayerPerceptronOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnMultiLayerPerceptronOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnMultiLayerPerceptronOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnMultiLayerPerceptronOpDesc).operatorInfo - info.userFriendlyName shouldBe "Multi-layer Perceptron" - info.operatorDescription shouldBe "Sklearn Multi-layer Perceptron Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnMultiLayerPerceptronOpDesc" should "default its config fields" in { - val d = new SklearnMultiLayerPerceptronOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnMultiLayerPerceptronOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnMultiLayerPerceptronOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnMultiLayerPerceptronOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnMultiLayerPerceptronOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.neural_network import MLPClassifier") - code should include("make_pipeline") - code should include("Multi-layer Perceptron") - } - - "SklearnMultiLayerPerceptronOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnMultiLayerPerceptronOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnMultiLayerPerceptron\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnMultiLayerPerceptronOpDesc] - val r = restored.asInstanceOf[SklearnMultiLayerPerceptronOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnMultiLayerPerceptronOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnMultiLayerPerceptronOpDesc, + "Multi-layer Perceptron", + "from sklearn.neural_network import MLPClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala index 6573e26cd93..514e9b8a310 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnMultinomialNaiveBayesOpDescSpec.scala @@ -19,63 +19,13 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnMultinomialNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnMultinomialNaiveBayesOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnMultinomialNaiveBayesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Multinomial Naive Bayes" - info.operatorDescription shouldBe "Sklearn Multinomial Naive Bayes Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnMultinomialNaiveBayesOpDesc" should "default its config fields" in { - val d = new SklearnMultinomialNaiveBayesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnMultinomialNaiveBayesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnMultinomialNaiveBayesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnMultinomialNaiveBayesOpDesc.generatePythonCode" should - "import and instantiate the MultinomialNB estimator" in { - val d = new SklearnMultinomialNaiveBayesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.naive_bayes import MultinomialNB") - code should include("make_pipeline") - code should include("Multinomial Naive Bayes") - } - - "SklearnMultinomialNaiveBayesOpDesc" should - "round-trip its config fields through the polymorphic base" in { - val d = new SklearnMultinomialNaiveBayesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnMultinomialNaiveBayes\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnMultinomialNaiveBayesOpDesc] - val r = restored.asInstanceOf[SklearnMultinomialNaiveBayesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnMultinomialNaiveBayesOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnMultinomialNaiveBayesOpDesc, + "Multinomial Naive Bayes", + "from sklearn.naive_bayes import MultinomialNB", + estimatorTestName = "import and instantiate the MultinomialNB estimator" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala index ba173413d57..fcf4740ad51 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnNearestCentroidOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnNearestCentroidOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnNearestCentroidOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnNearestCentroidOpDesc).operatorInfo - info.userFriendlyName shouldBe "Nearest Centroid" - info.operatorDescription shouldBe "Sklearn Nearest Centroid Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnNearestCentroidOpDesc" should "default its config fields" in { - val d = new SklearnNearestCentroidOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnNearestCentroidOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnNearestCentroidOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnNearestCentroidOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnNearestCentroidOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.neighbors import NearestCentroid") - code should include("make_pipeline") - code should include("Nearest Centroid") - } - - "SklearnNearestCentroidOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnNearestCentroidOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnNearestCentroid\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnNearestCentroidOpDesc] - val r = restored.asInstanceOf[SklearnNearestCentroidOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnNearestCentroidOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnNearestCentroidOpDesc, + "Nearest Centroid", + "from sklearn.neighbors import NearestCentroid" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala index 684b008cf7e..25500b466ab 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPassiveAggressiveOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnPassiveAggressiveOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnPassiveAggressiveOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnPassiveAggressiveOpDesc).operatorInfo - info.userFriendlyName shouldBe "Passive Aggressive" - info.operatorDescription shouldBe "Sklearn Passive Aggressive Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnPassiveAggressiveOpDesc" should "default its config fields" in { - val d = new SklearnPassiveAggressiveOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnPassiveAggressiveOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnPassiveAggressiveOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnPassiveAggressiveOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnPassiveAggressiveOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import PassiveAggressiveClassifier") - code should include("make_pipeline") - code should include("Passive Aggressive") - } - - "SklearnPassiveAggressiveOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnPassiveAggressiveOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnPassiveAggressive\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnPassiveAggressiveOpDesc] - val r = restored.asInstanceOf[SklearnPassiveAggressiveOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnPassiveAggressiveOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnPassiveAggressiveOpDesc, + "Passive Aggressive", + "from sklearn.linear_model import PassiveAggressiveClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala index 02b932792ce..80663989d80 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnPerceptronOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnPerceptronOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnPerceptronOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnPerceptronOpDesc).operatorInfo - info.userFriendlyName shouldBe "Linear Perceptron" - info.operatorDescription shouldBe "Sklearn Linear Perceptron Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnPerceptronOpDesc" should "default its config fields" in { - val d = new SklearnPerceptronOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnPerceptronOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnPerceptronOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnPerceptronOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnPerceptronOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import Perceptron") - code should include("make_pipeline") - code should include("Linear Perceptron") - } - - "SklearnPerceptronOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnPerceptronOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnPerceptron\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnPerceptronOpDesc] - val r = restored.asInstanceOf[SklearnPerceptronOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnPerceptronOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnPerceptronOpDesc, + "Linear Perceptron", + "from sklearn.linear_model import Perceptron" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala index 53a30df5b55..0b0cc8e5a3f 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnProbabilityCalibrationOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnProbabilityCalibrationOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnProbabilityCalibrationOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnProbabilityCalibrationOpDesc).operatorInfo - info.userFriendlyName shouldBe "Probability Calibration" - info.operatorDescription shouldBe "Sklearn Probability Calibration Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnProbabilityCalibrationOpDesc" should "default its config fields" in { - val d = new SklearnProbabilityCalibrationOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnProbabilityCalibrationOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnProbabilityCalibrationOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnProbabilityCalibrationOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnProbabilityCalibrationOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.calibration import CalibratedClassifierCV") - code should include("make_pipeline") - code should include("Probability Calibration") - } - - "SklearnProbabilityCalibrationOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnProbabilityCalibrationOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnProbabilityCalibration\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnProbabilityCalibrationOpDesc] - val r = restored.asInstanceOf[SklearnProbabilityCalibrationOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnProbabilityCalibrationOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnProbabilityCalibrationOpDesc, + "Probability Calibration", + "from sklearn.calibration import CalibratedClassifierCV" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala index 63c0f536e5a..c6a66ea1dd0 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRandomForestOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnRandomForestOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnRandomForestOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnRandomForestOpDesc).operatorInfo - info.userFriendlyName shouldBe "Random Forest" - info.operatorDescription shouldBe "Sklearn Random Forest Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnRandomForestOpDesc" should "default its config fields" in { - val d = new SklearnRandomForestOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnRandomForestOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnRandomForestOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnRandomForestOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnRandomForestOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import RandomForestClassifier") - code should include("make_pipeline") - code should include("Random Forest") - } - - "SklearnRandomForestOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnRandomForestOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnRandomForest\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnRandomForestOpDesc] - val r = restored.asInstanceOf[SklearnRandomForestOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnRandomForestOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnRandomForestOpDesc, + "Random Forest", + "from sklearn.ensemble import RandomForestClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala index 05892469d56..845ae4806df 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeCVOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnRidgeCVOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnRidgeCVOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnRidgeCVOpDesc).operatorInfo - info.userFriendlyName shouldBe "Ridge Regression Cross Validation" - info.operatorDescription shouldBe "Sklearn Ridge Regression Cross Validation Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnRidgeCVOpDesc" should "default its config fields" in { - val d = new SklearnRidgeCVOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnRidgeCVOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnRidgeCVOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnRidgeCVOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnRidgeCVOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import RidgeClassifierCV") - code should include("make_pipeline") - code should include("Ridge Regression Cross Validation") - } - - "SklearnRidgeCVOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnRidgeCVOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnRidgeCV\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnRidgeCVOpDesc] - val r = restored.asInstanceOf[SklearnRidgeCVOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnRidgeCVOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnRidgeCVOpDesc, + "Ridge Regression Cross Validation", + "from sklearn.linear_model import RidgeClassifierCV" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala index b13437a7d6e..397609cf543 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnRidgeOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnRidgeOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnRidgeOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnRidgeOpDesc).operatorInfo - info.userFriendlyName shouldBe "Ridge Regression" - info.operatorDescription shouldBe "Sklearn Ridge Regression Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnRidgeOpDesc" should "default its config fields" in { - val d = new SklearnRidgeOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnRidgeOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnRidgeOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnRidgeOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnRidgeOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import RidgeClassifier") - code should include("make_pipeline") - code should include("Ridge Regression") - } - - "SklearnRidgeOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnRidgeOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnRidge\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnRidgeOpDesc] - val r = restored.asInstanceOf[SklearnRidgeOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnRidgeOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnRidgeOpDesc, + "Ridge Regression", + "from sklearn.linear_model import RidgeClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala index cc114667d12..521975f78d4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSDGOpDescSpec.scala @@ -19,61 +19,12 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnSDGOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnSDGOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnSDGOpDesc).operatorInfo - info.userFriendlyName shouldBe "Stochastic Gradient Descent" - info.operatorDescription shouldBe "Sklearn Stochastic Gradient Descent Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnSDGOpDesc" should "default its config fields" in { - val d = new SklearnSDGOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnSDGOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnSDGOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnSDGOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnSDGOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import SGDClassifier") - code should include("make_pipeline") - code should include("Stochastic Gradient Descent") - } - - "SklearnSDGOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnSDGOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnSDG\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnSDGOpDesc] - val r = restored.asInstanceOf[SklearnSDGOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnSDGOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier( + new SklearnSDGOpDesc, + "Stochastic Gradient Descent", + "from sklearn.linear_model import SGDClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala index d88f9c71d85..e6c3c7db82a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnSVMOpDescSpec.scala @@ -19,61 +19,8 @@ package org.apache.texera.amber.operator.sklearn -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnSVMOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnSVMOpDesc.operatorInfo" should - "advertise the model name, Sklearn group, and the training/testing port shape" in { - val info = (new SklearnSVMOpDesc).operatorInfo - info.userFriendlyName shouldBe "Support Vector Machine" - info.operatorDescription shouldBe "Sklearn Support Vector Machine Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training", "testing") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnSVMOpDesc" should "default its config fields" in { - val d = new SklearnSVMOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnSVMOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnSVMOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnSVMOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnSVMOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.svm import SVC") - code should include("make_pipeline") - code should include("Support Vector Machine") - } - - "SklearnSVMOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnSVMOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnSVM\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnSVMOpDesc] - val r = restored.asInstanceOf[SklearnSVMOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnSVMOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnClassifier(new SklearnSVMOpDesc, "Support Vector Machine", "from sklearn.svm import SVC") } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala index dc26343aa8e..4d4269e4bb8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingAdaptiveBoostingOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingAdaptiveBoostingOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingAdaptiveBoostingOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingAdaptiveBoostingOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Adaptive Boosting" - info.operatorDescription shouldBe "Sklearn Training: Adaptive Boosting Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingAdaptiveBoostingOpDesc" should "default its config fields" in { - val d = new SklearnTrainingAdaptiveBoostingOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingAdaptiveBoostingOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingAdaptiveBoostingOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingAdaptiveBoostingOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingAdaptiveBoostingOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import AdaBoostClassifier") - code should include("make_pipeline") - code should include("Training: Adaptive Boosting") - } - - "SklearnTrainingAdaptiveBoostingOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingAdaptiveBoostingOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingAdaptiveBoosting\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingAdaptiveBoostingOpDesc] - val r = restored.asInstanceOf[SklearnTrainingAdaptiveBoostingOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingAdaptiveBoostingOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingAdaptiveBoostingOpDesc, + "Training: Adaptive Boosting", + "from sklearn.ensemble import AdaBoostClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala index 3ac4d1b612b..72dc7bc9b14 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBaggingOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingBaggingOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingBaggingOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingBaggingOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Bagging" - info.operatorDescription shouldBe "Sklearn Training: Bagging Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingBaggingOpDesc" should "default its config fields" in { - val d = new SklearnTrainingBaggingOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingBaggingOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingBaggingOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingBaggingOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingBaggingOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import BaggingClassifier") - code should include("make_pipeline") - code should include("Training: Bagging") - } - - "SklearnTrainingBaggingOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingBaggingOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingBagging\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingBaggingOpDesc] - val r = restored.asInstanceOf[SklearnTrainingBaggingOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingBaggingOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingBaggingOpDesc, + "Training: Bagging", + "from sklearn.ensemble import BaggingClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala index 8bccb3d666c..9a303b17a4c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingBernoulliNaiveBayesOpDescSpec.scala @@ -19,54 +19,23 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { +class SklearnTrainingBernoulliNaiveBayesOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { - "SklearnTrainingBernoulliNaiveBayesOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingBernoulliNaiveBayesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Bernoulli Naive Bayes" - info.operatorDescription shouldBe "Sklearn Training: Bernoulli Naive Bayes Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingBernoulliNaiveBayesOpDesc" should "default its config fields" in { - val d = new SklearnTrainingBernoulliNaiveBayesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingBernoulliNaiveBayesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingBernoulliNaiveBayesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingBernoulliNaiveBayesOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingBernoulliNaiveBayesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.naive_bayes import BernoulliNB") - code should include("make_pipeline") - code should include("Training: Bernoulli Naive Bayes") - } + sklearnTrainer( + new SklearnTrainingBernoulliNaiveBayesOpDesc, + "Training: Bernoulli Naive Bayes", + "from sklearn.naive_bayes import BernoulliNB" + ) // Every column but the target is a feature here, so a row missing any value is // one the estimator cannot be fitted on. - it should "drop rows with missing values before fitting" in { + "SklearnTrainingBernoulliNaiveBayesOpDesc.generatePythonCode" should + "drop rows with missing values before fitting" in { val d = new SklearnTrainingBernoulliNaiveBayesOpDesc d.target = "y" d.generatePythonCode() should include("table.dropna()") @@ -81,17 +50,4 @@ class SklearnTrainingBernoulliNaiveBayesOpDescSpec extends AnyFlatSpec with Matc d.text = List("note") d.generatePythonCode() should include("table.dropna(subset=[") } - - "SklearnTrainingBernoulliNaiveBayesOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingBernoulliNaiveBayesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingBernoulliNaiveBayes\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingBernoulliNaiveBayesOpDesc] - val r = restored.asInstanceOf[SklearnTrainingBernoulliNaiveBayesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala index b79c579beef..9708219033e 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingComplementNaiveBayesOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingComplementNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingComplementNaiveBayesOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingComplementNaiveBayesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Complement Naive Bayes" - info.operatorDescription shouldBe "Sklearn Training: Complement Naive Bayes Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingComplementNaiveBayesOpDesc" should "default its config fields" in { - val d = new SklearnTrainingComplementNaiveBayesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingComplementNaiveBayesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingComplementNaiveBayesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingComplementNaiveBayesOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingComplementNaiveBayesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.naive_bayes import ComplementNB") - code should include("make_pipeline") - code should include("Training: Complement Naive Bayes") - } - - "SklearnTrainingComplementNaiveBayesOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingComplementNaiveBayesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingComplementNaiveBayes\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingComplementNaiveBayesOpDesc] - val r = restored.asInstanceOf[SklearnTrainingComplementNaiveBayesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingComplementNaiveBayesOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingComplementNaiveBayesOpDesc, + "Training: Complement Naive Bayes", + "from sklearn.naive_bayes import ComplementNB" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala index f81a8c40952..75d9f91c738 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDecisionTreeOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingDecisionTreeOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingDecisionTreeOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingDecisionTreeOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Decision Tree" - info.operatorDescription shouldBe "Sklearn Training: Decision Tree Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingDecisionTreeOpDesc" should "default its config fields" in { - val d = new SklearnTrainingDecisionTreeOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingDecisionTreeOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingDecisionTreeOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingDecisionTreeOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingDecisionTreeOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.tree import DecisionTreeClassifier") - code should include("make_pipeline") - code should include("Training: Decision Tree") - } - - "SklearnTrainingDecisionTreeOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingDecisionTreeOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingDecisionTree\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingDecisionTreeOpDesc] - val r = restored.asInstanceOf[SklearnTrainingDecisionTreeOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingDecisionTreeOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingDecisionTreeOpDesc, + "Training: Decision Tree", + "from sklearn.tree import DecisionTreeClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala index fe3e2bb73f4..9c1537b6618 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingDummyClassifierOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingDummyClassifierOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingDummyClassifierOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingDummyClassifierOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Dummy Classifier" - info.operatorDescription shouldBe "Sklearn Training: Dummy Classifier Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingDummyClassifierOpDesc" should "default its config fields" in { - val d = new SklearnTrainingDummyClassifierOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingDummyClassifierOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingDummyClassifierOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingDummyClassifierOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingDummyClassifierOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.dummy import DummyClassifier") - code should include("make_pipeline") - code should include("Training: Dummy Classifier") - } - - "SklearnTrainingDummyClassifierOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingDummyClassifierOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingDummyClassifier\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingDummyClassifierOpDesc] - val r = restored.asInstanceOf[SklearnTrainingDummyClassifierOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingDummyClassifierOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingDummyClassifierOpDesc, + "Training: Dummy Classifier", + "from sklearn.dummy import DummyClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala index 509055f06b4..58e36b2cc3c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreeOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingExtraTreeOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingExtraTreeOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingExtraTreeOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Extra Tree" - info.operatorDescription shouldBe "Sklearn Training: Extra Tree Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingExtraTreeOpDesc" should "default its config fields" in { - val d = new SklearnTrainingExtraTreeOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingExtraTreeOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingExtraTreeOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingExtraTreeOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingExtraTreeOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.tree import ExtraTreeClassifier") - code should include("make_pipeline") - code should include("Training: Extra Tree") - } - - "SklearnTrainingExtraTreeOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingExtraTreeOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingExtraTree\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingExtraTreeOpDesc] - val r = restored.asInstanceOf[SklearnTrainingExtraTreeOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingExtraTreeOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingExtraTreeOpDesc, + "Training: Extra Tree", + "from sklearn.tree import ExtraTreeClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala index b0d9d1e4594..4a87869ee36 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingExtraTreesOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingExtraTreesOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingExtraTreesOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingExtraTreesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Extra Trees" - info.operatorDescription shouldBe "Sklearn Training: Extra Trees Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingExtraTreesOpDesc" should "default its config fields" in { - val d = new SklearnTrainingExtraTreesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingExtraTreesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingExtraTreesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingExtraTreesOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingExtraTreesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import ExtraTreesClassifier") - code should include("make_pipeline") - code should include("Training: Extra Trees") - } - - "SklearnTrainingExtraTreesOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingExtraTreesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingExtraTrees\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingExtraTreesOpDesc] - val r = restored.asInstanceOf[SklearnTrainingExtraTreesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingExtraTreesOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingExtraTreesOpDesc, + "Training: Extra Trees", + "from sklearn.ensemble import ExtraTreesClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala index 101f880387d..7ae399d3a44 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGaussianNaiveBayesOpDescSpec.scala @@ -19,63 +19,18 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingGaussianNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { +class SklearnTrainingGaussianNaiveBayesOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { - "SklearnTrainingGaussianNaiveBayesOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingGaussianNaiveBayesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Gaussian Naive Bayes" - info.operatorDescription shouldBe "Sklearn Training: Gaussian Naive Bayes Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingGaussianNaiveBayesOpDesc" should "default its config fields" in { - val d = new SklearnTrainingGaussianNaiveBayesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingGaussianNaiveBayesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingGaussianNaiveBayesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingGaussianNaiveBayesOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingGaussianNaiveBayesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.naive_bayes import GaussianNB") - code should include("make_pipeline") - code should include("Training: Gaussian Naive Bayes") - } - - "SklearnTrainingGaussianNaiveBayesOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingGaussianNaiveBayesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingGaussianNaiveBayes\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingGaussianNaiveBayesOpDesc] - val r = restored.asInstanceOf[SklearnTrainingGaussianNaiveBayesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } + sklearnTrainer( + new SklearnTrainingGaussianNaiveBayesOpDesc, + "Training: Gaussian Naive Bayes", + "from sklearn.naive_bayes import GaussianNB" + ) "SklearnTrainingGaussianNaiveBayesOpDesc.getOutputSchemas" should "reject Count Vectorizer the same way its classifier counterpart does" in { diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala index 90ec0d15de0..8edc715297a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingGradientBoostingOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingGradientBoostingOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingGradientBoostingOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingGradientBoostingOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Gradient Boosting" - info.operatorDescription shouldBe "Sklearn Training: Gradient Boosting Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingGradientBoostingOpDesc" should "default its config fields" in { - val d = new SklearnTrainingGradientBoostingOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingGradientBoostingOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingGradientBoostingOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingGradientBoostingOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingGradientBoostingOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import GradientBoostingClassifier") - code should include("make_pipeline") - code should include("Training: Gradient Boosting") - } - - "SklearnTrainingGradientBoostingOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingGradientBoostingOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingGradientBoosting\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingGradientBoostingOpDesc] - val r = restored.asInstanceOf[SklearnTrainingGradientBoostingOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingGradientBoostingOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingGradientBoostingOpDesc, + "Training: Gradient Boosting", + "from sklearn.ensemble import GradientBoostingClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala index 383bdec47f8..51551892321 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingKNNOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingKNNOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingKNNOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingKNNOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: K-nearest Neighbors" - info.operatorDescription shouldBe "Sklearn Training: K-nearest Neighbors Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingKNNOpDesc" should "default its config fields" in { - val d = new SklearnTrainingKNNOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingKNNOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingKNNOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingKNNOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingKNNOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.neighbors import KNeighborsClassifier") - code should include("make_pipeline") - code should include("Training: K-nearest Neighbors") - } - - "SklearnTrainingKNNOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingKNNOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingKNN\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingKNNOpDesc] - val r = restored.asInstanceOf[SklearnTrainingKNNOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingKNNOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingKNNOpDesc, + "Training: K-nearest Neighbors", + "from sklearn.neighbors import KNeighborsClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala index 31ae5860a15..05c6fd53db9 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearRegressionOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingLinearRegressionOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingLinearRegressionOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingLinearRegressionOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Linear Regression" - info.operatorDescription shouldBe "Sklearn Training: Linear Regression Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingLinearRegressionOpDesc" should "default its config fields" in { - val d = new SklearnTrainingLinearRegressionOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingLinearRegressionOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingLinearRegressionOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingLinearRegressionOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingLinearRegressionOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import LinearRegression") - code should include("make_pipeline") - code should include("Training: Linear Regression") - } - - "SklearnTrainingLinearRegressionOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingLinearRegressionOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingLinearRegression\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingLinearRegressionOpDesc] - val r = restored.asInstanceOf[SklearnTrainingLinearRegressionOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingLinearRegressionOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingLinearRegressionOpDesc, + "Training: Linear Regression", + "from sklearn.linear_model import LinearRegression" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala index 2ae012556b9..8de28984032 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLinearSVMOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingLinearSVMOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingLinearSVMOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingLinearSVMOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Linear Support Vector Machine" - info.operatorDescription shouldBe "Sklearn Training: Linear Support Vector Machine Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingLinearSVMOpDesc" should "default its config fields" in { - val d = new SklearnTrainingLinearSVMOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingLinearSVMOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingLinearSVMOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingLinearSVMOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingLinearSVMOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.svm import LinearSVC") - code should include("make_pipeline") - code should include("Training: Linear Support Vector Machine") - } - - "SklearnTrainingLinearSVMOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingLinearSVMOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingLinearSVM\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingLinearSVMOpDesc] - val r = restored.asInstanceOf[SklearnTrainingLinearSVMOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingLinearSVMOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingLinearSVMOpDesc, + "Training: Linear Support Vector Machine", + "from sklearn.svm import LinearSVC" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala index 15e11eaa2d8..190373d0916 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionCVOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingLogisticRegressionCVOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingLogisticRegressionCVOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingLogisticRegressionCVOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Logistic Regression Cross Validation" - info.operatorDescription shouldBe "Sklearn Training: Logistic Regression Cross Validation Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingLogisticRegressionCVOpDesc" should "default its config fields" in { - val d = new SklearnTrainingLogisticRegressionCVOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingLogisticRegressionCVOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingLogisticRegressionCVOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingLogisticRegressionCVOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingLogisticRegressionCVOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import LogisticRegressionCV") - code should include("make_pipeline") - code should include("Training: Logistic Regression Cross Validation") - } - - "SklearnTrainingLogisticRegressionCVOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingLogisticRegressionCVOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingLogisticRegressionCV\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingLogisticRegressionCVOpDesc] - val r = restored.asInstanceOf[SklearnTrainingLogisticRegressionCVOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingLogisticRegressionCVOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingLogisticRegressionCVOpDesc, + "Training: Logistic Regression Cross Validation", + "from sklearn.linear_model import LogisticRegressionCV" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala index 1e0b63e562c..5efa295a98a 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingLogisticRegressionOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingLogisticRegressionOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingLogisticRegressionOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingLogisticRegressionOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Logistic Regression" - info.operatorDescription shouldBe "Sklearn Training: Logistic Regression Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingLogisticRegressionOpDesc" should "default its config fields" in { - val d = new SklearnTrainingLogisticRegressionOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingLogisticRegressionOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingLogisticRegressionOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingLogisticRegressionOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingLogisticRegressionOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import LogisticRegression") - code should include("make_pipeline") - code should include("Training: Logistic Regression") - } - - "SklearnTrainingLogisticRegressionOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingLogisticRegressionOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingLogisticRegression\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingLogisticRegressionOpDesc] - val r = restored.asInstanceOf[SklearnTrainingLogisticRegressionOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingLogisticRegressionOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingLogisticRegressionOpDesc, + "Training: Logistic Regression", + "from sklearn.linear_model import LogisticRegression" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala index 2e8faaba08b..6d615078260 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultiLayerPerceptronOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingMultiLayerPerceptronOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingMultiLayerPerceptronOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingMultiLayerPerceptronOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Multi-layer Perceptron" - info.operatorDescription shouldBe "Sklearn Training: Multi-layer Perceptron Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingMultiLayerPerceptronOpDesc" should "default its config fields" in { - val d = new SklearnTrainingMultiLayerPerceptronOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingMultiLayerPerceptronOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingMultiLayerPerceptronOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingMultiLayerPerceptronOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingMultiLayerPerceptronOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.neural_network import MLPClassifier") - code should include("make_pipeline") - code should include("Training: Multi-layer Perceptron") - } - - "SklearnTrainingMultiLayerPerceptronOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingMultiLayerPerceptronOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingMultiLayerPerceptron\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingMultiLayerPerceptronOpDesc] - val r = restored.asInstanceOf[SklearnTrainingMultiLayerPerceptronOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingMultiLayerPerceptronOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingMultiLayerPerceptronOpDesc, + "Training: Multi-layer Perceptron", + "from sklearn.neural_network import MLPClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala index 3ca3c3706a1..d4e7be2f457 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingMultinomialNaiveBayesOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingMultinomialNaiveBayesOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingMultinomialNaiveBayesOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingMultinomialNaiveBayesOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Multinomial Naive Bayes" - info.operatorDescription shouldBe "Sklearn Training: Multinomial Naive Bayes Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingMultinomialNaiveBayesOpDesc" should "default its config fields" in { - val d = new SklearnTrainingMultinomialNaiveBayesOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingMultinomialNaiveBayesOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingMultinomialNaiveBayesOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingMultinomialNaiveBayesOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingMultinomialNaiveBayesOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.naive_bayes import MultinomialNB") - code should include("make_pipeline") - code should include("Training: Multinomial Naive Bayes") - } - - "SklearnTrainingMultinomialNaiveBayesOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingMultinomialNaiveBayesOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingMultinomialNaiveBayes\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingMultinomialNaiveBayesOpDesc] - val r = restored.asInstanceOf[SklearnTrainingMultinomialNaiveBayesOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingMultinomialNaiveBayesOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingMultinomialNaiveBayesOpDesc, + "Training: Multinomial Naive Bayes", + "from sklearn.naive_bayes import MultinomialNB" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala index a9f0d8efbe8..dd3c10a9292 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingNearestCentroidOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingNearestCentroidOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingNearestCentroidOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingNearestCentroidOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Nearest Centroid" - info.operatorDescription shouldBe "Sklearn Training: Nearest Centroid Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingNearestCentroidOpDesc" should "default its config fields" in { - val d = new SklearnTrainingNearestCentroidOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingNearestCentroidOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingNearestCentroidOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingNearestCentroidOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingNearestCentroidOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.neighbors import NearestCentroid") - code should include("make_pipeline") - code should include("Training: Nearest Centroid") - } - - "SklearnTrainingNearestCentroidOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingNearestCentroidOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingNearestCentroid\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingNearestCentroidOpDesc] - val r = restored.asInstanceOf[SklearnTrainingNearestCentroidOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingNearestCentroidOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingNearestCentroidOpDesc, + "Training: Nearest Centroid", + "from sklearn.neighbors import NearestCentroid" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala index dbb0efacefc..fb8f5e3d3c5 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPassiveAggressiveOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingPassiveAggressiveOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingPassiveAggressiveOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingPassiveAggressiveOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Passive Aggressive" - info.operatorDescription shouldBe "Sklearn Training: Passive Aggressive Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingPassiveAggressiveOpDesc" should "default its config fields" in { - val d = new SklearnTrainingPassiveAggressiveOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingPassiveAggressiveOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingPassiveAggressiveOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingPassiveAggressiveOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingPassiveAggressiveOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import PassiveAggressiveClassifier") - code should include("make_pipeline") - code should include("Training: Passive Aggressive") - } - - "SklearnTrainingPassiveAggressiveOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingPassiveAggressiveOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingPassiveAggressive\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingPassiveAggressiveOpDesc] - val r = restored.asInstanceOf[SklearnTrainingPassiveAggressiveOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingPassiveAggressiveOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingPassiveAggressiveOpDesc, + "Training: Passive Aggressive", + "from sklearn.linear_model import PassiveAggressiveClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala index f3641f91743..70e6a0a7fd8 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingPerceptronOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingPerceptronOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingPerceptronOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingPerceptronOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Linear Perceptron" - info.operatorDescription shouldBe "Sklearn Training: Linear Perceptron Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingPerceptronOpDesc" should "default its config fields" in { - val d = new SklearnTrainingPerceptronOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingPerceptronOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingPerceptronOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingPerceptronOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingPerceptronOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import Perceptron") - code should include("make_pipeline") - code should include("Training: Linear Perceptron") - } - - "SklearnTrainingPerceptronOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingPerceptronOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingPerceptron\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingPerceptronOpDesc] - val r = restored.asInstanceOf[SklearnTrainingPerceptronOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingPerceptronOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingPerceptronOpDesc, + "Training: Linear Perceptron", + "from sklearn.linear_model import Perceptron" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala index 58e6a3c6428..e9f1262c018 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingProbabilityCalibrationOpDescSpec.scala @@ -19,61 +19,15 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingProbabilityCalibrationOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingProbabilityCalibrationOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingProbabilityCalibrationOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Probability Calibration" - info.operatorDescription shouldBe "Sklearn Training: Probability Calibration Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingProbabilityCalibrationOpDesc" should "default its config fields" in { - val d = new SklearnTrainingProbabilityCalibrationOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingProbabilityCalibrationOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingProbabilityCalibrationOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingProbabilityCalibrationOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingProbabilityCalibrationOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.calibration import CalibratedClassifierCV") - code should include("make_pipeline") - code should include("Training: Probability Calibration") - } - - "SklearnTrainingProbabilityCalibrationOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingProbabilityCalibrationOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingProbabilityCalibration\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingProbabilityCalibrationOpDesc] - val r = restored.asInstanceOf[SklearnTrainingProbabilityCalibrationOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingProbabilityCalibrationOpDescSpec + extends AnyFlatSpec + with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingProbabilityCalibrationOpDesc, + "Training: Probability Calibration", + "from sklearn.calibration import CalibratedClassifierCV" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala index e41ff84488c..8cd49071777 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRandomForestOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingRandomForestOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingRandomForestOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingRandomForestOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Random Forest" - info.operatorDescription shouldBe "Sklearn Training: Random Forest Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingRandomForestOpDesc" should "default its config fields" in { - val d = new SklearnTrainingRandomForestOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingRandomForestOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingRandomForestOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingRandomForestOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingRandomForestOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.ensemble import RandomForestClassifier") - code should include("make_pipeline") - code should include("Training: Random Forest") - } - - "SklearnTrainingRandomForestOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingRandomForestOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingRandomForest\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingRandomForestOpDesc] - val r = restored.asInstanceOf[SklearnTrainingRandomForestOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingRandomForestOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingRandomForestOpDesc, + "Training: Random Forest", + "from sklearn.ensemble import RandomForestClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala index 8dfed45ee92..618f28955fd 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeCVOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingRidgeCVOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingRidgeCVOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingRidgeCVOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Ridge Regression Cross Validation" - info.operatorDescription shouldBe "Sklearn Training: Ridge Regression Cross Validation Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingRidgeCVOpDesc" should "default its config fields" in { - val d = new SklearnTrainingRidgeCVOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingRidgeCVOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingRidgeCVOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingRidgeCVOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingRidgeCVOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import RidgeClassifierCV") - code should include("make_pipeline") - code should include("Training: Ridge Regression Cross Validation") - } - - "SklearnTrainingRidgeCVOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingRidgeCVOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingRidgeCV\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingRidgeCVOpDesc] - val r = restored.asInstanceOf[SklearnTrainingRidgeCVOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingRidgeCVOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingRidgeCVOpDesc, + "Training: Ridge Regression Cross Validation", + "from sklearn.linear_model import RidgeClassifierCV" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala index 343463e9c0b..558a3687d1c 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingRidgeOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingRidgeOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingRidgeOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingRidgeOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Ridge Regression" - info.operatorDescription shouldBe "Sklearn Training: Ridge Regression Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingRidgeOpDesc" should "default its config fields" in { - val d = new SklearnTrainingRidgeOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingRidgeOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingRidgeOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingRidgeOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingRidgeOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import RidgeClassifier") - code should include("make_pipeline") - code should include("Training: Ridge Regression") - } - - "SklearnTrainingRidgeOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingRidgeOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingRidge\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingRidgeOpDesc] - val r = restored.asInstanceOf[SklearnTrainingRidgeOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingRidgeOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingRidgeOpDesc, + "Training: Ridge Regression", + "from sklearn.linear_model import RidgeClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala index 2c7b0707241..afc11540208 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSDGOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingSDGOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingSDGOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingSDGOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Stochastic Gradient Descent" - info.operatorDescription shouldBe "Sklearn Training: Stochastic Gradient Descent Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingSDGOpDesc" should "default its config fields" in { - val d = new SklearnTrainingSDGOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingSDGOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingSDGOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingSDGOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingSDGOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.linear_model import SGDClassifier") - code should include("make_pipeline") - code should include("Training: Stochastic Gradient Descent") - } - - "SklearnTrainingSDGOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingSDGOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingSDG\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingSDGOpDesc] - val r = restored.asInstanceOf[SklearnTrainingSDGOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingSDGOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingSDGOpDesc, + "Training: Stochastic Gradient Descent", + "from sklearn.linear_model import SGDClassifier" + ) } diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala index 6b7476d7a3b..3ef2d415fc5 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/training/SklearnTrainingSVMOpDescSpec.scala @@ -19,61 +19,13 @@ package org.apache.texera.amber.operator.sklearn.training -import org.apache.texera.amber.core.tuple.AttributeType -import org.apache.texera.amber.operator.LogicalOp -import org.apache.texera.amber.operator.metadata.OperatorGroupConstants -import org.apache.texera.amber.util.JSONUtils.objectMapper +import org.apache.texera.amber.operator.sklearn.SklearnModelOpDescBehaviors import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -class SklearnTrainingSVMOpDescSpec extends AnyFlatSpec with Matchers { - - "SklearnTrainingSVMOpDesc.operatorInfo" should - "advertise the model name, Sklearn Training group, and the single training port" in { - val info = (new SklearnTrainingSVMOpDesc).operatorInfo - info.userFriendlyName shouldBe "Training: Support Vector Machine" - info.operatorDescription shouldBe "Sklearn Training: Support Vector Machine Operator" - info.operatorGroupName shouldBe OperatorGroupConstants.SKLEARN_TRAINING_GROUP - info.inputPorts.map(_.displayName) shouldBe List("training") - info.outputPorts should have length 1 - info.outputPorts.head.blocking shouldBe true - } - - "SklearnTrainingSVMOpDesc" should "default its config fields" in { - val d = new SklearnTrainingSVMOpDesc - d.countVectorizer shouldBe false - d.tfidfTransformer shouldBe false - d.target shouldBe null - d.text shouldBe empty - } - - "SklearnTrainingSVMOpDesc.getOutputSchemas" should - "emit the model_name/model schema keyed by the declared output port" in { - val d = new SklearnTrainingSVMOpDesc - val schema = d.getOutputSchemas(Map.empty)(d.operatorInfo.outputPorts.head.id) - schema.getAttribute("model_name").getType shouldBe AttributeType.STRING - schema.getAttribute("model").getType shouldBe AttributeType.BINARY - } - - "SklearnTrainingSVMOpDesc.generatePythonCode" should "import the configured sklearn estimator" in { - val d = new SklearnTrainingSVMOpDesc - d.target = "y" - val code = d.generatePythonCode() - code should include("from sklearn.svm import SVC") - code should include("make_pipeline") - code should include("Training: Support Vector Machine") - } - - "SklearnTrainingSVMOpDesc" should "round-trip its config fields through the polymorphic base" in { - val d = new SklearnTrainingSVMOpDesc - d.target = "label" - d.countVectorizer = true - val json = objectMapper.writeValueAsString(d) - json should include("\"operatorType\":\"SklearnTrainingSVM\"") - val restored = objectMapper.readValue(json, classOf[LogicalOp]) - restored shouldBe a[SklearnTrainingSVMOpDesc] - val r = restored.asInstanceOf[SklearnTrainingSVMOpDesc] - r.target shouldBe "label" - r.countVectorizer shouldBe true - } +class SklearnTrainingSVMOpDescSpec extends AnyFlatSpec with SklearnModelOpDescBehaviors { + sklearnTrainer( + new SklearnTrainingSVMOpDesc, + "Training: Support Vector Machine", + "from sklearn.svm import SVC" + ) } From 49580b5194836d839a67a9e22e57dab4db0641d2 Mon Sep 17 00:00:00 2001 From: Xinyuan Lin Date: Wed, 28 Oct 2026 01:00:06 -0700 Subject: [PATCH 2/2] test(workflow-operator): bound the sklearn behaviors to each family --- .../amber/operator/sklearn/SklearnModelOpDescBehaviors.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescBehaviors.scala b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescBehaviors.scala index 102a5f8c51e..5554b7f0fc4 100644 --- a/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescBehaviors.scala +++ b/common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/sklearn/SklearnModelOpDescBehaviors.scala @@ -22,6 +22,7 @@ package org.apache.texera.amber.operator.sklearn import org.apache.texera.amber.core.tuple.AttributeType import org.apache.texera.amber.operator.LogicalOp import org.apache.texera.amber.operator.metadata.OperatorGroupConstants +import org.apache.texera.amber.operator.sklearn.training.SklearnTrainingOpDesc import org.apache.texera.amber.util.JSONUtils.objectMapper import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers @@ -48,7 +49,7 @@ import scala.reflect.ClassTag trait SklearnModelOpDescBehaviors extends AnyFlatSpecLike with Matchers { /** Registers the shared tests for a classifier descriptor. */ - protected def sklearnClassifier[T <: SklearnModelOpDesc: ClassTag]( + protected def sklearnClassifier[T <: SklearnClassifierOpDesc: ClassTag]( newDesc: => T, modelName: String, estimatorImport: String, @@ -66,7 +67,7 @@ trait SklearnModelOpDescBehaviors extends AnyFlatSpecLike with Matchers { ) /** Registers the shared tests for a training-only descriptor. */ - protected def sklearnTrainer[T <: SklearnModelOpDesc: ClassTag]( + protected def sklearnTrainer[T <: SklearnTrainingOpDesc: ClassTag]( newDesc: => T, modelName: String, estimatorImport: String