feat(benchmarks): Added dummy benchmark for testing

packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml

@@ -0,0 +1,13 @@
+# ADO Discovery Space Configuration for Test Benchmark
+# This benchmark tests the echo_experiment custom experiment
+
+# Entity space defining parameter values for the benchmark
+entitySpace:
+  - identifier: input_string
+    propertyDomain:
+      values: ["hello world", "test string", "benchmark test"]
+
+# List of experiments to run
+experiments:
+  - actuatorIdentifier: custom_experiments
+    experimentIdentifier: echo_experiment_2

packages/terratorch/benchmark_packages/test-package/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "test-package"
+version = "0.1.0"
+description = "Custom experiment package for terratorch benchmarks"
+requires-python = ">=3.10"
+dependencies = [
+  "ado-core",
+]
+
+[project.entry-points."ado.custom_experiments"]
+my_experiment = "test_package.experiments"

packages/terratorch/benchmark_packages/test-package/test_package/__init__.py

@@ -0,0 +1,8 @@
+# Copyright IBM Corp. 2026
+# SPDX-License-Identifier: Apache-2.0
+
+"""Test package for custom experiments."""
+
+__version__ = "0.1.0"
+
+# Made with Bob

packages/terratorch/benchmark_packages/test-package/test_package/experiments.py

@@ -0,0 +1,31 @@
+# Copyright IBM Corporation 2025, 2026
+# SPDX-License-Identifier: MIT
+
+"""Custom experiments for terratorch benchmarks."""
+
+from typing import Any
+
+from orchestrator.modules.actuators.custom_experiments import custom_experiment
+
+
+@custom_experiment(output_property_identifiers=["result"])
+def echo_experiment(input_string: str) -> dict[str, Any]:
+    """
+    Simple custom experiment that echoes the input string.
+
+    This is a minimal example demonstrating the structure required
+    by ADO (Automated Design Optimization).
+
+    Parameters
+    ----------
+    input_string : str
+        The string to echo back
+
+    Returns
+    -------
+    dict[str, Any]
+        Dictionary containing 'result' key with the input string
+    """
+    return {
+        "result": input_string,
+    }

packages/terratorch/models/prithvi/benchmark_instances/test_benchmark/space.yaml

@@ -0,0 +1,13 @@
+# ADO Discovery Space Configuration for Test Benchmark
+# This benchmark tests the echo_experiment custom experiment
+
+# Entity space defining parameter values for the benchmark
+entitySpace:
+  - identifier: input_string
+    propertyDomain:
+      values: ["hello world", "test string", "benchmark test"]
+
+# List of experiments to run
+experiments:
+  - actuatorIdentifier: custom_experiments
+    experimentIdentifier: echo_experiment

packages/terratorch/models/terramind_base_fire/benchmark_instances/test_benchmark/space.yaml

@@ -0,0 +1,15 @@
+# ADO Discovery Space Configuration for Test Benchmark
+# This benchmark tests the echo_experiment custom experiment
+
+# Entity space defining parameter values for the benchmark
+entitySpace:
+  - identifier: input_string
+    propertyDomain:
+      values: ["hello world", "test string", "benchmark test"]
+
+# List of experiments to run
+experiments:
+  - actuatorIdentifier: custom_experiments
+    experimentIdentifier: echo_experiment
+
+# Made with Bob

packages/terratorch/nexus.yaml

@@ -1,2 +1,11 @@
 package:
   name: terratorch
+
+  # Register benchmark packages with custom experiments
+  benchmark_packages:
+    - requirement_specifier: "./packages/terratorch/benchmark_packages/test-package"
+      experiments:
+        - "echo_experiment"
+    - requirement_specifier: "https://github.com/christian-pinto/test-ado-custom-experiment"
+      experiments:
+        - "echo_experiment_2"

src/algorithm_nexus/commands/benchmark_manager.py

@@ -711,21 +711,24 @@ def _create_discoveryspace(
         space_config = DiscoverySpaceConfiguration.model_validate(space_config_dict)
 
         # Generate descriptive name and description from instance path
-        # Extract PR number from URL (pr_url may be None in non-PR mode)
-        pr_number = self.pr_url.rstrip("/").split("/")[-1] if self.pr_url else "unknown"
-
         # Parse instance path to get package, model, and instance names
         package_name, model_name, instance_name = self._parse_instance_path(
             instance_path
         )
 
-        # Create descriptive name: space-pr123-package-model-instance
-        space_name = f"space-pr{pr_number}-{package_name}-{model_name}-{instance_name}"
-        space_description = f"Discovery space for benchmark instance from PR #{pr_number}: {package_name}/{model_name}/{instance_name}"
-
-        # Build custom labels with algorithm-nexus fields
+        # Build name, description, and labels depending on whether we are in PR mode
         labels = space_config.metadata.labels or {}
-        labels["algorithm-nexus.pr_url"] = self.pr_url
+        if self.pr_url:
+            pr_number = self.pr_url.rstrip("/").split("/")[-1]
+            space_name = (
+                f"space-pr{pr_number}-{package_name}-{model_name}-{instance_name}"
+            )
+            space_description = f"Discovery space for benchmark instance from PR #{pr_number}: {package_name}/{model_name}/{instance_name}"
+            labels["algorithm-nexus.pr_url"] = self.pr_url
+        else:
+            space_name = f"space-{package_name}-{model_name}-{instance_name}"
+            space_description = f"Discovery space for benchmark instance: {package_name}/{model_name}/{instance_name}"
+
         labels["algorithm-nexus.instance_path"] = str(instance_path)
 
         # Update metadata with descriptive name, description, and labels
@@ -863,35 +866,28 @@ def _create_operation(
         Returns:
             Dictionary with operation_id and ray_job_id (if remote execution)
         """
-        # Generate descriptive name and description from instance path
+        # Generate descriptive name, description, and labels depending on mode
+        custom_metadata = {}
         if instance_path:
-            # Extract PR number from URL (pr_url may be None in non-PR mode)
-            pr_number = (
-                self.pr_url.rstrip("/").split("/")[-1] if self.pr_url else "unknown"
-            )
-
-            # Parse instance path to get package, model, and instance names
             package_name, model_name, instance_name = self._parse_instance_path(
                 instance_path
             )
+            custom_metadata["algorithm-nexus.instance_path"] = str(instance_path)
 
-            # Create descriptive name: randomwalk-pr123-package-model-instance
-            operation_name = (
-                f"randomwalk-pr{pr_number}-{package_name}-{model_name}-{instance_name}"
-            )
-            operation_description = f"Random walk for benchmark instance from PR #{pr_number}: {package_name}/{model_name}/{instance_name}"
+            if self.pr_url:
+                pr_number = self.pr_url.rstrip("/").split("/")[-1]
+                operation_name = f"randomwalk-pr{pr_number}-{package_name}-{model_name}-{instance_name}"
+                operation_description = f"Random walk for benchmark instance from PR #{pr_number}: {package_name}/{model_name}/{instance_name}"
+                custom_metadata["algorithm-nexus.pr_url"] = self.pr_url
+            else:
+                operation_name = (
+                    f"randomwalk-{package_name}-{model_name}-{instance_name}"
+                )
+                operation_description = f"Random walk for benchmark instance: {package_name}/{model_name}/{instance_name}"
         else:
             operation_name = "randomwalk-all"
             operation_description = "Perform a random walk on all points in a space"
 
-        # Create custom metadata with algorithm-nexus fields
-        custom_metadata = {
-            "algorithm-nexus.pr_url": self.pr_url or "",
-            "algorithm-nexus.instance_path": str(instance_path)
-            if instance_path
-            else "",
-        }
-
         # Create operation config using the factory function
         operation_config = create_random_walk_operation_config(
             space_id=space_id,
@@ -967,38 +963,37 @@ def _create_operation(
             "ray_job_id": ray_job_id,
         }
 
-    def run(self) -> dict[str, Any]:
+    def run(
+        self,
+        packages_root: Path | None = None,
+        package_filter: str | None = None,
+    ) -> dict[str, Any]:
         """Main execution method.
 
+        Args:
+            packages_root: Path to packages directory (for all/package mode)
+            package_filter: Optional package name to filter by
+
         Returns:
             Dictionary with execution results
         """
         try:
-            console.print("Analyzing PR for new or changed benchmark instances...")
-            console.print(f"PR URL: {self.pr_url}")
-
-            # Always check if we need to checkout PR code at the beginning
-            if not self.is_local_repo_on_pr_commit():
-                console.print("[yellow]Local repository is not on PR commit[/yellow]")
-                console.print("Checking out PR code to temporary directory...")
-                self.checkout_pr_to_temp()
-            else:
-                console.print(
-                    "[green]✓[/green] Using local repository (already on PR commit)"
-                )
+            # Print mode header
+            self._print_mode_header(packages_root, package_filter, "Executing")
 
-            changed_files = self.get_changed_files()
+            # Discover benchmark instances
+            benchmark_instances = self._discover_instances(
+                packages_root, package_filter
+            )
 
-            benchmark_instances = self.find_benchmark_instances(changed_files)
+            # Print found instances (show list for dry-run/non-execution mode only)
+            self._print_instances_found(
+                benchmark_instances, package_filter, show_list=not self.execute
+            )
 
             if not benchmark_instances:
-                console.print(
-                    "[yellow]No new or changed benchmark instances found in this PR.[/yellow]"
-                )
                 return {"instances": []}
 
-            console.print(f"Found {len(benchmark_instances)} benchmark instance(s):")
-
             results: dict[str, Any] = {
                 "instances": [],
             }
@@ -1013,7 +1008,7 @@ def run(self) -> dict[str, Any]:
                     exec_result = self.execute_benchmark(instance_path)
                     results["instances"].append(exec_result.model_dump())
 
-                    if exec_result.status == "success":
+                    if exec_result.status in ("success", "started"):
                         successful += 1
                     else:
                         failed += 1
@@ -1024,9 +1019,13 @@ def run(self) -> dict[str, Any]:
                 console.print(f"  Successful: {successful}")
                 console.print(f"  Failed: {failed}")
                 console.print("=" * 60)
+
+                results["summary"] = {
+                    "successful": successful,
+                    "failed": failed,
+                }
             else:
                 for instance_path in benchmark_instances:
-                    console.print(f"  {instance_path}")
                     results["instances"].append({"instance_path": str(instance_path)})
 
             return results

src/algorithm_nexus/commands/run.py

@@ -34,12 +34,27 @@
 
 def run_benchmarks(
     pr: Annotated[
-        str,
+        str | None,
         typer.Option(
             "--pr",
-            help="GitHub Pull Request URL (e.g., https://github.com/IBM/algorithm-nexus/pull/123)",
+            help="GitHub Pull Request URL (e.g., https://github.com/IBM/algorithm-nexus/pull/123). "
+            "If not provided, runs benchmark instances from packages.",
+        ),
+    ] = None,
+    packages_root: Annotated[
+        Path,
+        typer.Option(
+            "--packages-root",
+            help="Path to packages directory",
+        ),
+    ] = Path("./packages"),
+    package: Annotated[
+        str | None,
+        typer.Option(
+            "--package",
+            help="Run only benchmark instances from a specific package",
         ),
-    ],
+    ] = None,
     remote: Annotated[
         Path | None,
         typer.Option(
@@ -87,28 +102,60 @@ def run_benchmarks(
         ),
     ] = None,
 ) -> None:
-    """Execute benchmarks from a GitHub Pull Request.
+    """Execute benchmarks.
 
-    Identifies new or changed benchmark instances in a PR and optionally
-    executes them using the ADO CLI. When executing with --remote flag,
-    automatically installs required benchmark packages in the Ray environment.
+    This command supports three modes:
+    1. PR mode: Execute instances modified in a PR (provide --pr)
+    2. All mode: Execute all benchmark instances (no --pr)
+    3. Package mode: Execute instances from a specific package (use --package)
 
-    The command automatically checks if the local repository is on the same
-    commit as the PR. If not, it will checkout the PR code in a temporary
-    directory.
+    Identifies benchmark instances and optionally executes them using the ADO CLI.
+    When executing with --remote flag, automatically installs required benchmark
+    packages in the Ray environment.
     """
     # Validate output format if specified
     if output_format:
         validate_output_format(output_format, allow_yaml=True, allow_csv=False)
 
-    try:
-        manager = BenchmarkManager(
-            pr_url=pr,
-            execute=not dry_run,
-            remote_context_file=remote,
-            context_file=context,
+    # Warn if both package filter and PR URL are provided (package is ignored in PR mode)
+    if package and pr:
+        console.print(
+            "[yellow]Warning:[/yellow] --package is ignored when --pr is specified. "
+            "In PR mode, only instances changed in the PR are executed."
         )
-        results = manager.run()
+
+    # Validate package exists if package filter is specified (non-PR mode only)
+    if package and not pr:
+        package_path = packages_root / package
+        if not package_path.is_dir():
+            console.print(
+                f"[red]Error:[/red] Package '{package}' not found in {packages_root.resolve()}"
+            )
+            console.print(
+                "\nTo see available packages, run: [cyan]nexus list packages[/cyan]"
+            )
+            raise typer.Exit(code=1)
+
+    try:
+        if pr:
+            manager = BenchmarkManager(
+                pr_url=pr,
+                execute=not dry_run,
+                remote_context_file=remote,
+                context_file=context,
+            )
+            results = manager.run()
+        else:
+            manager = BenchmarkManager(
+                pr_url=None,
+                execute=not dry_run,
+                remote_context_file=remote,
+                context_file=context,
+            )
+            results = manager.run(
+                packages_root=packages_root,
+                package_filter=package,
+            )
 
         # Output results
         # Determine output format
@@ -138,6 +185,8 @@ def run_benchmarks(
         if not dry_run and results.get("summary") and results["summary"]["failed"] > 0:
             raise typer.Exit(code=1)
 
+    except typer.Exit:
+        raise
     except KeyboardInterrupt:
         console_err.print("\n[yellow]Interrupted by user[/yellow]")
         raise typer.Exit(code=130)