From 6e8884232ba84cc0589d4e608a4a36baa0c5d8b5 Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Fri, 3 Jul 2026 15:27:10 +0100 Subject: [PATCH 1/2] Add app type hint to cli shell function --- src/buildstream/_frontend/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 8e42cca7e..e269ef7aa 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -20,6 +20,7 @@ import shutil import click from .. import _yaml +from .._frontend.app import App from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope @@ -723,7 +724,7 @@ def show(app, elements, deps, except_, order, format_): @click.argument("command", type=click.STRING, nargs=-1) @click.pass_obj def shell( - app, + app: App, target, command, mount, From 311b613b4071c6c518d9b0ff4cd8c40139507467 Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Fri, 3 Jul 2026 15:34:10 +0100 Subject: [PATCH 2/2] Shell API and CLI: Add option for staging additional runtime targets This enables users to add additional functionality such as debug tooling in the shell sandbox, without needing to modify the target element. This is achived through introducing a new option to shell. All existing API and UX is maintained, to not break existing scripts. An alternative design was considered, to have the additional elements as positional arguments similar to the existing element, but this would need manual parsing to handle the cases where `--` is present and not present, splitting based on a `.bst` suffix. This UX could be re-visited in future. Example usage: bst shell --with base.bst example.bst -- cat example.txt Where: - example.bst is a simple import element with no dependencies that imports a file called example.txt - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc). Changes: - Introduces `test_with_other_targets` integration test to the shell test suite. - Adds `--with` cli option to the shell subcommand and updates it's documentation. - option can be used multiple times by caller, providing a list of targets. - Extends the shell top level calling interface in Buildstream core to accept a list of other_targets - This is where the targets are loaded into elements and checked to make sure they are present - Extends the shell element implementation to accept a list of other targets - This is where the other elements are staged and integrated into the sandbox --- src/buildstream/_frontend/cli.py | 80 ++++++++++++++++++++++++------ src/buildstream/_loader/loader.py | 6 ++- src/buildstream/_stream.py | 81 +++++++++++++++++++++++++++++-- src/buildstream/_yaml.pyi | 3 ++ src/buildstream/element.py | 5 ++ src/buildstream/types.pyi | 8 +-- tests/integration/shell.py | 46 +++++++++++++++++- 7 files changed, 201 insertions(+), 28 deletions(-) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index e269ef7aa..2f7ebfe44 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -378,8 +378,6 @@ def cli(context, **kwargs): user preferences configuration file. """ - from .app import App - # Create the App, giving it the main arguments context.obj = App.create(dict(kwargs)) context.call_on_close(context.obj.cleanup) @@ -687,6 +685,13 @@ def show(app, elements, deps, except_, order, format_): metavar="HOSTPATH PATH", help="Mount a file or directory into the sandbox", ) +@click.option( + "--with", + "other_targets", + type=click.Path(readable=False), + multiple=True, + help="A additional target to stage into an element's sandbox environment", +) @click.option("--isolate", is_flag=True, help="Create an isolated build sandbox") @click.option( "--use-buildtree", @@ -726,6 +731,7 @@ def show(app, elements, deps, except_, order, format_): def shell( app: App, target, + other_targets, command, mount, isolate, @@ -749,13 +755,38 @@ def shell( otherwise bst may respond to them instead. e.g. \b - bst shell example.bst -- df -h + bst shell base.bst -- df -h Use the --build option to create a temporary sysroot for building the element instead. + Use the --with option to stage the artifacts of other elements + into the temporary sysroot to make them available to run e.g. + + \b + bst shell --with base.bst example.bst -- cat example.txt + If no COMMAND is specified, the default is to attempt to run an interactive shell. + + # Examples: + + \b + # Attempt to run an interactive shell with example.bst + bst shell example.bst + # Attempt to run an df -h with example.bst + bst shell example.bst -- df h + # In a workspace directory, attempt to shell into the workspace element + bst shell + # Attempt to run cat from base.bst to read example.txt from example.bst + bst shell --with base.bst example.bst -- cat example.txt + # Attempt to run an interactive shell with the sources and all dependencies of example.bst + bst shell --build example.bst + + For all examples on this page: + - example.bst is a simple import element with no dependencies + that imports a file called example.txt + - base.bst provides a basic alpine sysroot with a standard set of unix tooling (sh, df, cat etc). """ # Buildtree can only be used with build shells @@ -765,6 +796,7 @@ def shell( scope = _Scope.BUILD if build_ else _Scope.RUN with app.initialized(): + assert app.stream, "Must have Stream initialised" if not target: target = app.stream.get_default_target() if not target: @@ -773,19 +805,35 @@ def shell( mounts = [_HostMount(path, host_path) for host_path, path in mount] try: - exitcode = app.stream.shell( - target, - scope, - app.shell_prompt, - mounts=mounts, - isolate=isolate, - command=command, - usebuildtree=cli_buildtree, - artifact_remotes=artifact_remotes, - source_remotes=source_remotes, - ignore_project_artifact_remotes=ignore_project_artifact_remotes, - ignore_project_source_remotes=ignore_project_source_remotes, - ) + if other_targets: + exitcode = app.stream.shell_with( + target, + other_targets, + scope, + app.shell_prompt, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=cli_buildtree, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) + else: + exitcode = app.stream.shell( + target, + scope, + app.shell_prompt, + mounts=mounts, + isolate=isolate, + command=command, + usebuildtree=cli_buildtree, + artifact_remotes=artifact_remotes, + source_remotes=source_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ignore_project_source_remotes=ignore_project_source_remotes, + ) except BstError as e: raise AppError("Error launching shell: {}".format(e), detail=e.detail, reason=e.reason) from e diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index af98504cc..4bdd53905 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -21,7 +21,7 @@ from ..exceptions import LoadErrorReason from .. import _yaml from ..element import Element -from ..node import Node +from ..node import Node, MappingNode from .._profile import Topics, PROFILER from .._includes import Includes from .._utils import valid_chars_name @@ -1014,7 +1014,9 @@ def _shallow_load_path(self, path, provenance_node): # - (str): name of the element # - (Loader): loader for sub-project # - def _parse_name(self, name, provenance_node, *, load_subprojects=True): + def _parse_name( + self, name: str, provenance_node: MappingNode, *, load_subprojects: bool = True + ) -> tuple[str | None, str, "Loader"]: # We allow to split only once since deep junctions names are forbidden. # Users who want to refer to elements in sub-sub-projects are required # to create junctions on the top level project. diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 91971073f..3d98be0fc 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -16,6 +16,7 @@ # Jürg Billeter # Tristan Maat + import itertools import os import sys @@ -27,7 +28,11 @@ from contextlib import contextmanager, suppress from collections import deque from typing import List, Tuple, Optional, Iterable, Callable +from ruamel.yaml import CommentedMap + +from ._context import Context +from .node import MappingNode from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError @@ -44,7 +49,7 @@ ) from .element import Element from ._profile import Topics, PROFILER -from ._project import ProjectRefStorage +from ._project import ProjectRefStorage, Project from ._remotespec import RemoteSpec from ._state import State from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount @@ -79,11 +84,11 @@ def __init__( # # Private members # - self._context = context + self._context: Context = context self._artifacts = None self._elementsourcescache = None self._sourcecache = None - self._project = None + self._project: Optional[Project] = None self._state = State(session_start) # Owned by Stream, used by Core to set state self._notification_queue = deque() @@ -163,7 +168,7 @@ def load_selection( ignore_project_artifact_remotes: bool = False, ignore_project_source_remotes: bool = False, need_state: bool = True, - ): + ) -> list[Element]: with PROFILER.profile(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, "-") for t in targets)): target_objects = self._load( targets, @@ -235,6 +240,69 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source task.add_current_progress() + # shell_with() + # + # Run a shell with other targets. + # + # Automatically creates a temporary target based on 'target' with 'other_targets' as runtime or build dependencies. + # + # Note: Method will build the temporary target, before entering into it's shell. + # + # Args: + # target (str): The name of the element to run the shell for + # other_targets: (Iterable[str]): The name of the other elements to run the shell with. + # scope: _Scope: Either BUILD or RUN + # *args, **kwargs: Passed to shell() untouched. + # + # Returns: + # (int): The exit code of the launched shell + # + def shell_with(self, target: str, other_targets: Iterable[str], scope: _Scope, *args, **kwargs): + + assert self._project, "Must have a project" + assert self._project.loader, "Project must have loader" + + target_junction, target_name, target_loader = self._project.loader._parse_name( + target, MappingNode.from_dict({}) + ) + + target_path = os.path.join(target_loader._basedir, target_name) + target_node: CommentedMap = _yaml.roundtrip_load(target_path) + + if scope == _Scope.RUN: + r_depends = target_node.get("runtime-depends", []) + + for other_target in other_targets: + r_depends.append(other_target) + + target_node["runtime-depends"] = r_depends + elif scope == _Scope.BUILD: + r_depends = target_node.get("build-depends", []) + + for other_target in other_targets: + r_depends.append(other_target) + + target_node["build-depends"] = r_depends + else: + raise StreamError( + "Only BUILD and RUN scopes are supported", + detail="Use the --build and --use-buildtree options to shell into a build tree", + reason="only-build-run-supported", + ) + + with tempfile.NamedTemporaryFile( + dir=target_loader._basedir, delete_on_close=False, prefix=f"{target_name}_temp", suffix=".bst" + ) as temp_target_file: + _yaml.roundtrip_dump(target_node, temp_target_file) + temp_target_file.close() # delete_on_close is false so this doesn't remove the file, but delete is True(default) so we delete the file when we leave the context manager. + + new_target = os.path.relpath(temp_target_file.name, target_loader._basedir) + if target_junction: + new_target = f"{target_junction}:{new_target}" + + self.build([new_target]) + return self.shell(new_target, scope, *args, **kwargs) + # shell() # # Run a shell @@ -243,6 +311,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source # target: The name of the element to run the shell for # scope: The scope for the shell, only BUILD or RUN are valid (_Scope) # prompt: A function to return the prompt to display in the shell + # other_targets (Iterable[str]): The name of other elements to stage in the shell # unique_id: (str): A unique_id to use to lookup an Element instance # mounts: Additional directories to mount into the sandbox # isolate (bool): Whether to isolate the environment like we do in builds @@ -1043,6 +1112,7 @@ def workspace_open( self.workspace_close(target._get_full_name(), remove_dir=not no_checkout) if not custom_dir: + assert self._context.workspacedir, "Must have workspace dir" directory = os.path.abspath(os.path.join(self._context.workspacedir, target.name)) if directory[-4:] == ".bst": directory = directory[:-4] @@ -2114,6 +2184,8 @@ def _expand_and_classify_targets( # project directory and element path prefix, to produce only element names. # all_elements = [] + assert self._project, "Must have a project" + assert self._project.element_path, "Must have a project" element_path_length = len(self._project.element_path) + 1 for dirpath, _, filenames in os.walk(self._project.element_path): for filename in filenames: @@ -2133,6 +2205,7 @@ def _expand_and_classify_targets( # Glob the artifact names and add the results to the set # + assert self._artifacts, "Must have artifacts" for glob in artifact_globs: glob_results = self._artifacts.list_artifacts(glob=glob) for artifact_name in glob_results: diff --git a/src/buildstream/_yaml.pyi b/src/buildstream/_yaml.pyi index 224abc51a..b301cab68 100644 --- a/src/buildstream/_yaml.pyi +++ b/src/buildstream/_yaml.pyi @@ -12,7 +12,10 @@ # limitations under the License. # from typing import Optional +from ruamel.yaml import CommentedMap from .node import MappingNode def load(filename: str, shortname: str, copy_tree: bool = False, project: Optional[object] = None) -> MappingNode: ... +def roundtrip_load(filename: str, *, allow_missing: bool = False) -> CommentedMap: ... +def roundtrip_dump(contents, file): ... diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 209b9e9a9..62d74eb1d 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -62,6 +62,9 @@ --------------- """ +# For 3.7+ support, not necessary and deprecated in 3.14+ +from __future__ import annotations + import os import re import stat @@ -74,6 +77,7 @@ from threading import Lock from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence + from pyroaring import BitMap # pylint: disable=no-name-in-module from . import _yaml @@ -2058,6 +2062,7 @@ def _push(self): # prompt (str): A suitable prompt string for PS1 # command (list): An argv to launch in the sandbox # usebuildtree (bool): Use the buildtree as its source + # other_elements (List[Element]): Optional list of other runtime elements to stage in the sandbox # # Returns: Exit code def _shell( diff --git a/src/buildstream/types.pyi b/src/buildstream/types.pyi index 8a75b9f13..db9d59d06 100644 --- a/src/buildstream/types.pyi +++ b/src/buildstream/types.pyi @@ -116,10 +116,10 @@ class OverlapAction(Enum): IGNORE: str class _Scope(Enum): - ALL: int - BUILD: int - RUN: int - NONE: int + ALL = 1 + BUILD = 2 + RUN = 3 + NONE = 4 class _KeyStrength(Enum): STRONG: int diff --git a/tests/integration/shell.py b/tests/integration/shell.py index 63c7af8b3..4d0b4ce6b 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -16,12 +16,15 @@ # pylint: disable=redefined-outer-name import os +from typing import Dict, List, Tuple import uuid + import pytest from buildstream import _yaml from buildstream._testing import cli_integration as cli # pylint: disable=unused-import +from buildstream._testing.runcli import CliIntegration from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN from buildstream.exceptions import ErrorDomain from buildstream import utils @@ -46,11 +49,25 @@ # mount (tuple): A (host, target) tuple for the `--mount` option # element (str): The element to build and run a shell with # isolate (bool): Whether to pass --isolate to `bst shell` -# -def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False): +# other_elements (list(str)): Other elements to stage in the sandbox +def execute_shell( + cli: CliIntegration, + project: str, + command: List[str], + *, + config: None | Dict = None, + mount: Tuple[str, str] | None = None, + element: str = "base.bst", + isolate: bool = False, + other_elements: List[str] | None = None +): # Ensure the element is built result = cli.run_project_config(project=project, project_config=config, args=["build", element]) assert result.exit_code == 0 + if other_elements is not None: + for other_element in other_elements: + result = cli.run_project_config(project=project, project_config=config, args=["build", other_element]) + assert result.exit_code == 0 args = ["shell"] if isolate: @@ -58,6 +75,9 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element="ba if mount is not None: host_path, target_path = mount args += ["--mount", host_path, target_path] + if other_elements is not None: + for other_element in other_elements: + args += ["--with", other_element] args += [element, "--", *command] return cli.run_project_config(project=project, project_config=config, args=args) @@ -86,6 +106,28 @@ def test_executable(cli, datafiles): assert result.output == "Horseys!\n" +# Test staging and running additional targets in the shell of the main target for debugging. +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_with_other_targets(cli, datafiles): + project = str(datafiles) + + # Show we can't cat in a shell for manual/import-file.bst + result = execute_shell(cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst") + assert ( + result.exit_code == -1 + ), "Shouldn't be able to read content of test.txt as manual/import-file.bst is a simple import element with no dependencies" + + # Show we can now cat with base.bst in a shell for manual/import-file.bst + result = execute_shell( + cli, project, ["/bin/cat", "test.txt"], element="manual/import-file.bst", other_elements=["base.bst"] + ) + assert ( + result.exit_code == 0 + ), "Should be able to read content of test.txt as we now stage in base.bst that provides /bin/cat" + assert result.output == "This is a test\n" + + # Test shell environment variable explicit assignments @pytest.mark.parametrize("animal", [("Horse"), ("Pony")]) @pytest.mark.datafiles(DATA_DIR)