diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py index 075be8fe3..61f983bc0 100644 --- a/src/buildstream/_artifact.py +++ b/src/buildstream/_artifact.py @@ -26,17 +26,24 @@ import os import tempfile -from typing import Dict, Tuple +from typing import Dict, Tuple, Optional, TYPE_CHECKING +from ._cas.cascache import CASCache +from .sandbox.sandbox import Sandbox +from .storage.directory import Directory from ._protos.buildstream.v2.artifact_pb2 import Artifact as ArtifactProto from . import _yaml from . import utils -from .node import Node +from .node import Node, MappingNode from .types import _Scope from .storage._casbaseddirectory import CasBasedDirectory from .sandbox._config import SandboxConfig from ._variables import Variables +if TYPE_CHECKING: + from .element import Element + from ._context import Context + # An Artifact class to abstract artifact operations # from the Element class @@ -52,22 +59,33 @@ class Artifact: version = 2 - def __init__(self, element, context, *, strong_key=None, strict_key=None, weak_key=None): - self._element = element - self._context = context - self._cache_key = strong_key - self._strict_key = strict_key - self._weak_cache_key = weak_key - self._artifactdir = context.artifactdir - self._cas = context.get_cascache() - self._tmpdir = context.tmpdir - self._proto = None - - self._metadata_keys = None # Strong, strict and weak key tuple extracted from the artifact - self._metadata_dependencies = None # Dictionary of dependency strong keys from the artifact - self._metadata_workspaced = None # Boolean of whether it's a workspaced artifact - self._metadata_workspaced_dependencies = None # List of which dependencies are workspaced from the artifact - self._cached = None # Boolean of whether the artifact is cached + def __init__( + self, + element: "Element", + context: "Context", + *, + strong_key: Optional[str] = None, + strict_key: Optional[str] = None, + weak_key: Optional[str] = None, + ): + self._element: "Element" = element + self._context: "Context" = context + self._cache_key: Optional[str] = strong_key + self._strict_key: Optional[str] = strict_key + self._weak_cache_key: Optional[str] = weak_key + self._artifactdir: Optional[str] = context.artifactdir + self._cas: CASCache = context.get_cascache() + self._tmpdir: Optional[str] = context.tmpdir + self._proto: Optional[ArtifactProto] = None + + self._metadata_keys: Optional[tuple[str, str, str]] = ( + None # Strong, strict and weak key tuple extracted from the artifact + ) + self._metadata_workspaced: Optional[bool] = None # Boolean of whether it's a workspaced artifact + self._metadata_workspaced_dependencies: Optional[list[str]] = ( + None # List of which dependencies are workspaced from the artifact + ) + self._cached: Optional[bool] = None # Boolean of whether the artifact is cached # strong_key(): # @@ -76,7 +94,8 @@ def __init__(self, element, context, *, strong_key=None, strict_key=None, weak_k # or whether it was the strong key loaded from artifact metadata. # @property - def strong_key(self) -> str: + def strong_key(self) -> Optional[str]: + key: str | None if self.cached(): key, _, _ = self.get_metadata_keys() else: @@ -91,7 +110,8 @@ def strong_key(self) -> str: # or whether it was the strict key loaded from artifact metadata. # @property - def strict_key(self) -> str: + def strict_key(self) -> Optional[str]: + key: str | None if self.cached(): _, key, _ = self.get_metadata_keys() else: @@ -106,7 +126,8 @@ def strict_key(self) -> str: # or whether it was the weak key loaded from artifact metadata. # @property - def weak_key(self) -> str: + def weak_key(self) -> Optional[str]: + key: str | None if self.cached(): _, _, key = self.get_metadata_keys() else: @@ -121,7 +142,7 @@ def weak_key(self) -> str: # Returns: # (Directory): The virtual directory object # - def get_files(self): + def get_files(self) -> Directory: files_digest = self._get_field_digest("files") return CasBasedDirectory(self._cas, digest=files_digest) @@ -235,6 +256,9 @@ def cache( artifact.build_error_details = "" if not buildresult[2] else buildresult[2] # Store keys + assert self._cache_key, "Key should be ready by now" + assert self._strict_key, "Key should be ready by now" + assert self._weak_cache_key, "Key should be ready by now" artifact.strong_key = self._cache_key artifact.strict_key = self._strict_key artifact.weak_key = self._weak_cache_key @@ -292,7 +316,7 @@ def cache( digests = self._cas.add_objects(paths=[entry[0] for entry in files_to_capture]) # add_objects() should guarantee this. # `zip(..., strict=True)` could be used in Python 3.10+ - assert len(files_to_capture) == len(digests) + assert len(files_to_capture) == len(digests), "files_to_capture and digests should be the same length" for entry, digest in zip(files_to_capture, digests): entry[1].CopyFrom(digest) @@ -336,6 +360,7 @@ def cache( digest = artifact.buildsandbox.subsandbox_digests.add() digest.CopyFrom(vdir._get_digest()) + assert self._artifactdir, "An artifact dir is required at this point" os.makedirs(os.path.dirname(os.path.join(self._artifactdir, element.get_artifact_name())), exist_ok=True) keys = utils._deduplicate([self._cache_key, self._weak_cache_key]) for key in keys: @@ -370,7 +395,7 @@ def cached_buildroot(self): # Returns: # (bool): True if artifact was created with buildroot # - def buildroot_exists(self): + def buildroot_exists(self) -> bool: artifact = self._get_proto() return bool(str(artifact.buildroot)) @@ -386,7 +411,7 @@ def buildroot_exists(self): # missing expected buildtree. Note this only confirms # if a buildtree is present, not its contents. # - def cached_buildtree(self): + def cached_buildtree(self) -> bool: buildtree_digest = self._get_field_digest("buildtree") if buildtree_digest: @@ -430,7 +455,7 @@ def cached_sources(self): # Returns: # (dict): The artifacts cached public data # - def load_public_data(self): + def load_public_data(self) -> MappingNode: # Load the public data from the artifact artifact = self._get_proto() @@ -546,7 +571,7 @@ def get_metadata_keys(self) -> Tuple[str, str, str]: # Returns: # (bool): Whether the given artifact was workspaced # - def get_metadata_workspaced(self): + def get_metadata_workspaced(self) -> bool: if self._metadata_workspaced is not None: return self._metadata_workspaced @@ -565,7 +590,7 @@ def get_metadata_workspaced(self): # Returns: # (list): List of which dependencies are workspaced # - def get_metadata_workspaced_dependencies(self): + def get_metadata_workspaced_dependencies(self) -> list[str]: if self._metadata_workspaced_dependencies is not None: return self._metadata_workspaced_dependencies @@ -645,7 +670,7 @@ def query_cache(self): # (bool): Whether artifact is in local cache # def cached(self, *, buildtree=False): - assert self._cached is not None + assert self._cached is not None, "_cached should have been initialised before calling this method" ret = self._cached if buildtree: ret = ret and (self.cached_buildtree() or not self.buildtree_exists()) @@ -671,7 +696,7 @@ def cached_logs(self): # def set_cached(self): self._proto = self._load_proto() - assert self._proto + assert self._proto, "We expect a value is returned by _load_proto()" self._cached = True # pull() @@ -700,7 +725,7 @@ def pull(self, *, pull_buildtrees): return True - def configure_sandbox(self, sandbox): + def configure_sandbox(self, sandbox: Sandbox): artifact = self._get_proto() if artifact.HasField("buildsandbox") and artifact.buildsandbox.environment: @@ -722,13 +747,13 @@ def configure_sandbox(self, sandbox): # load_proto() # # Returns: - # (Artifact): Artifact proto + # (ArtifactProto): Artifact proto # - def _load_proto(self): + def _load_proto(self) -> Optional["ArtifactProto"]: key = self.get_extract_key() - + assert self._artifactdir, "Must have and artifact dir to load proto" proto_path = os.path.join(self._artifactdir, self._element.get_artifact_name(key=key)) - artifact = ArtifactProto() + artifact: ArtifactProto = ArtifactProto() try: with open(proto_path, mode="r+b") as f: artifact.ParseFromString(f.read()) diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index c8328f109..55aa04efb 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -14,15 +14,20 @@ # Authors: # Tristan Maat + import os +from typing import TYPE_CHECKING from ._assetcache import AssetCache from ._cas.casremote import BlobNotFound -from ._exceptions import ArtifactError, AssetCacheError, CASError, CASRemoteError +from ._exceptions import ArtifactError, AssetCacheError, CASError, CASRemoteError, BstError from ._protos.buildstream.v2 import artifact_pb2 - from . import utils +if TYPE_CHECKING: + from . import Element + from ._context import Context + REMOTE_ASSET_ARTIFACT_URN_TEMPLATE = "urn:fdc:buildstream.build:2020:artifact:{}" @@ -32,11 +37,12 @@ # context (Context): The BuildStream context # class ArtifactCache(AssetCache): - def __init__(self, context): + def __init__(self, context: "Context"): super().__init__(context) # create artifact directory self._basedir = context.artifactdir + assert self._basedir is not None, "Must have a base dir" os.makedirs(self._basedir, exist_ok=True) # preflight(): @@ -57,9 +63,9 @@ def preflight(self): # # Returns: True if the artifact is in the cache, False otherwise # - def contains(self, element, key): + def contains(self, element: "Element", key: str): ref = element.get_artifact_name(key) - + assert self._basedir is not None, "Must have a base directory" return os.path.exists(os.path.join(self._basedir, ref)) # list_artifacts(): @@ -151,7 +157,7 @@ def push(self, element, artifact): # Returns: # (bool): True if pull was successful, False if artifact was not available # - def pull(self, element, key, *, pull_buildtrees=False): + def pull(self, element: "Element", key: str, *, pull_buildtrees: bool = False) -> bool: artifact_digest = None display_key = key[: self.context.log_key_length] project = element._get_project() @@ -161,7 +167,7 @@ def pull(self, element, key, *, pull_buildtrees=False): index_remotes, storage_remotes = self.get_remotes(project.name, False) - errors = [] + errors: list[BstError] = [] # Start by pulling our artifact proto, so that we know which # blobs to pull for remote in index_remotes: @@ -192,22 +198,24 @@ def pull(self, element, key, *, pull_buildtrees=False): errors = [] # If we do, we can pull it! - for remote in storage_remotes: - remote.init() + for storage_remote in storage_remotes: + storage_remote.init() try: - element.status("Pulling data for artifact {} <- {}".format(display_key, remote)) + element.status("Pulling data for artifact {} <- {}".format(display_key, storage_remote)) - if self._pull_artifact_storage(element, key, artifact_digest, remote, pull_buildtrees=pull_buildtrees): - element.info("Pulled artifact {} <- {}".format(display_key, remote)) + if self._pull_artifact_storage( + element, key, artifact_digest, storage_remote, pull_buildtrees=pull_buildtrees + ): + element.info("Pulled artifact {} <- {}".format(display_key, storage_remote)) return True - element.info("Remote ({}) does not have artifact {} cached".format(remote, display_key)) + element.info("Remote ({}) does not have artifact {} cached".format(storage_remote, display_key)) except BlobNotFound as e: - # Not all blobs are available on this remote - element.info("Remote cas ({}) does not have blob {} cached".format(remote, e.blob)) + # Not all blobs are available on this storage_remote + element.info("Remote cas ({}) does not have blob {} cached".format(storage_remote, e.blob)) continue except CASError as e: - element.warn("Could not pull from remote {}: {}".format(remote, e)) + element.warn("Could not pull from remote {}: {}".format(storage_remote, e)) errors.append(e) if errors: @@ -235,7 +243,7 @@ def link_key(self, element, oldkey, newkey): if oldref == newref: # The two refs are identical, nothing to do return - + assert self._basedir, "base dir should be set" utils.safe_link(os.path.join(self._basedir, oldref), os.path.join(self._basedir, newref)) # check_remotes_for_element() @@ -248,7 +256,7 @@ def link_key(self, element, oldkey, newkey): # Returns: # (bool): True if the element is available remotely # - def check_remotes_for_element(self, element): + def check_remotes_for_element(self, element: "Element") -> bool: project = element._get_project() index_remotes, _ = self.get_remotes(project.name, False) @@ -421,7 +429,8 @@ def _pull_artifact_storage(self, element, key, artifact_digest, remote, pull_bui with self.cas.open(artifact_digest, "rb") as f: artifact.ParseFromString(f.read()) - # Write the artifact proto to cache + # Write the artifact proto to cache# + assert self._basedir, "Must have a base directory" artifact_path = os.path.join(self._basedir, artifact_name) os.makedirs(os.path.dirname(artifact_path), exist_ok=True) with utils.save_file_atomic(artifact_path, mode="wb") as f: diff --git a/src/buildstream/_assetcache.py b/src/buildstream/_assetcache.py index 9406b9cf9..7140ebc1e 100644 --- a/src/buildstream/_assetcache.py +++ b/src/buildstream/_assetcache.py @@ -14,9 +14,10 @@ # Authors: # Raoul Hidalgo Charman # + import os import re -from typing import List, Dict, Tuple, Iterable, Optional +from typing import List, Dict, Tuple, Iterable, Optional, TYPE_CHECKING import grpc from . import utils @@ -28,6 +29,9 @@ from ._protos.build.buildgrid import local_cas_pb2 from ._protos.google.rpc import code_pb2 +if TYPE_CHECKING: + from buildstream._context import Context + class AssetRemote(BaseRemote): def __init__(self, spec, casd): @@ -61,6 +65,7 @@ def _check(self): request.instance_name = self.instance_name try: + assert self.fetch_service, "Can't fetch blob without a fetch service" self.fetch_service.FetchBlob(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.INVALID_ARGUMENT: @@ -80,6 +85,7 @@ def _check(self): request.instance_name = self.instance_name try: + assert self.push_service, "Can't push blob without a push service" self.push_service.PushBlob(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.INVALID_ARGUMENT: @@ -121,6 +127,7 @@ def fetch_blob(self, uris, *, qualifiers=None): request.qualifiers.extend(qualifiers) try: + assert self.fetch_service, "Can't fetch blob without a fetch service" response = self.fetch_service.FetchBlob(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: @@ -162,6 +169,7 @@ def fetch_directory(self, uris, *, qualifiers=None): request.qualifiers.extend(qualifiers) try: + assert self.fetch_service, "Can't fetch directory without a fetch service" response = self.fetch_service.FetchDirectory(request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: @@ -208,6 +216,7 @@ def push_blob(self, uris, blob_digest, *, qualifiers=None, references_blobs=None request.references_directories.extend(references_directories) try: + assert self.push_service, "Can't push blob without a push service" self.push_service.PushBlob(request) except grpc.RpcError as e: raise AssetCacheError("PushBlob failed with status {}: {}".format(e.code().name, e.details())) from e @@ -245,6 +254,7 @@ def push_directory( request.references_directories.extend(references_directories) try: + assert self.push_service, "Can't push directory without a push service" self.push_service.PushDirectory(request) except grpc.RpcError as e: raise AssetCacheError("PushDirectory failed with status {}: {}".format(e.code().name, e.details())) from e @@ -285,7 +295,7 @@ def __init__(self, casd: CASDProcessManager, spec: RemoteSpec): # Base Asset Cache for Caches to derive from # class AssetCache: - def __init__(self, context): + def __init__(self, context: "Context"): self.context = context self.cas: CASCache = context.get_cascache() @@ -298,7 +308,7 @@ def __init__(self, context): self._has_fetch_remotes: bool = False self._has_push_remotes: bool = False - self._basedir = None + self._basedir: Optional[str] = None # setup_remotes(): # @@ -466,6 +476,7 @@ def list_refs_mtimes(self, base_path, *, glob_expr=None): # def remove_ref(self, ref): try: + assert self._basedir, "Need a base directory" utils._remove_path_with_parents(self._basedir, ref) except FileNotFoundError as e: raise AssetCacheError("Could not find ref '{}'".format(ref)) from e diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 3e4da1f2b..110ad3458 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -337,7 +337,7 @@ def load(self, config: Optional[str] = None) -> None: raise LoadError("{} must be an absolute path".format(directory), LoadErrorReason.INVALID_DATA) # add directories not set by users - assert self.cachedir + assert self.cachedir, "Need a cache dir at this stage" self.tmpdir = os.path.join(self.cachedir, "tmp") self.casdir = os.path.join(self.cachedir, "cas") self.builddir = os.path.join(self.cachedir, "build") @@ -671,7 +671,7 @@ def get_workspaces(self) -> Workspaces: # It is an error to call this early on before the Workspaces # has been instantiated # - assert self._workspaces + assert self._workspaces, "Must have workspaces before calling this" return self._workspaces # get_workspace_project_cache(): diff --git a/src/buildstream/_elementproxy.py b/src/buildstream/_elementproxy.py index dc7f03090..2659d5c25 100644 --- a/src/buildstream/_elementproxy.py +++ b/src/buildstream/_elementproxy.py @@ -13,15 +13,15 @@ # # Authors: # Tristan Van Berkom -from typing import TYPE_CHECKING, cast, Optional, Iterator, Dict, List, Sequence + +from typing import TYPE_CHECKING, cast, Optional, Iterator, Dict, List, Sequence, Iterable from .types import _Scope, OverlapAction from .utils import FileListResult from ._pluginproxy import PluginProxy +from .node import MappingNode, ScalarNode, SequenceNode if TYPE_CHECKING: - from typing import Any - from .node import MappingNode, ScalarNode, SequenceNode from .sandbox import Sandbox from .source import Source from .element import Element # pylint: disable=cyclic-import @@ -84,7 +84,7 @@ def node_subst_sequence_vars(self, node: "SequenceNode[ScalarNode]") -> List[str def compute_manifest( self, *, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True - ) -> str: + ) -> Iterable[str]: return cast("Element", self._plugin).compute_manifest(include=include, exclude=exclude, orphans=orphans) def get_artifact_name(self, key: Optional[str] = None) -> str: @@ -137,7 +137,7 @@ def stage_dependency_artifacts( def integrate(self, sandbox: "Sandbox") -> None: cast("Element", self._plugin).integrate(sandbox) - def get_public_data(self, domain: str) -> "MappingNode[Any]": + def get_public_data(self, domain: str) -> MappingNode | None: return cast("Element", self._plugin).get_public_data(domain) def get_environment(self) -> Dict[str, str]: diff --git a/src/buildstream/_elementsources.py b/src/buildstream/_elementsources.py index 1d862c176..4e52faf97 100644 --- a/src/buildstream/_elementsources.py +++ b/src/buildstream/_elementsources.py @@ -11,15 +11,18 @@ # See the License for the specific language governing permissions and # limitations under the License. + import os from contextlib import contextmanager -from typing import TYPE_CHECKING, Iterator +from typing import TYPE_CHECKING, Iterator, Optional from . import _cachekey from ._exceptions import SkipJob from ._context import Context from ._protos.buildstream.v2 import source_pb2 from .plugin import Plugin +from ._sourcecache import SourceCache +from ._elementsourcescache import ElementSourcesCache from .storage._casbaseddirectory import CasBasedDirectory @@ -37,16 +40,16 @@ class ElementSources: def __init__(self, context: Context, project: "Project", plugin: Plugin): - self._context = context - self._project = project - self._plugin = plugin - self._sources = [] # type: List[Source] - self._sourcecache = context.sourcecache # Source cache - self._elementsourcescache = context.elementsourcescache # Cache of staged element sources + self._context: Context = context + self._project: Project = project + self._plugin: Plugin = plugin + self._sources: list[Source] = [] + self._sourcecache: SourceCache = context.sourcecache # Source cache + self._elementsourcescache: ElementSourcesCache = context.elementsourcescache # Cache of staged element sources self._is_resolved = False # Whether the source is fully resolved or not - self._cached = None # If the sources are known to be successfully cached in CAS - self._cache_key = None # Our cached cache key - self._proto = None # The cached Source proto + self._cached: Optional[bool] = None # If the sources are known to be successfully cached in CAS + self._cache_key: Optional[str] = None # Our cached cache key + self._proto: Optional[source_pb2.Source] = None # The cached Source proto # get_project(): # @@ -138,7 +141,8 @@ def stage_and_cache(self): # def get_files(self): # Assert sources are cached - assert self.cached() + assert self.cached(), "Must be cached to get files" + assert self._proto, "Must have proto" cas = self._context.get_cascache() return CasBasedDirectory(cas, digest=self._proto.files) @@ -288,6 +292,8 @@ def get_cache_key(self): def get_brief_display_key(self): context = self._context key = self._cache_key + assert key, "Must have key for this" + assert context.log_key_length, "Must have log key length for this" length = min(len(key), context.log_key_length) return key[:length] diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index 2120b5918..fe7ed3cf0 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -14,17 +14,22 @@ # Authors: # Tristan Van Berkom -from contextlib import contextmanager + +from typing import Optional, Any + import os import sys -import threading import traceback import datetime from textwrap import TextWrapper +from threading import Lock + +from contextlib import contextmanager import click from click import UsageError # Import various buildstream internals +from ..element import Element from .._context import Context from .._project import Project from .._exceptions import BstError, StreamError, LoadError, AppError @@ -54,29 +59,29 @@ # command, before any subcommand # class App: - def __init__(self, main_options): + def __init__(self, main_options: dict[str, Any]): # # Public members # - self.context = None # The Context object - self.stream = None # The Stream object - self.project = None # The toplevel Project object - self.logger = None # The LogLine object - self.interactive = None # Whether we are running in interactive mode - self.colors = None # Whether to use colors in logging + self.context: Optional[Context] = None # The Context object + self.stream: Optional[Stream] = None # The Stream object + self.project: Optional[Project] = None # The toplevel Project object + self.logger: Optional[LogLine] = None # The LogLine object + self.interactive: Optional[bool] = None # Whether we are running in interactive mode + self.colors: Optional[bool] = None # Whether to use colors in logging # # Private members # - self._session_start = datetime.datetime.now() - self._session_name = None - self._main_options = main_options # Main CLI options, before any command - self._status = None # The Status object - self._fail_messages = {} # Failure messages by unique plugin id - self._interactive_failures = None # Whether to handle failures interactively - self._started = False # Whether a session has started - self._set_project_dir = False # Whether -C option was used + self._session_start: datetime.datetime = datetime.datetime.now() + self._session_name: Optional[str] = None + self._main_options: dict[str, Any] = main_options # Main CLI options, before any command + self._status: Optional[Status] = None # The Status object + self._fail_messages: dict[str, Message] = {} # Failure messages by unique plugin id + self._interactive_failures: Optional[bool] = None # Whether to handle failures interactively + self._started: bool = False # Whether a session has started + self._set_project_dir: bool = False # Whether -C option was used self._state = None # Frontend reads this and registers callbacks # UI Colors Profiles @@ -87,9 +92,9 @@ def __init__(self, main_options): self._detail_profile = Profile(dim=True) # Cached messages - self._cached_message_lock = threading.Lock() - self._cached_message_text = "" - self._cache_messages = None + self._cached_message_lock: Lock = Lock() + self._cached_message_text: str = "" + self._cache_messages: Optional[int] = None # # Early initialization @@ -363,11 +368,11 @@ def initialized(self, *, session_name=None): # def init_project( self, - project_name, - min_version, - element_path, - force=False, - target_directory=None, + project_name: str, + min_version: str, + element_path: str, + force: bool = False, + target_directory: Optional[str] = None, ): if target_directory: directory = os.path.abspath(target_directory) @@ -517,6 +522,7 @@ def _notify(self, title, text): # Local message propagator # def _message(self, message_type, message, **kwargs): + assert self.context, "App should have a loaded context here" self.context.messenger.message(Message(message_type, message, **kwargs)) # Flush any potentially cached messages immediately @@ -535,6 +541,7 @@ def _global_exception_handler(self, etype, value, tb, exc=True): # If the scheduler has started, try to terminate all jobs gracefully, # otherwise exit immediately. + assert self.stream, "App must have a stream here" if self.stream.running: self.stream.terminate() else: @@ -549,7 +556,8 @@ def _global_exception_handler(self, etype, value, tb, exc=True): # Returns: # (str): The rendered text of only this message # - def _cache_message(self, message): + def _cache_message(self, message: str): + assert self.logger, "App must have a logger for this" text = self.logger.render(message) with self._cached_message_lock: @@ -587,7 +595,7 @@ def _render_status(self): # Handle ^C SIGINT interruptions in the scheduling main loop # def _interrupt_handler(self): - + assert self.stream, "Must have a Stream in App for this" # Only handle ^C interactively in interactive mode if not self.interactive: self.stream.terminate() @@ -644,7 +652,9 @@ def _interrupt_handler(self): # task_id (str): The unique identifier of the task # element (tuple): If an element job failed a tuple of Element instance unique_id & display key # - def _job_failed(self, task_id, element=None): + def _job_failed(self, task_id: str, element: Optional[tuple[Element, str, str]] = None): + assert self._state, "App must have a state for this" + assert self.stream, "App must have a stream for this" task = self._state.tasks[task_id] # Flush any pending messages when handling a failure @@ -680,6 +690,7 @@ def _handle_failure(self, element, task, failure): # Handle non interactive mode setting of what to do when a job fails. if not self._interactive_failures: + assert self.context, "App must have a context for this" if self.context.sched_error_action == _SchedulerErrorAction.TERMINATE: self.stream.terminate() @@ -713,6 +724,7 @@ def _handle_failure(self, element, task, failure): if failure.sandbox: choices += ["shell"] + assert self.stream, "App must have a stream for this" choice = "" while choice not in ["continue", "quit", "terminate", "retry"]: click.echo(summary, err=True) diff --git a/src/buildstream/_loader/loadcontext.py b/src/buildstream/_loader/loadcontext.py index 5f3f4eda4..37744059a 100644 --- a/src/buildstream/_loader/loadcontext.py +++ b/src/buildstream/_loader/loadcontext.py @@ -14,23 +14,31 @@ # Authors: # Tristan Van Berkom + +from typing import Callable, Optional, TYPE_CHECKING + + from .._exceptions import LoadError from ..exceptions import LoadErrorReason from ..types import _ProjectInformation +if TYPE_CHECKING: + from .._context import Context + from .._loader.loader import Loader + # ProjectLoaders() # # An object representing all of the loaders for a given project. # class ProjectLoaders: - def __init__(self, project_name): + def __init__(self, project_name: str): # The project name self._name = project_name # A list of all loaded loaders for this project - self._collect = [] + self._collect: list["Loader"] = [] # register_loader() # @@ -39,7 +47,7 @@ def __init__(self, project_name): # Args: # loader (Loader): The loader to register # - def register_loader(self, loader): + def register_loader(self, loader: "Loader"): assert loader.project.name == self._name self._collect.append(loader) @@ -101,9 +109,9 @@ def loaded_projects(self): # (list): A list of Loader objects who's project has marked # this junction as internal # - def _search_project_relationships(self, loader): - duplicates = [] - internal = [] + def _search_project_relationships(self, loader: "Loader"): + duplicates: list[Loader] = [] + internal: list[Loader] = [] for parent in loader.ancestors(): if parent.project.junction_is_duplicated(self._name, loader): duplicates.append(parent) @@ -179,16 +187,16 @@ def _loader_description(self, loader, duplicates, internals): # context (Context): The invocation context # class LoadContext: - def __init__(self, context): + def __init__(self, context: "Context"): # Keep track of global context required throughout the recursive load self.context = context self.rewritable = False - self.fetch_subprojects = None + self.fetch_subprojects: Optional[Callable] = None self.task = None # A table of all Loaders, indexed by project name - self._loaders = {} + self._loaders: dict[str, ProjectLoaders] = {} # set_rewritable() # @@ -257,7 +265,7 @@ def register_loader(self, loader): # loaded_projects() # - # A generator which yeilds all of the loaded projects + # A generator which yields all of the loaded projects # # Yields: # (_ProjectInformation): A descriptive project information object diff --git a/src/buildstream/_loader/loadelement.pyi b/src/buildstream/_loader/loadelement.pyi index 8f4881fef..d2067e1db 100644 --- a/src/buildstream/_loader/loadelement.pyi +++ b/src/buildstream/_loader/loadelement.pyi @@ -11,19 +11,242 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from typing import List +from buildstream._loader.loader import Loader +import enum +from buildstream._project import Project +from typing import List, Optional -from ..node import Node, ScalarNode +from buildstream.plugins.elements.junction import JunctionElement +from buildstream.node import Node, ScalarNode, MappingNode, SequenceNode -def extract_depends_from_node(node: Node) -> List[Dependency]: ... +def extract_depends_from_node(node: Node) -> List[Dependency]: + """ + extract_depends_from_node(): -class Dependency: ... -class DependencyType: ... + Creates an array of Dependency objects from a given dict node 'node', + allows both strings and dicts for expressing the dependency and + throws a comprehensive LoadError in the case that the node is malformed. + + After extracting depends, the symbol is deleted from the node + + Args: + node (Node): A YAML loaded dictionary + + Returns: + (list): a list of Dependency objects + """ + ... + +def sort_dependencies(element: LoadElement, visited: set[LoadElement]): + """ + sort_dependencies(): + + Sort dependencies of each element by their dependencies, + so that direct dependencies which depend on other direct + dependencies (directly or indirectly) appear later in the + list. + + This avoids the need for performing multiple topological + sorts throughout the build process. + + Args: + element (LoadElement): The element to sort + visited (set): a list of elements that should not be treated because + because they already have been treated. + This is useful when wanting to sort dependencies of + multiple top level elements that might have a common + part. + """ + ... + +class Dependency: + """ + Dependency(): + + Early stage data model for dependencies objects, the LoadElement has + Dependency objects which in turn refer to other LoadElements in the data + model. + + The constructor is incomplete, normally dependencies are loaded + via the Dependency.load() API below. The constructor arguments are + only used as a convenience to create the dummy Dependency objects + at the toplevel of the load sequence in the Loader. + + Args: + element (LoadElement): a LoadElement on which there is a dependency + dep_type (DependencyType): the type of dependency this dependency link is + """ + + name: str + """ + The project local dependency name + """ + node: Node + """ + The original node of the dependency + """ + element: LoadElement + """ + The resolved LoadElement + """ + dep_type: DependencyType + """ + The dependency type (runtime or build or both) + """ + junction: Optional[str] + """ + The junction path of the dependency name, if any + """ + config_nodes: List[MappingNode] + """ + The custom config nodes for Element.configure_dependencies() + """ + strict: bool + """ + Whether this is a strict dependency + """ + + path: str + """ + The path of the dependency represented as a single string, + instead of junction and name being separate. + """ + + def __init__(self, element: LoadElement, dep_type: DependencyType): + """ + Dependency(): + + Early stage data model for dependencies objects, the LoadElement has + Dependency objects which in turn refer to other LoadElements in the data + model. + + The constructor is incomplete, normally dependencies are loaded + via the Dependency.load() API below. The constructor arguments are + only used as a convenience to create the dummy Dependency objects + at the toplevel of the load sequence in the Loader. + + Args: + element (LoadElement): a LoadElement on which there is a dependency + dep_type (DependencyType): the type of dependency this dependency link is + """ + + def set_element(self, element: LoadElement) -> None: + """ + set_element() + + Sets the resolved LoadElement + + When Dependencies are initially loaded, the `element` member + will be None until later on when the Loader loads the LoadElement + objects based on the Dependency `name` and `junction`, the Loader + will then call this to resolve the `element` member. + + Args: + element (LoadElement): The resolved LoadElement + """ + + def merge(self, other: Dependency) -> None: + """ + merge() + + Merge the attributes of an existing dependency into this dependency + + Args: + other (Dependency): The dependency to merge into this one + """ + +class DependencyType(enum.Enum): + """ + DependencyType + + A bitfield to represent dependency types + """ + + BUILD = 0x001 + """ + A build dependency + """ + RUNTIME = 0x002 + """ + A runtime dependency + """ + ALL = 0x003 + """ + Both build and runtime dependencies + """ class LoadElement: + """ + LoadElement(): + + A transient object breaking down what is loaded allowing us to + do complex operations in multiple passes. + + Args: + node (dict): A YAML loaded dictionary + name (str): The element name + loader (Loader): The Loader object for this element + """ + first_pass: bool + """ + Whether the element should be included in a first pass when processing + + e.g. link or junction elements that need resolving before usage. + """ kind: str + """ + The Element kind + """ name: str + """ + The element name + """ + full_name: str + """ + The element full name (with associated junction) + """ description: str - node: Node + """ + The element description + """ + node: MappingNode + """ + The YAML node + """ link_target: ScalarNode + """ + The target of a link element (ScalarNode) + """ + fully_loaded: bool + """ + Whether we entered the loop to load dependencies or not + """ + + project: "Project" + """ + The Project the Element is from + """ + junction: Optional[JunctionElement] + """ + The Optional JunctionElement the Element is from + """ + dependencies: List[Dependency] + """ + The Element dependencies + """ + + def __init__(self, node: MappingNode, filename: str, loader: Loader): + """ + LoadElement(): + + A transient object breaking down what is loaded allowing us to + do complex operations in multiple passes. + + Args: + node (dict): A YAML loaded dictionary + name (str): The element name + loader (Loader): The Loader object for this element + """ + + def mark_fully_loaded(self): ... diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index af98504cc..6bad0c828 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -14,22 +14,32 @@ # Authors: # Tristan Van Berkom + import os -from contextlib import suppress +from typing import Optional, Self, TYPE_CHECKING, Iterable, cast -from .._exceptions import LoadError -from ..exceptions import LoadErrorReason from .. import _yaml from ..element import Element -from ..node import Node -from .._profile import Topics, PROFILER + +from .._exceptions import LoadError from .._includes import Includes +from .._profile import PROFILER, Topics from .._utils import valid_chars_name +from ..exceptions import LoadErrorReason +from ..node import Node, ScalarNode from ..types import _KeyStrength - -from .types import Symbol from . import loadelement -from .loadelement import LoadElement, Dependency, DependencyType, extract_depends_from_node +from .loadelement import ( + Dependency, + DependencyType, + LoadElement, + extract_depends_from_node, +) +from .types import Symbol + +if TYPE_CHECKING: + from .._project import Project + from .._loader.loadcontext import LoadContext # Loader(): @@ -44,20 +54,21 @@ # provenance_node (Node): The provenance of the reference to this project's junction # class Loader: - def __init__(self, project, *, parent=None, provenance_node=None): + def __init__(self, project: "Project", *, parent: Optional[Self] = None, provenance_node: Optional[Node] = None): # Ensure we have an absolute path for the base directory basedir = project.element_path + assert basedir is not None, "Must have a base directory" if not os.path.isabs(basedir): basedir = os.path.abspath(basedir) # # Public members # - self.load_context = project.load_context # The LoadContext + self.load_context: LoadContext = project.load_context # The LoadContext self.project = project # The associated Project self.provenance_node = provenance_node # The provenance of whence this loader was instantiated - self.loaded = None # The number of loaded Elements + self.loaded: int | None = None # The number of loaded Elements # # Private members @@ -66,13 +77,14 @@ def __init__(self, project, *, parent=None, provenance_node=None): self._basedir = basedir # Base project directory self._first_pass_options = project.first_pass_config.options # Project options (OptionPool) self._parent = parent # The parent loader - self._alternative_parents = [] # Overridden parent loaders + self._alternative_parents: list["Loader"] = [] # Overridden parent loaders - self._meta_elements = {} # Dict of resolved meta elements by name - self._elements = {} # Dict of elements - self._links = {} # Dict of link target target paths indexed by link element paths - self._loaders = {} # Dict of junction loaders - self._loader_search_provenances = {} # Dictionary of provenance nodes of ongoing child loader searches + self._elements: dict[str, LoadElement] = {} # Dict of elements + self._links: dict[str, str] = {} # Dict of link target target paths indexed by link element paths + self._loaders: dict[str, "Loader"] = {} # Dict of junction loaders + self._loader_search_provenances: dict[str, Node] = ( + {} + ) # Dictionary of provenance nodes of ongoing child loader searches self._includes = Includes(self, copy_tree=True) @@ -110,7 +122,7 @@ def __str__(self): # Returns: # (list): The corresponding LoadElement instances matching the `targets` # - def load(self, targets): + def load(self, targets: list[str]) -> list[LoadElement]: for filename in targets: if os.path.isabs(filename): @@ -124,12 +136,14 @@ def load(self, targets): # First pass, recursively load files and populate our table of LoadElements # - target_elements = [] + target_elements: list[LoadElement] = [] for target in targets: with PROFILER.profile(Topics.LOAD_PROJECT, target): _junction, name, loader = self._parse_name(target, None) + assert loader, "Must have a loader" element = loader._load_file(name, None) + assert element, "Must get an element back here" target_elements.append(element) # @@ -153,10 +167,9 @@ def load(self, targets): # # Keep a list of all visited elements, to not sort twice the same - visited_elements = set() + visited_elements: set[LoadElement] = set() for element in target_elements: - loader = element._loader with PROFILER.profile(Topics.SORT_DEPENDENCIES, element.name): loadelement.sort_dependencies(element, visited_elements) @@ -168,7 +181,7 @@ def load(self, targets): return target_elements - def _normalize_element_name(self, filename): + def _normalize_element_name(self, filename) -> str: # Keep equivalent relative spellings on the same loader identity path, # so ref lookup, cache keys and workspace handling stay consistent. return os.path.normpath(filename) @@ -191,7 +204,7 @@ def _normalize_element_name(self, filename): # Returns: # (Loader): loader for sub-project # - def get_loader(self, name, provenance_node, *, load_subprojects=True): + def get_loader(self, name: str, provenance_node: Node, *, load_subprojects=True) -> "Loader | None": junction_path = name.split(":") loader = self @@ -213,11 +226,12 @@ def get_loader(self, name, provenance_node, *, load_subprojects=True): self._loader_search_provenances[name] = provenance_node for junction_name in junction_path: - loader = loader._get_loader(junction_name, provenance_node, load_subprojects=load_subprojects) - if not loader: + junction_loader = loader._get_loader(junction_name, provenance_node, load_subprojects=load_subprojects) + if not junction_loader: # `loader` should never be None if `load_subprojects` is True assert not load_subprojects return None + loader = junction_loader if load_subprojects and provenance_node: del self._loader_search_provenances[name] @@ -232,7 +246,7 @@ def get_loader(self, name, provenance_node, *, load_subprojects=True): # Yields: # (Loader): Each loader in the ancestry # - def ancestors(self): + def ancestors(self) -> Iterable[Self]: traversed = {} def foreach_parent(parent): @@ -271,7 +285,9 @@ def foreach_parent(parent): # Returns: # (LoadElement): A partially-loaded LoadElement # - def _load_file_no_deps(self, filename, provenance_node=None, only_first_pass=False): + def _load_file_no_deps( + self, filename: str, provenance_node: Optional[Node] = None, only_first_pass=False + ) -> Optional[LoadElement]: self._assert_element_name(filename, provenance_node) @@ -310,6 +326,7 @@ def _load_file_no_deps(self, filename, provenance_node=None, only_first_pass=Fal kind = node.get_str(Symbol.KIND) if kind in ("junction", "link"): + assert self._first_pass_options, "Must have first pass options" self._first_pass_options.process_node(node) elif only_first_pass: return None @@ -319,7 +336,7 @@ def _load_file_no_deps(self, filename, provenance_node=None, only_first_pass=Fal self._includes.process(node) element = LoadElement(node, filename, self) - + assert self._elements is not None, "elements area must be initialised in loader, (can be empty)" self._elements[filename] = element # @@ -338,6 +355,7 @@ def _load_file_no_deps(self, filename, provenance_node=None, only_first_pass=Fal loader = self while loader._parent: junction = loader.project.junction + assert junction, "Must have a junction element for loaders that have a parent" link_path = junction.name + ":" + link_path target_path = junction.name + ":" + target_path @@ -360,7 +378,7 @@ def _load_file_no_deps(self, filename, provenance_node=None, only_first_pass=Fal # link_path (str): The local project relative real path to a link # target_path (str): The new target for this link # - def _resolve_link(self, link_path, target_path): + def _resolve_link(self, link_path: str, target_path: str): self._links[link_path] = target_path for cached_link_path, cached_target_path in self._links.items(): @@ -378,7 +396,7 @@ def _resolve_link(self, link_path, target_path): # Returns: # (str): The same path with any links expanded # - def _expand_link(self, path): + def _expand_link(self, path: str) -> str: # FIXME: This simply returns the first link, maybe # this needs to be more iterative, or sorted by @@ -403,7 +421,9 @@ def _expand_link(self, path): # (LoadElement): A LoadElement, which might be shallow loaded or fully loaded, # or None, if loading of the subproject is disabled. # - def _load_one_file(self, filename, provenance_node, *, load_subprojects=True): + def _load_one_file( + self, filename: str, provenance_node: Optional[Node], *, load_subprojects=True + ) -> Optional[LoadElement]: filename = self._normalize_element_name(filename) @@ -427,6 +447,7 @@ def _load_one_file(self, filename, provenance_node, *, load_subprojects=True): # Shallow load if it's not yet loaded. element = self._load_file_no_deps(filename, provenance_node) + assert element, "Should have an element by this point" # Check if there was an override for this element # override = self._search_for_override_element(filename) @@ -473,7 +494,9 @@ def _load_one_file(self, filename, provenance_node, *, load_subprojects=True): # Returns: # (LoadElement): A loaded LoadElementor None, if loading of the subproject is disabled. # - def _load_file(self, filename, provenance_node, *, load_subprojects=True): + def _load_file( + self, filename: str, provenance_node: Optional[Node], *, load_subprojects=True + ) -> Optional[LoadElement]: top_element = self._load_one_file(filename, provenance_node, load_subprojects=load_subprojects) @@ -498,7 +521,9 @@ def _load_file(self, filename, provenance_node, *, load_subprojects=True): # [0] is the LoadElement instance # [1] is a stack of Dependency objects to load # [2] is a Dict[LoadElement, Dependency] of loaded dependencies - loader_queue = [(top_element, list(reversed(dependencies)), {})] + loader_queue: list[tuple[LoadElement, list[Dependency], dict[LoadElement, Dependency]]] = [ + (top_element, list(reversed(dependencies)), {}) + ] # Load all dependency files for the new LoadElement while loader_queue: @@ -510,7 +535,9 @@ def _load_file(self, filename, provenance_node, *, load_subprojects=True): if dep.junction: loader = self.get_loader(dep.junction, dep.node) + assert loader, "Must have a loader for this junction" dep_element = loader._load_file(dep.name, dep.node) + assert dep_element, "Should have a dependant element" else: @@ -564,13 +591,13 @@ def _load_file(self, filename, provenance_node, *, load_subprojects=True): # dependencies already resolved. # # Args: - # element (str): The element to check + # top_element (LoadElement): The element to check # # Raises: # (LoadError): In case there was a circular dependency error # @staticmethod - def _check_circular_deps(top_element): + def _check_circular_deps(top_element: LoadElement) -> None: sequence = [top_element] sequence_indices = [0] @@ -622,14 +649,14 @@ def _check_circular_deps(top_element): # Returns: # (ScalarNode): The overridding node from this project's junction, or None # - def _search_for_local_override(self, override_path): + def _search_for_local_override(self, override_path: str) -> Optional[ScalarNode]: junction = self.project.junction if junction is None: return None # Try the override without any link substitutions first - with suppress(KeyError): - return junction.overrides[override_path] + if (override_node := junction.overrides.get(override_path)) is not None: + return override_node # # If we did not get an exact match here, we might still have @@ -663,6 +690,7 @@ def _search_for_overrides(self, filename): overriding_loaders = [] while loader._parent: junction = loader.project.junction + assert junction, "loader with parent must have a junction element" override_node = loader._search_for_local_override(override_path) if override_node: overriding_loaders.append((loader._parent, override_node)) @@ -764,7 +792,7 @@ def _search_for_override_element(self, filename): # # Returns: A Loader or None if specified junction does not exist # - def _get_loader(self, filename, provenance_node, *, load_subprojects=True): + def _get_loader(self, filename: str, provenance_node: Node, *, load_subprojects=True) -> Optional["Loader"]: loader = None # return previously determined result @@ -841,6 +869,10 @@ def provenance_str(): element = Element._new_from_load_element(load_element) + from ..plugins.elements.junction import JunctionElement + + element = cast(JunctionElement, element) + # Handle the case where a subproject has no ref # if not element._has_all_sources_resolved(): @@ -854,6 +886,7 @@ def provenance_str(): # Handle the case where a subproject needs to be fetched # element._query_source_cache() + assert self.load_context.fetch_subprojects, "fetch subprojects should be available in load_context" if element._should_fetch(): self.load_context.fetch_subprojects([element]) @@ -875,9 +908,10 @@ def provenance_str(): # we haven't yet for this element), # element._get_cache_key() can fail if used with the # default _KeyStrength.STRONG. - basedir = os.path.join( - self.project.directory, ".bst", "staged-junctions", filename, element._get_cache_key(_KeyStrength.WEAK) - ) + assert self.project.directory, "The loaders project must have a project directory" + key = element._get_cache_key(_KeyStrength.WEAK) + assert key, "We expect a weak key is always available" + basedir = os.path.join(self.project.directory, ".bst", "staged-junctions", filename, key) if not os.path.exists(basedir): os.makedirs(basedir, exist_ok=True) element._stage_sources_at(basedir) @@ -909,6 +943,7 @@ def provenance_str(): raise loader = project.loader + assert loader, "There must be a project loader by this point" self._loaders[filename] = loader # Now we've loaded a junction and it's project, we need to try to shallow @@ -1014,7 +1049,9 @@ def _shallow_load_path(self, path, provenance_node): # - (str): name of the element # - (Loader): loader for sub-project # - def _parse_name(self, name, provenance_node, *, load_subprojects=True): + def _parse_name( + self, name, provenance_node, *, load_subprojects=True + ) -> tuple[Optional[str], str, "Loader | None"]: # We allow to split only once since deep junctions names are forbidden. # Users who want to refer to elements in sub-sub-projects are required # to create junctions on the top level project. @@ -1087,5 +1124,4 @@ def _clean_caches(self): if loader is not None: loader._clean_caches() - self._meta_elements = {} self._elements = {} diff --git a/src/buildstream/_pluginfactory/elementfactory.py b/src/buildstream/_pluginfactory/elementfactory.py index cc95273b0..1096cc20a 100644 --- a/src/buildstream/_pluginfactory/elementfactory.py +++ b/src/buildstream/_pluginfactory/elementfactory.py @@ -54,5 +54,5 @@ def __init__(self, plugin_base): def create(self, context: "Context", project: "Project", load_element: LoadElement) -> Element: plugin_type, default_config = self.lookup(context.messenger, load_element.kind, load_element.node) element_type = cast(Type[Element], plugin_type) - element = element_type(context, project, load_element, default_config) + element: Element = element_type(context, project, load_element, default_config) return element diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 252616220..7c9619849 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -15,22 +15,25 @@ # Tristan Van Berkom # Tiago Gomes -from typing import TYPE_CHECKING, Optional, Dict, Union, List, Sequence, Callable import os import urllib.parse from pathlib import Path +from typing import TYPE_CHECKING, Optional, Dict, Union, List, Sequence, Callable, Self + from pluginbase import PluginBase + from . import utils from . import _site from . import _yaml +from ._loader.loadelement import LoadElement from ._variables import Variables from .utils import UtilError from ._profile import Topics, PROFILER from ._exceptions import LoadError from .exceptions import LoadErrorReason from ._options import OptionPool -from .node import ScalarNode, MappingNode, ProvenanceInformation, _assert_symbol_name +from .node import ScalarNode, MappingNode, _assert_symbol_name, Node, SequenceNode from ._pluginfactory import ElementFactory, SourceFactory, SourceMirrorFactory, load_plugin_origin from .types import CoreWarnings, _HostMount, _SourceUriPolicy from ._projectrefs import ProjectRefs, ProjectRefStorage @@ -40,7 +43,7 @@ from ._workspaces import WORKSPACE_PROJECT_FILE from ._remotespec import RemoteSpec from .sourcemirror import SourceMirror -from .source import AliasSubstitution, SourceError +from .source import AliasSubstitution, SourceError, Source if TYPE_CHECKING: from ._context import Context @@ -53,14 +56,14 @@ # Represents project configuration that can have different values for junctions. class ProjectConfig: - def __init__(self): - self.options = None # OptionPool - self.base_variables = None # The base set of variables - self.element_overrides = {} # Element specific configurations - self.source_overrides = {} # Source specific configurations - self.mirrors = {} # Dictionary of SourceMirror objects - self.default_mirror = None # The name of the preferred mirror. - self._aliases = None # Aliases dictionary + def __init__(self: Self): + self.options: Optional[OptionPool] = None # OptionPool + self.base_variables: Optional[MappingNode] = None # The base set of variables + self.element_overrides: MappingNode = Node.from_dict({}) # Element specific configurations + self.source_overrides: MappingNode = Node.from_dict({}) # Source specific configurations + self.mirrors: dict[str, SourceMirror] = {} # Dictionary of SourceMirror objects + self.default_mirror: Optional[str] = None # The name of the preferred mirror. + self._aliases: Optional[MappingNode] = None # Aliases dictionary # Project() @@ -87,7 +90,7 @@ def __init__( cli_options: Optional[Dict[str, str]] = None, default_mirror: Optional[str] = None, parent_loader: Optional[Loader] = None, - provenance_node: Optional[ProvenanceInformation] = None, + provenance_node: Optional[Node] = None, search_for_project: bool = True, fetch_subprojects=None ): @@ -102,7 +105,7 @@ def __init__( self.loader: Optional[Loader] = None # The loader associated to this project self.junction: Optional["JunctionElement"] = junction # The junction Element object, if this is a subproject - self.ref_storage: Optional[ProjectRefStorage] = None # Where to store source refs + self.ref_storage: Optional[str] = None # Where to store source refs self.refs: Optional[ProjectRefs] = None self.junction_refs: Optional[ProjectRefs] = None self.disallow_subproject_uris: bool = False @@ -110,7 +113,7 @@ def __init__( self.config: ProjectConfig = ProjectConfig() self.first_pass_config: ProjectConfig = ProjectConfig() - self.base_environment: Union[MappingNode, Dict[str, str]] = {} # The base set of environment variables + self.base_environment: MappingNode = Node.from_dict({}) # The base set of environment variables self.base_env_nocache: List[str] = [] # The base nocache mask (list) for the environment # Remote specs for communicating with remote services @@ -148,17 +151,17 @@ def __init__( # This is a lookup table of lists indexed by project, # the child dictionaries are lists of ScalarNodes indicating # junction names - self._junction_duplicates: Dict[str, List[str]] = {} + self._junction_duplicates: Dict[str, List[ScalarNode]] = {} # A list of project relative junctions to consider as 'internal', # stored as ScalarNodes. - self._junction_internal: List[str] = [] + self._junction_internal: SequenceNode | list[ScalarNode] = [] self._partially_loaded: bool = False self._fully_loaded: bool = False self._project_includes: Optional[Includes] = None - self._fully_loaded_callbacks: List[Callable[[], None]] = [] + self._fully_loaded_callbacks: Optional[List[Callable[[], None]]] = [] # # Initialization body @@ -192,15 +195,15 @@ def options(self): return self.config.options @property - def base_variables(self): + def base_variables(self) -> Optional[MappingNode]: return self.config.base_variables @property - def element_overrides(self): + def element_overrides(self) -> MappingNode: return self.config.element_overrides @property - def source_overrides(self): + def source_overrides(self) -> MappingNode: return self.config.source_overrides ######################################################## @@ -244,7 +247,7 @@ def get_alias_url(self, alias: str, *, first_pass: bool = False) -> Optional[str # This method is provided for :class:`.Source` objects to resolve # fully qualified urls based on the shorthand which is allowed # to be specified in the YAML - def translate_url(self, url, *, source, first_pass=False): + def translate_url(self, url: str, *, source: Source, first_pass=False) -> str: if url and utils._ALIAS_SEPARATOR in url: url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) @@ -312,6 +315,7 @@ def get_shell_config(self): def get_path_from_node(self, node, *, check_is_file=False, check_is_dir=False): path_str = node.as_str() path = Path(path_str) + assert self._absolute_directory_path, "Must have an absolute directory path" full_path = self._absolute_directory_path / path if full_path.is_symlink(): @@ -389,7 +393,8 @@ def get_path_from_node(self, node, *, check_is_file=False, check_is_dir=False): # Returns: # (Element): A newly created Element object of the appropriate kind # - def create_element(self, load_element): + def create_element(self, load_element: LoadElement) -> Element: + assert self.element_factory, "must have a element factory" return self.element_factory.create(self._context, self, load_element) # create_source() @@ -404,6 +409,7 @@ def create_element(self, load_element): # (Source): A newly created Source object of the appropriate kind # def create_source(self, meta, variables): + assert self.source_factory, "must have a source factory" return self.source_factory.create(self._context, self, meta, variables) # alias_exists() @@ -443,7 +449,7 @@ def alias_exists(self, alias, *, source, first_pass=False): ), reason="missing-alias-mapping", ) - + assert config._aliases, "Must have aliases" return config._aliases.get_str(alias, default=None) is not None # get_alias_uris() @@ -467,6 +473,7 @@ def get_alias_uris( else: config = self.config + assert config._aliases, "Must have aliases" if not alias or alias not in config._aliases: # pylint: disable=unsupported-membership-test return [None] @@ -494,6 +501,7 @@ def get_alias_uris( uri_list += list_to_add if policy in (_SourceUriPolicy.ALL, _SourceUriPolicy.ALIASES): + assert config._aliases, "Must have aliases" uri_list.append(config._aliases.get_str(alias)) return [AliasSubstitution(alias, mirror) for mirror in uri_list] @@ -509,9 +517,10 @@ def get_alias_uris( # (list): A list of loaded Element # def load_elements(self, targets): - + assert self.loader, "must have a loader" with self._context.messenger.simple_task("Loading elements", silent_nested=True) as task: self.load_context.set_task(task) + load_elements = self.loader.load(targets) self.load_context.set_task(None) @@ -630,9 +639,11 @@ def get_default_targets(self): # Returns: # (bool): Whether the loader is specified as duplicate # - def junction_is_duplicated(self, project_name, loader): + def junction_is_duplicated(self, project_name: str, loader: Loader) -> bool: + + assert self.loader, "must have a loader" - junctions = self._junction_duplicates.get(project_name, {}) + junctions: list[ScalarNode] = self._junction_duplicates.get(project_name, []) # Iterate over all paths specified by this project and see # if we find a match for the specified loader. @@ -660,7 +671,9 @@ def junction_is_duplicated(self, project_name, loader): # Returns: # (bool): Whether the loader is specified as internal # - def junction_is_internal(self, loader): + def junction_is_internal(self, loader: Loader) -> bool: + + assert self.loader, "must have a loader" # Iterate over all paths specified by this project and see # if we find a match for the specified loader. @@ -702,7 +715,8 @@ def loaded_projects(self): # callback (Callable[[], None]): A function to call once fully loaded # def register_fully_loaded_callback(self, callback: Callable[[], None]): - assert not self._fully_loaded + assert not self._fully_loaded, "You can't register a new callback after the project is fully loaded." + assert self._fully_loaded_callbacks is not None, "Callbacks are missing, so they must have been processed" self._fully_loaded_callbacks.append(callback) ######################################################## @@ -839,9 +853,10 @@ def _validate_version(self, config_node): # # Raises: LoadError if there was a problem with the project.conf # - def _load(self, *, parent_loader=None, provenance_node=None): + def _load(self, *, parent_loader=None, provenance_node: Optional[Node] = None): # Load builtin default + assert self.directory, "project must have a directory by this point" projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE) self._default_config_node = _yaml.load(_site.default_project_config, shortname="projectconfig.yaml") @@ -935,6 +950,7 @@ def _load(self, *, parent_loader=None, provenance_node=None): ) if self.ref_storage == ProjectRefStorage.PROJECT_REFS: + assert self.junction_refs, "Project must have junction refs by this point" self.junction_refs.load(self.first_pass_config.options) # _load_second_pass() @@ -943,6 +959,7 @@ def _load(self, *, parent_loader=None, provenance_node=None): # def _load_second_pass(self): project_conf_second_pass = self._project_conf.clone() + assert self._project_includes, "Project must have includes by this point" self._project_includes.process(project_conf_second_pass, process_project_options=False) config = self._default_config_node.clone() project_conf_second_pass._composite(config) @@ -993,6 +1010,7 @@ def _load_second_pass(self): # Load project.refs if it exists, this may be ignored. if self.ref_storage == ProjectRefStorage.PROJECT_REFS: + assert self.refs, "Project must have refs by this point" self.refs.load(self.options) # Parse shell options @@ -1030,8 +1048,9 @@ def _load_second_pass(self): "source-provenance-attributes", None ) or config.get_mapping("source-provenance-attributes") - for callback in self._fully_loaded_callbacks: - callback() + if self._fully_loaded_callbacks: + for callback in self._fully_loaded_callbacks: + callback() self._fully_loaded_callbacks = None # _load_pass(): @@ -1044,10 +1063,11 @@ def _load_second_pass(self): # output (ProjectConfig) - ProjectConfig to load configuration onto. # ignore_unknown (bool) - Whether option loader shoud ignore unknown options. # - def _load_pass(self, config, output, *, ignore_unknown=False): + def _load_pass(self, config: MappingNode, output: ProjectConfig, *, ignore_unknown: bool = False): # Load project options options_node = config.get_mapping("options", default={}) + assert output.options, "Must have options" output.options.load(options_node) if self.junction: # load before user configuration @@ -1144,6 +1164,7 @@ def _load_pass(self, config, output, *, ignore_unknown=False): # even if the mirrors are specified in user configuration. variables.expand(mirrors_node) + assert self.source_mirror_factory, "Project must have a mirror factory by this point" # Collect SourceMirror objects for mirror_node in mirrors_node: mirror = self.source_mirror_factory.create(self._context, self, mirror_node) diff --git a/src/buildstream/_state.py b/src/buildstream/_state.py index b581642b3..4c9be457a 100644 --- a/src/buildstream/_state.py +++ b/src/buildstream/_state.py @@ -150,7 +150,7 @@ def set_task_changed_callback(self, callback: Optional[Callable[[], None]]) -> N # Args: # progress: The maximum progress possible for this task # - def set_maximum_progress(self, progress: int) -> None: + def set_maximum_progress(self, progress: int | None) -> None: self.maximum_progress = progress self._notify_task_changed() diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index d326b9532..7ec841e67 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -258,7 +258,7 @@ def query_cache(self, elements, *, sources_of_cached_elements=False, only_source # def shell( self, - target: str, + target: Optional[str], scope: _Scope, prompt: Callable[[Element], str], *, @@ -278,6 +278,9 @@ def shell( if unique_id and target is None: element = Plugin._lookup(unique_id) else: + assert ( + target + ), "If method is called from bst shell cli, the cli ensures a target is present, but other uses must provide a unique_id or target" if usebuildtree: selection = _PipelineSelection.NONE elif scope == _Scope.BUILD: @@ -414,6 +417,7 @@ def build( # Assert that the elements are consistent _pipeline.assert_consistent(self._context, elements) + assert self._sourcecache, "Stream must have source cache" source_push_enabled = self._sourcecache.has_push_remotes() # If source push is enabled, the source cache status of all elements @@ -423,7 +427,7 @@ def build( # Now construct the queues # self._reset() - + assert self._artifacts, "Stream must have artifacts" if self._artifacts.has_fetch_remotes(): self._add_queue(PullQueue(self._scheduler)) @@ -552,6 +556,7 @@ def source_push( self.query_cache(elements, only_sources=True) + assert self._sourcecache, "Stream must have source cache" if not self._sourcecache.has_push_remotes(): raise StreamError("No source caches available for pushing sources") @@ -601,6 +606,7 @@ def pull( ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) + assert self._artifacts, "Stream must have artifacts" if not self._artifacts.has_fetch_remotes(): raise StreamError("No artifact caches available for pulling artifacts") @@ -649,6 +655,7 @@ def push( ignore_project_artifact_remotes=ignore_project_artifact_remotes, ) + assert self._artifacts, "Stream must have artifacts" if not self._artifacts.has_push_remotes(): raise StreamError("No artifact caches available for pushing artifacts") @@ -812,7 +819,7 @@ def artifact_show( self.query_cache(target_objects) not_cached_locally = [element for element in target_objects if not element._cached()] - + assert self._artifacts, "Stream must have artifacts" if self._artifacts.has_fetch_remotes(): self._resolve_cached_remotely(not_cached_locally) @@ -905,6 +912,7 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE): ref_removed = False for ref in remove_refs: try: + assert self._artifacts, "Stream must have artifacts" self._artifacts.remove(ref) except ArtifactError as e: self._context.messenger.warn(str(e)) @@ -936,7 +944,7 @@ def source_checkout( self, target: str, *, - location: Optional[str] = None, + location: str = "", force: bool = False, deps=_PipelineSelection.NONE, except_targets: Iterable[str] = (), @@ -1230,6 +1238,7 @@ def workspace_list(self): # (list of str): The element names after redirecting # def redirect_element_names(self, elements): + assert self._project, "Stream must have project to redirect element names" element_dir = self._project.element_path load_elements = [] output_elements = set() @@ -1274,7 +1283,8 @@ def get_default_target(self): # def get_default_targets(self): self._assert_project("Unable to determine default targets") - return self._project.get_default_targets() + # (Type checker still thinks self._project could be None, but we assert it with the function above, so error ignored) + return self._project.get_default_targets() # type: ignore ############################################################# # Scheduler API forwarding # @@ -1445,6 +1455,7 @@ def _load_elements(self, target_groups): targets = list(itertools.chain(*target_groups)) with PROFILER.profile(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, "-") for t in targets)): + assert self._project, "Stream must have a project loaded" elements = self._project.load_elements(targets) # Now create element groups to match the input target groups @@ -1618,6 +1629,7 @@ def _track_cross_junction_filter(self, project, elements, cross_junction_request # We can track anything if the toplevel project uses project.refs # + assert self._project, "Stream must have a project loaded to track cross junction filter" if self._project.ref_storage == ProjectRefStorage.PROJECT_REFS: return elements @@ -1914,7 +1926,7 @@ def _check_location_writable(self, location, force=False, tar=False): def _source_checkout( self, elements, - location=None, + location: str = "", force=False, deps="none", tar=False, @@ -2135,6 +2147,7 @@ def _expand_and_classify_targets( # Glob the artifact names and add the results to the set # + assert self._artifacts, "Stream must have artifacts to process globs" for glob in artifact_globs: glob_results = self._artifacts.list_artifacts(glob=glob) for artifact_name in glob_results: diff --git a/src/buildstream/_testing/_cachekeys.py b/src/buildstream/_testing/_cachekeys.py index aa28f156a..7989d7f6b 100644 --- a/src/buildstream/_testing/_cachekeys.py +++ b/src/buildstream/_testing/_cachekeys.py @@ -22,7 +22,7 @@ class CacheKeyTestError(Exception): pass -def check_cache_key_stability(project_path: os.PathLike, cli: Cli) -> None: +def check_cache_key_stability(project_path: str, cli: Cli) -> None: """ Check that the cache key of various elements has not changed. diff --git a/src/buildstream/_testing/integration.py b/src/buildstream/_testing/integration.py index 4fcb9ae92..e30643e44 100644 --- a/src/buildstream/_testing/integration.py +++ b/src/buildstream/_testing/integration.py @@ -67,6 +67,19 @@ def assert_contains(directory, expected, strict=False): class IntegrationCache: + root: str + """ + local path to the root of the cache + """ + sources: str + """ + local path to the sources directory of the cache + """ + cachedir: str + """ + path to the temporary directory containing the root of the cache + """ + def __init__(self, cache): self.root = os.path.abspath(cache) os.makedirs(cache, exist_ok=True) diff --git a/src/buildstream/_testing/runcli.py b/src/buildstream/_testing/runcli.py index 24af09da7..03708fdec 100644 --- a/src/buildstream/_testing/runcli.py +++ b/src/buildstream/_testing/runcli.py @@ -32,6 +32,10 @@ import itertools import traceback from contextlib import contextmanager, ExitStack +from types import TracebackType +from os import PathLike +from itertools import batched +from typing import Optional, Any, Generator, TYPE_CHECKING from ruamel import yaml import pytest @@ -42,27 +46,43 @@ # CliRunner convenience API (click.testing module) does not support # separation of stdout/stderr. # -from _pytest.capture import MultiCapture, FDCapture, FDCaptureBinary +from _pytest.capture import MultiCapture, FDCapture, FDCaptureBinary, CaptureBase # Import the main cli entrypoint from buildstream._frontend import cli as bst_cli from buildstream import _yaml, node from buildstream._cas import CASCache -from buildstream.element import _get_normal_name, _compose_artifact_name +from buildstream.element import _get_normal_name, _compose_artifact_name, Element +from buildstream.node import MappingNode +from buildstream.exceptions import ErrorDomain +from buildstream._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Digest +from buildstream._testing.integration import IntegrationCache # Special private exception accessor, for test case purposes from buildstream._exceptions import BstError, get_last_exception, get_last_task_error from buildstream._protos.buildstream.v2 import artifact_pb2 +if TYPE_CHECKING: + from tests.conftest import RemoteServices + # Wrapper for the click.testing result class Result: - def __init__(self, exit_code=None, exception=None, exc_info=None, output=None, stderr=None): - self.exit_code = exit_code - self.exc = exception - self.exc_info = exc_info - self.output = output - self.stderr = stderr + def __init__( + self, + exit_code: int | None = None, + exception: SystemExit | Exception | None = None, + exc_info: tuple[type[BaseException], BaseException, TracebackType] | tuple[None, None, None] | None = None, + output: str | None = None, + stderr: str | None = None, + ): + self.exit_code: int | None = exit_code + self.exc: SystemExit | Exception | None = exception + self.exc_info: tuple[type[BaseException], BaseException, TracebackType] | tuple[None, None, None] | None = ( + exc_info + ) + self.output: str | None = output + self.stderr: str | None = stderr self.unhandled_exception = False # The last exception/error state is stored at exception @@ -99,11 +119,11 @@ def __init__(self, exit_code=None, exception=None, exc_info=None, output=None, s # Raises: # (AssertionError): If the session did not complete successfully # - def assert_success(self, fail_message=""): + def assert_success(self, fail_message: str = ""): assert self.exit_code == 0, fail_message assert self.exc is None, fail_message assert self.exception is None, fail_message - assert self.unhandled_exception is False + assert self.unhandled_exception is False, fail_message # assert_main_error() # @@ -119,7 +139,10 @@ def assert_success(self, fail_message=""): # Raises: # (AssertionError): If any of the assertions fail # - def assert_main_error(self, error_domain, error_reason, fail_message="", *, debug=False): + def assert_main_error( + self, error_domain: ErrorDomain, error_reason: Any, fail_message: str = "", *, debug: bool = False + ): + assert self.exception is not None, fail_message if debug: print(""" Exit code: {} @@ -129,9 +152,8 @@ def assert_main_error(self, error_domain, error_reason, fail_message="", *, debu """.format(self.exit_code, self.exception, self.exception.domain, self.exception.reason)) assert self.exit_code == -1, fail_message assert self.exc is not None, fail_message - assert self.exception is not None, fail_message assert isinstance(self.exception, BstError), fail_message - assert self.unhandled_exception is False + assert self.unhandled_exception is False, fail_message assert self.exception.domain == error_domain, fail_message assert self.exception.reason == error_reason, fail_message @@ -150,7 +172,7 @@ def assert_main_error(self, error_domain, error_reason, fail_message="", *, debu # Raises: # (AssertionError): If any of the assertions fail # - def assert_task_error(self, error_domain, error_reason, fail_message=""): + def assert_task_error(self, error_domain: ErrorDomain, error_reason, fail_message: str = ""): assert self.exit_code == -1, fail_message assert self.exc is not None, fail_message @@ -172,7 +194,7 @@ def assert_task_error(self, error_domain, error_reason, fail_message=""): # Raises: # (AssertionError): If any of the assertions fail # - def assert_shell_error(self, fail_message=""): + def assert_shell_error(self, fail_message: str = ""): assert self.exit_code == 1, fail_message # get_start_order() @@ -186,7 +208,8 @@ def assert_shell_error(self, fail_message=""): # Returns: # (list): A list of element names in the order which they first appeared in the result # - def get_start_order(self, activity): + def get_start_order(self, activity: str): + assert self.stderr is not None, "No stderr available to read" results = re.findall(r"\[\s*{}:(\S+)\s*\]\s*START\s*.*\.log".format(activity), self.stderr) if results is None: return [] @@ -203,6 +226,7 @@ def get_start_order(self, activity): # (list): A list of element names # def get_tracked_elements(self): + assert self.stderr is not None, "No stderr available to read" tracked = re.findall(r"\[\s*track:(\S+)\s*]", self.stderr) if tracked is None: return [] @@ -210,6 +234,7 @@ def get_tracked_elements(self): return list(tracked) def get_built_elements(self): + assert self.stderr is not None, "No stderr available to read" built = re.findall(r"\[\s*build:(\S+)\s*\]\s*SUCCESS\s*Caching artifact", self.stderr) if built is None: return [] @@ -217,6 +242,7 @@ def get_built_elements(self): return list(built) def get_pushed_elements(self): + assert self.stderr is not None, "No stderr available to read" pushed = re.findall(r"\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed artifact", self.stderr) if pushed is None: return [] @@ -224,6 +250,7 @@ def get_pushed_elements(self): return list(pushed) def get_pulled_elements(self): + assert self.stderr is not None, "No stderr available to read" pulled = re.findall(r"\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact", self.stderr) if pulled is None: return [] @@ -231,6 +258,7 @@ def get_pulled_elements(self): return list(pulled) def get_discarded_elements(self): + assert self.stderr is not None, "No stderr available to read" discarded = re.findall(r"\[\s*(?:main|pull):(\S+)\s*\]\s*INFO\s*Discarded failed build", self.stderr) if discarded is None: return [] @@ -239,18 +267,18 @@ def get_discarded_elements(self): class Cli: - def __init__(self, directory, verbose=True, default_options=None): - self.directory = directory - self.config = None - self.verbose = verbose - self.artifact = TestArtifact() + def __init__(self, directory: str, verbose: bool = True, default_options: list[str] | None = None): + self.directory: str = directory + self.config: dict[str, Any] | None = None + self.verbose: bool = verbose + self.artifact: TestArtifact = TestArtifact() os.makedirs(directory) if default_options is None: default_options = [] - self.default_options = default_options + self.default_options: list[str] = default_options # configure(): # @@ -260,7 +288,7 @@ def __init__(self, directory, verbose=True, default_options=None): # Args: # config (dict): The user configuration to use # - def configure(self, config): + def configure(self, config: dict): if self.config is None: self.config = {} @@ -276,16 +304,18 @@ def configure(self, config): # element_name (str): The name of the element artifact # cache_dir (str): Specific cache dir to remove artifact from # - def remove_artifact_from_cache(self, project, element_name, *, cache_dir=None): + def remove_artifact_from_cache(self, project: str, element_name: str, *, cache_dir: str | None = None): # Read configuration to figure out where artifacts are stored if not cache_dir: default = os.path.join(project, "cache") if self.config is not None: cache_dir = self.config.get("cachedir", default) + assert isinstance( + cache_dir, str + ), f"Cache directory in the config is not a valid string path: it's a {type(cache_dir)}" else: cache_dir = default - self.artifact.remove_artifact_from_cache(cache_dir, element_name) # run(): @@ -301,7 +331,16 @@ def remove_artifact_from_cache(self, project, element_name, *, cache_dir=None): # args (list): A list of arguments to pass buildstream # binary_capture (bool): Whether to capture the stdout/stderr as binary # - def run(self, project=None, silent=False, env=None, cwd=None, options=None, args=None, binary_capture=False): + def run( + self, + project: str | None = None, + silent: bool = False, + env: Optional[dict[str, str]] = None, + cwd: str | None = None, + options: list[str] | None = None, + args: list[str] | None = None, + binary_capture: bool = False, + ) -> Result: # We don't want to carry the state of one bst invocation into another # bst invocation. Since node _FileInfo objects hold onto BuildStream @@ -333,7 +372,7 @@ def run(self, project=None, silent=False, env=None, cwd=None, options=None, args if project: bst_args += ["--directory", str(project)] - for option, value in options: + for option, value in batched(options, n=2): bst_args += ["--option", option, value] bst_args += args @@ -355,22 +394,22 @@ def run(self, project=None, silent=False, env=None, cwd=None, options=None, args if result.stderr: print("Program stderr was:\n{}".format(result.stderr)) - if result.exc_info and result.exc_info[0] != SystemExit: + if result.exc_info and result.exc_info[0] is not SystemExit: traceback.print_exception(*result.exc_info) return result - def _invoke(self, cli_object, args=None, binary_capture=False): + def _invoke(self, cli_object, args: list[str] | None = None, binary_capture: bool = False) -> Result: exc_info = None - exception = None - exit_code = 0 + exception: Optional[Exception | SystemExit] = None + exit_code: int | str | None = 0 # Temporarily redirect sys.stdin to /dev/null to ensure that # Popen doesn't attempt to read pytest's dummy stdin. old_stdin = sys.stdin with open(os.devnull, "rb") as devnull: sys.stdin = devnull - capture_kind = FDCaptureBinary if binary_capture else FDCapture + capture_kind: type[CaptureBase] = FDCaptureBinary if binary_capture else FDCapture capture = MultiCapture(out=capture_kind(1), err=capture_kind(2), in_=None) capture.start_capturing() @@ -398,7 +437,7 @@ def _invoke(self, cli_object, args=None, binary_capture=False): sys.stdin = old_stdin out, err = capture.readouterr() capture.stop_capturing() - + assert isinstance(exit_code, int | None), "Exit code should be a number or None at this point" return Result(exit_code=exit_code, exception=exception, exc_info=exc_info, output=out, stderr=err) # Fetch an element state by name by @@ -407,22 +446,26 @@ def _invoke(self, cli_object, args=None, binary_capture=False): # If you need to get the states of multiple elements, # then use get_element_states(s) instead. # - def get_element_state(self, project, element_name): + def get_element_state(self, project: str, element_name: str) -> str: result = self.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{state}", element_name] ) result.assert_success() + assert result.output is not None, "bst show was successful, but doesn't seem to have shown anything in stdout" + return result.output.strip() # Fetch the states of elements for a given target / deps # # Returns a dictionary with the element names as keys # - def get_element_states(self, project, targets, deps="all"): + def get_element_states(self, project: str, targets: list[str], deps: str = "all") -> dict[str, str]: result = self.run( project=project, silent=True, args=["show", "--deps", deps, "--format", "%{name}||%{state}", *targets] ) result.assert_success() + assert result.output is not None, "bst show was successful, but doesn't seem to have shown anything in stdout" + lines = result.output.splitlines() states = {} for line in lines: @@ -433,16 +476,18 @@ def get_element_states(self, project, targets, deps="all"): # Fetch an element's cache key by invoking bst show # on the project with the CLI # - def get_element_key(self, project, element_name): + def get_element_key(self, project: str, element_name: str) -> str: result = self.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{full-key}", element_name] ) result.assert_success() + + assert result.output is not None, "bst show was successful, but doesn't seem to have shown anything in stdout" return result.output.strip() # Get the decoded config of an element. # - def get_element_config(self, project, element_name): + def get_element_config(self, project: str, element_name: str) -> Any: result = self.run( project=project, silent=True, args=["show", "--deps", "none", "--format", "%{config}", element_name] ) @@ -453,7 +498,7 @@ def get_element_config(self, project, element_name): # Fetch the elements that would be in the pipeline with the given # arguments. # - def get_pipeline(self, project, elements, except_=None, scope="all"): + def get_pipeline(self, project: str, elements: list[str], except_: list[str] | None = None, scope: str = "all"): if except_ is None: except_ = [] @@ -462,12 +507,15 @@ def get_pipeline(self, project, elements, except_=None, scope="all"): result = self.run(project=project, silent=True, args=args + elements) result.assert_success() + + assert result.output is not None, "bst show was successful, but doesn't seem to have shown anything in stdout" return result.output.splitlines() # Fetch an element's complete artifact name, cache_key will be generated # if not given. # - def get_artifact_name(self, project, project_name, element_name, cache_key=None): + def get_artifact_name(self, project: str, project_name: str, element_name: str, cache_key: str | None = None): + if not cache_key: cache_key = self.get_element_key(project, element_name) @@ -482,7 +530,16 @@ class CliIntegration(Cli): # # This supports the same arguments as Cli.run(), see run_project_config(). # - def run(self, project=None, silent=False, env=None, cwd=None, options=None, args=None, binary_capture=False): + def run( + self, + project: str | None = None, + silent: bool = False, + env: dict | None = None, + cwd: str | None = None, + options: list[str] | None = None, + args: list[str] | None = None, + binary_capture: bool = False, + ): return self.run_project_config( project=project, silent=silent, env=env, cwd=cwd, options=options, args=args, binary_capture=binary_capture ) @@ -500,7 +557,7 @@ def run(self, project=None, silent=False, env=None, cwd=None, options=None, args # be a dictionary of additional project configuration options, and # will be composited on top of the already loaded project.conf # - def run_project_config(self, *, project_config=None, **kwargs): + def run_project_config(self, *, project_config: MappingNode | None = None, **kwargs) -> Result: # First load the project.conf and substitute {project_dir} # @@ -569,9 +626,16 @@ class CliRemote(CliIntegration): # # Returns a list of configured services (by names). # - def ensure_services(self, actions=True, execution=True, storage=True, artifacts=False, sources=False): + def ensure_services( + self, + actions: bool = True, + execution: bool = True, + storage: bool = True, + artifacts: bool = False, + sources: bool = False, + ) -> list[str]: # Build a list of configured services by name: - configured_services = [] + configured_services: list[str] = [] if not self.config: return configured_services @@ -621,7 +685,7 @@ class TestArtifact: # cache_dir (str): Specific cache dir to remove artifact from # element_name (str): The name of the element artifact # - def remove_artifact_from_cache(self, cache_dir, element_name): + def remove_artifact_from_cache(self, cache_dir: str, element_name: str): cache_dir = os.path.join(cache_dir, "artifacts", "refs") @@ -648,7 +712,7 @@ def remove_artifact_from_cache(self, cache_dir, element_name): # Returns: # (bool): If the cache contains the element's artifact # - def is_cached(self, cache_dir, element, element_key): + def is_cached(self, cache_dir: str, element: Element, element_key: str) -> bool: # cas = CASCache(str(cache_dir)) artifact_ref = element.get_artifact_name(element_key) @@ -666,7 +730,7 @@ def is_cached(self, cache_dir, element, element_key): # Returns: # (Digest): The digest stored in the ref # - def get_digest(self, cache_dir, element, element_key): + def get_digest(self, cache_dir: str, element: Element, element_key: str) -> Digest: artifact_ref = element.get_artifact_name(element_key) artifact_dir = os.path.join(cache_dir, "artifacts", "refs") @@ -688,10 +752,12 @@ def get_digest(self, cache_dir, element, element_key): # (str): path to extracted buildtree directory, does not guarantee # existence. @contextmanager - def extract_buildtree(self, cache_dir, tmpdir, ref): + def extract_buildtree( + self, cache_dir: str | PathLike[str], tmpdir: str | PathLike[str], ref: str + ) -> Generator[str | None]: artifact = artifact_pb2.Artifact() try: - with open(os.path.join(cache_dir, "artifacts", "refs", ref), "rb") as f: + with open(os.path.join(str(cache_dir), "artifacts", "refs", ref), "rb") as f: artifact.ParseFromString(f.read()) except FileNotFoundError: yield None @@ -717,7 +783,7 @@ def extract_buildtree(self, cache_dir, tmpdir, ref): # (str): path to extracted subdir directory, does not guarantee # existence. @contextmanager - def _extract_subdirectory(self, tmpdir, digest): + def _extract_subdirectory(self, tmpdir: str | PathLike[str], digest: Digest) -> Generator[str | None]: with tempfile.TemporaryDirectory() as extractdir: try: cas = CASCache(str(tmpdir), casd=None) @@ -732,7 +798,7 @@ def _extract_subdirectory(self, tmpdir, digest): # Use result = cli.run([arg1, arg2]) to run buildstream commands # @pytest.fixture() -def cli(tmpdir): +def cli(tmpdir: str | PathLike[str]) -> Cli: directory = os.path.join(str(tmpdir), "cache") return Cli(directory) @@ -744,7 +810,7 @@ def cli(tmpdir): # when running `bst shell`, but unfortunately cannot produce nice # stacktraces. @pytest.fixture() -def cli_integration(tmpdir, integration_cache): +def cli_integration(tmpdir: str | PathLike[str], integration_cache: IntegrationCache) -> Generator[CliIntegration]: directory = os.path.join(str(tmpdir), "cache") fixture = CliIntegration(directory) @@ -776,7 +842,7 @@ def cli_integration(tmpdir, integration_cache): # when running `bst shell`, but unfortunately cannot produce nice # stacktraces. @pytest.fixture() -def cli_remote_execution(tmpdir, remote_services): +def cli_remote_execution(tmpdir: str | PathLike[str], remote_services: "RemoteServices") -> CliRemote: directory = os.path.join(str(tmpdir), "cache") fixture = CliRemote(directory) @@ -823,7 +889,7 @@ def cli_remote_execution(tmpdir, remote_services): @contextmanager -def chdir(directory): +def chdir(directory: str | PathLike[str]) -> Generator[None]: old_dir = os.getcwd() os.chdir(directory) yield @@ -831,7 +897,7 @@ def chdir(directory): @contextmanager -def environment(env): +def environment(env: dict) -> Generator[None]: old_env = {} for key, value in env.items(): @@ -851,7 +917,7 @@ def environment(env): @contextmanager -def configured(directory, config=None): +def configured(directory: str | PathLike[str], config: dict | None = None) -> Generator[str]: # Ensure we've at least relocated the caches to a temp directory if not config: diff --git a/src/buildstream/_variables.pyi b/src/buildstream/_variables.pyi index 8d9acdbdc..e84b434ce 100644 --- a/src/buildstream/_variables.pyi +++ b/src/buildstream/_variables.pyi @@ -13,10 +13,11 @@ # from typing import Optional -from .node import MappingNode, Node +from .node import MappingNode, Node, ScalarNode class Variables: def __init__(self, node: MappingNode) -> None: ... def check(self) -> None: ... def expand(self, node: Node) -> None: ... def get(self, name: str) -> Optional[str]: ... + def subst(self, node: ScalarNode) -> str: ... diff --git a/src/buildstream/_workspaces.py b/src/buildstream/_workspaces.py index 4b9aaf13c..c4b1e02f9 100644 --- a/src/buildstream/_workspaces.py +++ b/src/buildstream/_workspaces.py @@ -374,7 +374,7 @@ def create_workspace(self, target, path, *, checkout): # Returns: # (None|Workspace) # - def get_workspace(self, element_name): + def get_workspace(self, element_name: str) -> Workspace | None: if element_name not in self._workspaces: return None return self._workspaces[element_name] diff --git a/src/buildstream/_yaml.pyi b/src/buildstream/_yaml.pyi index 224abc51a..1b855a00a 100644 --- a/src/buildstream/_yaml.pyi +++ b/src/buildstream/_yaml.pyi @@ -13,6 +13,12 @@ # from typing import Optional -from .node import MappingNode +from .node import MappingNode, _SYNTHETIC_FILE_INDEX, SequenceNode def load(filename: str, shortname: str, copy_tree: bool = False, project: Optional[object] = None) -> MappingNode: ... +def load_data( + data: str, file_index: int = _SYNTHETIC_FILE_INDEX, file_name: str | None = None, copy_tree: bool = False +) -> MappingNode: ... +def roundtrip_dump(contents, file=None): ... +def roundtrip_dump_string(node: dict | list) -> str: ... +def roundtrip_load(filename: str, allow_missing: bool = False) -> MappingNode: ... diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 209b9e9a9..d346bd312 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -62,29 +62,51 @@ --------------- """ +# For 3.7+ support, not necessary and deprecated in 3.14+ +from __future__ import annotations +from collections.abc import Callable + import os import re import stat import copy import warnings -from contextlib import contextmanager, suppress +from contextlib import contextmanager from functools import partial from itertools import chain import string from threading import Lock -from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence +from typing import ( + cast, + TYPE_CHECKING, + Dict, + Iterator, + Iterable, + List, + Optional, + Set, + Sequence, + Pattern, + Union, + TextIO, + Generator, + Any, + TypeAlias, +) from pyroaring import BitMap # pylint: disable=no-name-in-module -from . import _yaml + +from ._workspaces import Workspace +from ._sourcecache import SourceCache +from ._artifactcache import ArtifactCache +from ._state import Task +from . import _yaml, utils, _cachekey, _site from ._variables import Variables from ._versions import BST_CORE_ARTIFACT_VERSION from ._exceptions import BstError, LoadError, ImplError, SourceCacheError, CachedFailure from .exceptions import ErrorDomain, LoadErrorReason from .utils import FileListResult, BST_ARBITRARY_TIMESTAMP -from . import utils -from . import _cachekey -from . import _site from .node import Node, MappingNode, ScalarNode from .plugin import Plugin from .sandbox import _SandboxFlags, SandboxCommandError @@ -148,15 +170,15 @@ class DependencyConfiguration: :func:`Element.configure_dependencies() ` """ - def __init__(self, element: "Element", path: str, config: Optional["MappingNode"]): + def __init__(self, element: Element, path: str, config: Optional[MappingNode]): - self.element = element # type: Element + self.element: Element = element """The dependency Element""" - self.path = path # type: str + self.path: str = path """The path used to refer to this dependency""" - self.config = config # type: Optional[MappingNode] + self.config: Optional[MappingNode] = config """The custom :term:`dependency configuration `, or ``None`` if no custom configuration was provided""" @@ -171,11 +193,11 @@ class Element(Plugin): """ # The defaults from the yaml file and project - __defaults = None + __defaults: Optional[MappingNode] = None # A hash of Element by LoadElement - __instantiated_elements = {} # type: Dict[LoadElement, Element] + __instantiated_elements: Dict[LoadElement, Element] = {} # A list of (source, ref) tuples which were redundantly specified - __redundant_source_refs = [] # type: List[Tuple[Source, SourceRef]] + __redundant_source_refs: List[Tuple[Source, SourceRef]] = [] BST_ARTIFACT_VERSION = 0 """The element plugin's artifact version @@ -214,15 +236,15 @@ class Element(Plugin): def __init__( self, - context: "Context", - project: "Project", - load_element: "LoadElement", + context: Context, + project: Project, + load_element: LoadElement, plugin_conf: Optional[str], *, artifact_key: Optional[str] = None, ): - self.__cache_key_dict = None # Dict for cache key calculation + self.__cache_key_dict: Optional[dict[str, Any]] = None # Dict for cache key calculation self.__cache_key: Optional[str] = None # Our cached cache key super().__init__(load_element.name, context, project, load_element.node, "element") @@ -253,59 +275,66 @@ def __init__( # # Internal instance properties # - self._depth = None # Depth of Element in its current dependency graph + self._depth: int | None = None # Depth of Element in its current dependency graph self._overlap_collectors: Dict[Sandbox, OverlapCollector] = {} # Active overlap collector per sandbox - self._description = load_element.description or "" # type: str + self._description: str = load_element.description or "" # # Private instance properties # # Cache of proxies instantiated, indexed by the proxy owner - self.__proxies = {} # type: Dict[Element, ElementProxy] + self.__proxies: dict[Element, ElementProxy] = {} # Direct runtime dependency Elements - self.__runtime_dependencies = [] # type: List[Element] + self.__runtime_dependencies: list[Element] = [] # Direct build dependency Elements - self.__build_dependencies = [] # type: List[Element] + self.__build_dependencies: list[Element] = [] # Direct build dependency subset which require strict rebuilds - self.__strict_dependencies = [] # type: List[Element] + self.__strict_dependencies: list[Element] = [] # Direct reverse build dependency Elements - self.__reverse_build_deps = set() # type: Set[Element] + self.__reverse_build_deps: set[Element] = set() # Direct reverse runtime dependency Elements - self.__reverse_runtime_deps = set() # type: Set[Element] - self.__build_deps_uncached = None # Build dependencies which are not yet cached - self.__runtime_deps_uncached = None # Runtime dependencies which are not yet cached - self.__ready_for_runtime_and_cached = False # Whether all runtime deps are cached, as well as the element - self.__cached_remotely = None # Whether the element is cached remotely - self.__sources = ElementSources(context, project, self) # The element sources + self.__reverse_runtime_deps: set[Element] = set() + self.__build_deps_uncached: Optional[int] = None # Build dependencies which are not yet cached + self.__runtime_deps_uncached: Optional[int] = None # Runtime dependencies which are not yet cached + self.__ready_for_runtime_and_cached: bool = ( + False # Whether all runtime deps are cached, as well as the element + ) + self.__cached_remotely: Optional[bool] = None # Whether the element is cached remotely + self.__sources: ElementSources = ElementSources(context, project, self) # The element sources self.__weak_cache_key: Optional[str] = None # Our cached weak cache key self.__strict_cache_key: Optional[str] = None # Our cached cache key for strict builds - self.__artifacts = context.artifactcache # Artifact cache - self.__sourcecache = context.sourcecache # Source cache - self.__assemble_scheduled = False # Element is scheduled to be assembled - self.__assemble_done = False # Element is assembled - self.__pull_pending = False # Whether pull is pending - self.__cached_successfully = None # If the Element is known to be successfully cached - self.__splits = None # Resolved regex objects for computing split domains - self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap - self.__tainted = None # Whether the artifact is tainted and should not be shared - self.__required = False # Whether the artifact is required in the current session - self.__build_result = None # The result of assembling this Element (success, description, detail) + self.__artifacts: ArtifactCache = context.artifactcache # Artifact cache + self.__sourcecache: SourceCache = context.sourcecache # Source cache + self.__assemble_scheduled: bool = False # Element is scheduled to be assembled + self.__assemble_done: bool = False # Element is assembled + self.__pull_pending: bool = False # Whether pull is pending + self.__cached_successfully: Optional[bool] = None # If the Element is known to be successfully cached + self.__splits: Optional[dict[str, Pattern]] = None # Resolved regex objects for computing split domains + self.__whitelist_regex: Optional[Pattern] = ( + None # Resolved regex object to check if file is allowed to overlap + ) + self.__tainted: Optional[bool] = None # Whether the artifact is tainted and should not be shared + self.__required: bool = False # Whether the artifact is required in the current session + self.__build_result: Optional[tuple[bool, str, str | None]] = ( + None # The result of assembling this Element (success, description, detail) + ) # Artifact class for direct artifact composite interaction - self.__artifact = None # type: Optional[Artifact] - self.__dynamic_public = None - self.__sandbox_config = None # type: Optional[SandboxConfig] + self.__artifact: Optional[Artifact] = None + self.__dynamic_public: Optional[MappingNode] = None + self.__sandbox_config: Optional[SandboxConfig] = None + self.__public: Optional[MappingNode] = None # Callbacks - self.__required_callback = None # Callback to Queues - self.__can_query_cache_callback = None # Callback to PullQueue/FetchQueue - self.__buildable_callback = None # Callback to BuildQueue + self.__required_callback: Optional[Callable] = None # Callback to Queues + self.__can_query_cache_callback: Optional[Callable] = None # Callback to PullQueue/FetchQueue + self.__buildable_callback: Optional[Callable] = None # Callback to BuildQueue - self.__resolved_initial_state = False # Whether the initial state of the Element has been resolved + self.__resolved_initial_state: bool = False # Whether the initial state of the Element has been resolved - self.__environment: Dict[str, str] = {} + self.__environment: dict[str, str] = {} self.__variables: Optional[Variables] = None - self.__dynamic_public_guard = Lock() + self.__dynamic_public_guard: Lock = Lock() if artifact_key: self.__initialize_from_artifact_key(artifact_key) @@ -358,7 +387,7 @@ def configure_dependencies(self, dependencies: Iterable[DependencyConfiguration] # assert False, "Code should not be reached" - def configure_sandbox(self, sandbox: "Sandbox") -> None: + def configure_sandbox(self, sandbox: Sandbox) -> None: """Configures the the sandbox for execution Args: @@ -372,7 +401,7 @@ def configure_sandbox(self, sandbox: "Sandbox") -> None: """ raise ImplError("element plugin '{kind}' does not implement configure_sandbox()".format(kind=self.get_kind())) - def stage(self, sandbox: "Sandbox") -> None: + def stage(self, sandbox: Sandbox) -> None: """Stage inputs into the sandbox directories Args: @@ -388,7 +417,7 @@ def stage(self, sandbox: "Sandbox") -> None: """ raise ImplError("element plugin '{kind}' does not implement stage()".format(kind=self.get_kind())) - def assemble(self, sandbox: "Sandbox") -> str: + def assemble(self, sandbox: Sandbox) -> str: """Assemble the output artifact Args: @@ -429,7 +458,7 @@ def generate_script(self) -> str: ############################################################# # Public Methods # ############################################################# - def sources(self) -> Iterator["Source"]: + def sources(self) -> Iterator[Source]: """A generator function to enumerate the element sources Yields: @@ -438,8 +467,8 @@ def sources(self) -> Iterator["Source"]: return self.__sources.sources() def dependencies( - self, selection: Optional[Sequence["Element"]] = None, *, recurse: bool = True - ) -> Iterator["Element"]: + self, selection: Optional[Sequence[Element]] = None, *, recurse: bool = True + ) -> Iterator[Element]: """A generator function which yields the build dependencies of the given element. This generator gives the Element access to all of the dependencies which it is has @@ -476,19 +505,16 @@ def dependencies( selection = [self] for element in selection: - if element is self: - scope = _Scope.BUILD - else: - scope = _Scope.RUN + scope: _Scope = _Scope.BUILD if element is self else _Scope.RUN # Elements in the `selection` will actually be `ElementProxy` objects, but # those calls will be forwarded to their actual internal `_dependencies()` # methods. # for dep in element._dependencies(scope, recurse=recurse, visited=visited): - yield cast("Element", dep.__get_proxy(self)) + yield cast(Element, dep.__get_proxy(self)) - def search(self, name: str) -> Optional["Element"]: + def search(self, name: str) -> Optional[Element]: """Search for a dependency by name Args: @@ -501,7 +527,7 @@ def search(self, name: str) -> Optional["Element"]: if search is self: return self elif search: - return cast("Element", search.__get_proxy(self)) + return cast(Element, search.__get_proxy(self)) return None @@ -533,7 +559,7 @@ def node_subst_vars(self, node: "ScalarNode") -> str: ) return node.as_str() - def node_subst_sequence_vars(self, node: "SequenceNode[ScalarNode]") -> List[str]: + def node_subst_sequence_vars(self, node: SequenceNode[ScalarNode]) -> List[str]: """Substitute any variables in the given sequence **Warning**: The method is deprecated and will get removed in the next version @@ -556,7 +582,7 @@ def node_subst_sequence_vars(self, node: "SequenceNode[ScalarNode]") -> List[str def compute_manifest( self, *, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True - ) -> str: + ) -> Iterable[str]: """Compute and return this element's selective manifest The manifest consists on the list of file paths in the @@ -601,7 +627,7 @@ def get_artifact_name(self, key: Optional[str] = None) -> str: def stage_artifact( self, - sandbox: "Sandbox", + sandbox: Sandbox, *, path: Optional[str] = None, action: OverlapAction = OverlapAction.WARNING, @@ -660,8 +686,8 @@ def stage_artifact( def stage_dependency_artifacts( self, - sandbox: "Sandbox", - selection: Optional[Sequence["Element"]] = None, + sandbox: Sandbox, + selection: Optional[Sequence[Element]] = None, *, path: Optional[str] = None, action: OverlapAction = OverlapAction.WARNING, @@ -700,7 +726,7 @@ def stage_dependency_artifacts( for dep in self.dependencies(selection): dep._stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans, owner=self) - def integrate(self, sandbox: "Sandbox") -> None: + def integrate(self, sandbox: Sandbox) -> None: """Integrate currently staged filesystem against this artifact. Args: @@ -720,7 +746,7 @@ def integrate(self, sandbox: "Sandbox") -> None: for command in bstdata.get_str_list("integration-commands", []): sandbox.run(["sh", "-e", "-c", command], env=environment, cwd="/", label=command) - def stage_sources(self, sandbox: "Sandbox", directory: str) -> None: + def stage_sources(self, sandbox: Sandbox, directory: str) -> None: """Stage this element's sources to a directory in the sandbox Args: @@ -729,7 +755,7 @@ def stage_sources(self, sandbox: "Sandbox", directory: str) -> None: """ self._stage_sources_in_sandbox(sandbox, directory) - def get_public_data(self, domain: str) -> "MappingNode[Node]": + def get_public_data(self, domain: str) -> MappingNode[Node] | None: """Fetch public data on this element Args: @@ -747,15 +773,14 @@ def get_public_data(self, domain: str) -> "MappingNode[Node]": if self.__dynamic_public is None: self.__load_public_data() - # Disable type-checking since we can't easily tell mypy that - # `self.__dynamic_public` can't be None here. - data = self.__dynamic_public.get_mapping(domain, default=None) # type: ignore + assert self.__dynamic_public is not None, "Element should have dynamic public data at this stage" + data = self.__dynamic_public.get_mapping(domain, default=None) if data is not None: data = data.clone() return data - def set_public_data(self, domain: str, data: "MappingNode[Node]") -> None: + def set_public_data(self, domain: str, data: MappingNode[Node]) -> None: """Set public data on this element Args: @@ -770,13 +795,14 @@ def set_public_data(self, domain: str, data: "MappingNode[Node]") -> None: with self.__dynamic_public_guard: if self.__dynamic_public is None: self.__load_public_data() + assert self.__dynamic_public, "We have loaded public data by this point, so this should never happen" if data is not None: data = data.clone() - self.__dynamic_public[domain] = data # type: ignore + self.__dynamic_public[domain] = data - def get_environment(self) -> Dict[str, str]: + def get_environment(self) -> dict[str, str]: """Fetch the environment suitable for running in the sandbox Returns: @@ -795,10 +821,10 @@ def get_variable(self, varname: str) -> Optional[str]: The resolved value for *varname*, or None if no variable was declared with the given name. """ - assert self.__variables + assert self.__variables, "{}: has no Variables object".format(self.name) return self.__variables.get(varname) - def run_cleanup_commands(self, sandbox: "Sandbox") -> None: + def run_cleanup_commands(self, sandbox: Sandbox) -> None: """Run commands to cleanup the build directory. Args: @@ -821,7 +847,7 @@ def run_cleanup_commands(self, sandbox: "Sandbox") -> None: build_root = self.get_variable("build-root") install_root = self.get_variable("install-root") - assert build_root + assert build_root, "There should be a build root at this stage" if install_root and (build_root.startswith(install_root) or install_root.startswith(build_root)): # Preserve the build directory if cleaning would affect the install directory return @@ -829,7 +855,7 @@ def run_cleanup_commands(self, sandbox: "Sandbox") -> None: sandbox._clean_directory(build_root) @contextmanager - def subsandbox(self, sandbox: "Sandbox") -> Iterator["Sandbox"]: + def subsandbox(self, sandbox: Sandbox) -> Iterator[Sandbox]: """A context manager for a subsandbox. Args: @@ -864,13 +890,13 @@ def subsandbox(self, sandbox: "Sandbox") -> Iterator["Sandbox"]: # Yields: # (Element): The dependencies in `scope`, in deterministic staging order # - def _dependencies(self, scope: _Scope, *, recurse=True, visited=None): + def _dependencies(self, scope: _Scope, *, recurse=True, visited=None) -> Generator[Element]: # The format of visited is (BitMap(), BitMap()), with the first BitMap # containing element that have been visited for the `_Scope.BUILD` case # and the second one relating to the `_Scope.RUN` case. if not recurse: - result: Set["Element"] = set() + result: Set[Element] = set() if scope in (_Scope.BUILD, _Scope.ALL): for dep in self.__build_dependencies: if dep not in result: @@ -968,14 +994,14 @@ def _search(self, scope, name): # def _stage_artifact( self, - sandbox: "Sandbox", + sandbox: Sandbox, *, path: Optional[str] = None, action: OverlapAction = OverlapAction.WARNING, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True, - owner: Optional["Element"] = None, + owner: Optional[Element] = None, ) -> FileListResult: owner = owner or self @@ -993,9 +1019,7 @@ def _stage_artifact( self.__assert_cached() self.status("Staging {}/{}".format(self.name, self._get_display_key().brief)) - # Disable type checking since we can't easily tell mypy that - # `self.__artifact` can't be None at this stage. - files_vdir = self.__artifact.get_files() # type: ignore + files_vdir = self._get_artifact().get_files() # Import files into the staging area # @@ -1030,7 +1054,16 @@ def _stage_artifact( # yet produced artifacts, or if forbidden overlaps # occur. # - def _stage_dependency_artifacts(self, sandbox, scope, *, path=None, include=None, exclude=None, orphans=True): + def _stage_dependency_artifacts( + self, + sandbox: Sandbox, + scope: _Scope, + *, + path: Optional[str] = None, + include: Optional[list[str]] = None, + exclude: Optional[list[str]] = None, + orphans: bool = True, + ): with self._overlap_collectors[sandbox].session(OverlapAction.WARNING, path): for dep in self._dependencies(scope): dep._stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans, owner=self) @@ -1051,25 +1084,26 @@ def _stage_dependency_artifacts(self, sandbox, scope, *, path=None, include=None # (Element): A newly created Element instance # @classmethod - def _new_from_load_element(cls, load_element, task=None): + def _new_from_load_element(cls, load_element: LoadElement, task: Optional[Task] = None) -> Element: if not load_element.first_pass: load_element.project.ensure_fully_loaded() - with suppress(KeyError): - return cls.__instantiated_elements[load_element] + if (instantiated_element := cls.__instantiated_elements.get(load_element)) is not None: + return instantiated_element - element = load_element.project.create_element(load_element) + element: Element = load_element.project.create_element(load_element) cls.__instantiated_elements[load_element] = element # If the element implements configure_dependencies(), we will collect # the dependency configurations for it, otherwise we will consider # it an error to specify `config` on dependencies. # - if element.configure_dependencies.__func__ is not Element.configure_dependencies: - custom_configurations = [] - else: - custom_configurations = None + custom_configurations: list[DependencyConfiguration] | None = ( + [] + if element.configure_dependencies.__func__ is not Element.configure_dependencies # type: ignore[attr-defined] + else None + ) # Load the sources from the LoadElement element.__load_sources(load_element) @@ -1078,7 +1112,7 @@ def _new_from_load_element(cls, load_element, task=None): for dep in load_element.dependencies: dependency = Element._new_from_load_element(dep.element, task) - if dep.dep_type & DependencyType.BUILD: + if dep.dep_type & DependencyType.BUILD: # type: ignore element.__build_dependencies.append(dependency) dependency.__reverse_build_deps.add(element) @@ -1096,6 +1130,7 @@ def _new_from_load_element(cls, load_element, task=None): # Ensure variables are substituted first # + assert element.__variables, "Variables should not be none at this stage in element" for config in dep.config_nodes: element.__variables.expand(config) @@ -1115,7 +1150,7 @@ def _new_from_load_element(cls, load_element, task=None): LoadErrorReason.INVALID_DEPENDENCY_CONFIG, ) - if dep.dep_type & DependencyType.RUNTIME: + if dep.dep_type & DependencyType.RUNTIME: # type: ignore element.__runtime_dependencies.append(dependency) dependency.__reverse_runtime_deps.add(element) @@ -1163,7 +1198,7 @@ def _clear_meta_elements_cache(cls): # # This is used to produce a warning @classmethod - def _get_redundant_source_refs(cls): + def _get_redundant_source_refs(cls) -> List[Tuple[Source, SourceRef]]: return cls.__redundant_source_refs # _reset_load_state() @@ -1181,15 +1216,15 @@ def _reset_load_state(cls): # (bool): Whether this element is already present in # the artifact cache # - def _cached(self): - return self.__artifact.cached() + def _cached(self) -> bool: + return self._get_artifact().cached() # _cached_remotely(): # # Returns: # (bool): Whether this element is present in a remote cache # - def _cached_remotely(self): + def _cached_remotely(self) -> bool: if self.__cached_remotely is None: self.__cached_remotely = self.__artifacts.check_remotes_for_element(self) return self.__cached_remotely @@ -1201,10 +1236,12 @@ def _cached_remotely(self): # (str): Short description of the result # (str): Detailed description of the result # - def _get_build_result(self): + def _get_build_result(self) -> tuple[bool, str, str | None]: if self.__build_result is None: self.__load_build_result() + assert self.__build_result, "Build result should not be none after __load_build_result" + return self.__build_result # __set_build_result(): @@ -1216,7 +1253,7 @@ def _get_build_result(self): # description (str): Short description of the result # detail (str): Detailed description of the result # - def __set_build_result(self, success, description, detail=None): + def __set_build_result(self, success: bool, description: str, detail: str | None = None): self.__build_result = (success, description, detail) # _cached_success(): @@ -1225,7 +1262,7 @@ def __set_build_result(self, success, description, detail=None): # (bool): Whether this element is already present in # the artifact cache and the element assembled successfully # - def _cached_success(self): + def _cached_success(self) -> bool: # FIXME: _cache() and _cached_success() should be converted to # push based functions where we only update __cached_successfully # once we know this has changed. This will allow us to cheaply check @@ -1249,7 +1286,7 @@ def _cached_success(self): # (bool): Whether this element is already present in # the artifact cache and the element did not assemble successfully # - def _cached_failure(self): + def _cached_failure(self) -> bool: if not self._cached(): return False @@ -1261,7 +1298,7 @@ def _cached_failure(self): # Returns: # (bool): Whether this element can currently be built # - def _buildable(self): + def _buildable(self) -> bool: # This check must be before `_fetch_needed()` as source cache status # is not always available for non-build pipelines. if not self.__assemble_scheduled: @@ -1284,9 +1321,11 @@ def _buildable(self): # # None is returned if information for the cache key is missing. # - def _get_cache_key(self, strength=_KeyStrength.STRONG): + def _get_cache_key(self, strength=_KeyStrength.STRONG) -> str | None: if strength == _KeyStrength.STRONG: return self.__cache_key + elif strength == _KeyStrength.STRICT: + return self.__strict_cache_key else: return self.__weak_cache_key @@ -1394,6 +1433,7 @@ def _get_display_key(self): # that would be used in strict build mode strict = True + assert context.log_key_length, "log key length should be present" length = min(len(cache_key), context.log_key_length) return _DisplayKey(cache_key, cache_key[0:length], strict) @@ -1426,7 +1466,9 @@ def _track(self): # is used to stage things by the `bst artifact checkout` codepath # @contextmanager - def _prepare_sandbox(self, scope, shell=False, integrate=True, usebuildtree=False): + def _prepare_sandbox( + self, scope: _Scope, shell: bool = False, integrate: bool = True, usebuildtree: bool = False + ) -> Generator[Sandbox]: # Assert first that we have a sandbox configuration if not self.__sandbox_config: @@ -1441,11 +1483,12 @@ def _prepare_sandbox(self, scope, shell=False, integrate=True, usebuildtree=Fals with self.__sandbox(config=self.__sandbox_config, allow_remote=False) as sandbox: if usebuildtree: + artifact = self._get_artifact() # Configure the sandbox from artifact metadata - self.__artifact.configure_sandbox(sandbox) + artifact.configure_sandbox(sandbox) # Use the cached buildroot directly - buildrootvdir = self.__artifact.get_buildroot() + buildrootvdir = artifact.get_buildroot() sandbox_vroot = sandbox.get_virtual_directory() sandbox_vroot._import_files_internal(buildrootvdir, collect_result=False) elif shell and scope == _Scope.BUILD: @@ -1480,7 +1523,7 @@ def _prepare_sandbox(self, scope, shell=False, integrate=True, usebuildtree=Fals # sandbox (:class:`.Sandbox`): The build sandbox # directory (str): An absolute path to stage the sources at # - def _stage_sources_in_sandbox(self, sandbox, directory): + def _stage_sources_in_sandbox(self, sandbox: Sandbox, directory: str): # Stage all sources that need to be copied sandbox_vroot = sandbox.get_virtual_directory() @@ -1494,7 +1537,7 @@ def _stage_sources_in_sandbox(self, sandbox, directory): # Args: # vdirectory (Union[str, Directory]): A virtual directory object or local path to stage sources to. # - def _stage_sources_at(self, vdirectory): + def _stage_sources_at(self, vdirectory: Union[str, Directory]): # It's advantageous to have this temporary directory on # the same file system as the rest of our cache. @@ -1532,7 +1575,7 @@ def _stage_sources_at(self, vdirectory): # Args: # scope (_Scope): The scope of dependencies to mark as required # - def _set_required(self, scope=_Scope.RUN): + def _set_required(self, scope: _Scope = _Scope.RUN): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" if self.__required: @@ -1561,7 +1604,7 @@ def _set_required(self, scope=_Scope.RUN): # # Returns whether this element has been marked as required. # - def _is_required(self): + def _is_required(self) -> bool: return self.__required # __should_schedule() @@ -1569,7 +1612,7 @@ def _is_required(self): # Returns: # bool - Whether the element can be scheduled for a build. # - def __should_schedule(self): + def __should_schedule(self) -> bool: # We're processing if we're already scheduled, we've # finished assembling or if we're waiting to pull. processing = self.__assemble_scheduled or self.__assemble_done or self._pull_pending() @@ -1583,7 +1626,7 @@ def __should_schedule(self): self._is_required() and # We have figured out the state of our artifact - self.__artifact + self.__artifact is not None and # And we're not cached yet not self._cached_success() @@ -1626,19 +1669,20 @@ def __schedule_assembly_when_necessary(self): # Args: # successful (bool): Whether the build was successful # - def _assemble_done(self, successful): - assert self.__assemble_scheduled + def _assemble_done(self, successful: bool): + assert self.__assemble_scheduled, "Assembly should be scheduled before calling this method on element" assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" + artifact = self._get_artifact() self.__assemble_done = True if successful: # Directly set known cached status as optimization to avoid # querying buildbox-casd and the filesystem. - self.__artifact.set_cached() + artifact.set_cached() self.__cached_successfully = True else: - self.__artifact.query_cache() + artifact.query_cache() # When we're building in non-strict mode, we may have # assembled everything to this point without a strong cache @@ -1646,8 +1690,8 @@ def _assemble_done(self, successful): # can be set, so we do so. self.__update_cache_key_non_strict() self._update_ready_for_runtime_and_cached() - - if self._get_workspace() and self._cached(): + workspace = self._get_workspace() + if workspace and self._cached(): # Note that this block can only happen in the # main process, since `self._cached_success()` cannot # be true when assembly is successful in the task. @@ -1656,7 +1700,6 @@ def _assemble_done(self, successful): # save the workspaces configuration # key = self._get_cache_key() - workspace = self._get_workspace() workspace.last_build = key self._get_context().get_workspaces().save_config() @@ -1675,11 +1718,11 @@ def _assemble(self): # to allow for retrying the job if self._cached_failure() and not self.__assemble_done: with self._output_file() as output_file: - for log_path in self.__artifact.get_logs(): + for log_path in self._get_logs(): with open(log_path, encoding="utf-8") as log_file: output_file.write(log_file.read()) - _, description, detail = self._get_build_result() + [_, description, detail] = self._get_build_result() e = CachedFailure(description, detail=detail) # Shelling into a sandbox is useful to debug this error e.sandbox = True @@ -1692,6 +1735,7 @@ def _assemble(self): with self._output_file() as output_file: # Explicitly clean it up, keep the build dir around if exceptions are raised + assert context.builddir, "A build dir should be present" os.makedirs(context.builddir, exist_ok=True) with self.__sandbox(output_file, output_file, self.__sandbox_config) as sandbox: @@ -1707,6 +1751,7 @@ def _assemble(self): # By default, the dynamic public data is the same as the static public data. # The plugin's assemble() method may modify this, though. with self.__dynamic_public_guard: + assert self.__public, "Public data should be present in Element" self.__dynamic_public = self.__public.clone() # Call the abstract plugin methods @@ -1736,9 +1781,10 @@ def _assemble(self): else: self._cache_artifact(sandbox, collect) - def _cache_artifact(self, sandbox, collect): + def _cache_artifact(self, sandbox: Sandbox, collect: str | None): context = self._get_context() + assert self.__build_result, "Build result should be present at this stage" buildresult = self.__build_result with self.__dynamic_public_guard: publicdata = self.__dynamic_public @@ -1761,7 +1807,9 @@ def _cache_artifact(self, sandbox, collect): cache_buildtrees == _CacheBuildTrees.AUTO and (not build_success or self._get_workspace()) ): try: - sandbox_build_dir = sandbox_vroot.open_directory(self.get_variable("build-root").lstrip(os.sep)) + build_root = self.get_variable("build-root") + assert build_root, "Build root should be present at this stage" + sandbox_build_dir = sandbox_vroot.open_directory(build_root.lstrip(os.sep)) sandbox._fetch_missing_blobs(sandbox_build_dir) except DirectoryError: # Directory could not be found. Pre-virtual @@ -1779,12 +1827,12 @@ def _cache_artifact(self, sandbox, collect): except DirectoryError: pass - # We should always have cache keys already set when caching an artifact - assert self.__cache_key is not None - assert self.__artifact._cache_key is not None + assert self.__cache_key is not None, "We should always have cache keys already set when caching an artifact" + artifact = self._get_artifact() + assert artifact._cache_key is not None, "Cache key should also be present in the artfact" with self.timed_activity("Caching artifact"): - self.__artifact.cache( + artifact.cache( buildrootvdir=buildrootvdir, sandbox_build_dir=sandbox_build_dir, collectvdir=collectvdir, @@ -1810,7 +1858,7 @@ def _cache_artifact(self, sandbox, collect): # Args: # fetched_original (bool): Whether the original sources had been asked (and fetched) or not # - def _fetch_done(self, fetched_original): + def _fetch_done(self, fetched_original: bool) -> None: assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" self.__sources.fetch_done(fetched_original) @@ -1824,7 +1872,7 @@ def _fetch_done(self, fetched_original): # Returns: # (bool): Whether a pull operation is pending # - def _pull_pending(self): + def _pull_pending(self) -> bool: return self.__pull_pending # _load_artifact_done() @@ -1840,11 +1888,9 @@ def _pull_pending(self): def _load_artifact_done(self): assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" - assert self.__artifact - context = self._get_context() - if not context.get_strict() and self.__artifact.cached(): + if not context.get_strict() and self._get_artifact().cached(): # In non-strict mode, strong cache key becomes available when # the artifact is cached self.__update_cache_key_non_strict() @@ -1867,7 +1913,7 @@ def _load_artifact_done(self): # # Returns: True if the artifact has been downloaded, False otherwise # - def _load_artifact(self, *, pull, strict=None): + def _load_artifact(self, *, pull: bool, strict: Optional[bool] = None) -> bool: context = self._get_context() if strict is None: @@ -1976,15 +2022,15 @@ def _load_artifact(self, *, pull, strict=None): self.__artifact = artifact return pulled - def _query_source_cache(self): + def _query_source_cache(self) -> None: self.__sources.query_cache() - def _skip_source_push(self): + def _skip_source_push(self) -> bool: if not self.sources() or self._get_workspace(): return True return not (self.__sourcecache.has_push_remotes(plugin=self) and self._cached_sources()) - def _source_push(self): + def _source_push(self) -> None: return self.__sources.push() # _skip_push(): @@ -1997,7 +2043,7 @@ def _source_push(self): # Returns: # (bool): True if this element does not need a push job to be created # - def _skip_push(self, *, skip_uncached): + def _skip_push(self, *, skip_uncached: bool) -> bool: if not self.__artifacts.has_push_remotes(plugin=self): # No push remotes for this element's project return True @@ -2022,7 +2068,7 @@ def _skip_push(self, *, skip_uncached): # (bool): True if the remote was updated, False if it already existed # and no updated was required # - def _push(self): + def _push(self) -> bool: if not self._cached(): raise ElementError("Push failed: {} is not cached".format(self.name)) @@ -2039,7 +2085,7 @@ def _push(self): return False # Push all keys used for local commit via the Artifact member - pushed = self.__artifacts.push(self, self.__artifact) + pushed = self.__artifacts.push(self, self._get_artifact()) if not pushed: return False @@ -2062,14 +2108,14 @@ def _push(self): # Returns: Exit code def _shell( self, - scope: _Scope | None = None, + scope: _Scope = _Scope.NONE, *, mounts: List[_HostMount] | None = None, isolate: bool = False, prompt: str | None = None, command: List[str] | None = None, usebuildtree: bool = False, - ): + ) -> Optional[int]: with self._prepare_sandbox(scope, shell=True, usebuildtree=usebuildtree) as sandbox: environment = sandbox._get_configured_environment() or self.get_environment() @@ -2125,7 +2171,7 @@ def _shell( # This requires that a workspace already be created in # the workspaces metadata first. # - def _open_workspace(self): + def _open_workspace(self) -> None: assert utils._is_in_main_thread(), "This writes to a global file and therefore must be run in the main thread" context = self._get_context() @@ -2140,6 +2186,7 @@ def _open_workspace(self): # files in the target directory actually works without any # additional support from Source implementations. # + assert context.builddir, "build dir is required for this" os.makedirs(context.builddir, exist_ok=True) with utils._tempdir(dir=context.builddir, prefix="workspace-{}".format(self.normal_name)) as temp: self.__sources.init_workspace(temp) @@ -2152,7 +2199,7 @@ def _open_workspace(self): # Returns: # (Workspace|None): A workspace associated with this element # - def _get_workspace(self): + def _get_workspace(self) -> Workspace | None: workspaces = self._get_context().get_workspaces() return workspaces.get_workspace(self._get_full_name()) @@ -2193,7 +2240,7 @@ def _write_script(self, directory): # element B is a filter element that depends on element A. The source # element of B is A, since B depends on A, and A has sources. # - def _get_source_element(self): + def _get_source_element(self) -> Element: return self # _cached_buildtree() @@ -2208,11 +2255,11 @@ def _get_source_element(self): # Note this only confirms if a buildtree is present, # not its contents. # - def _cached_buildtree(self): + def _cached_buildtree(self) -> bool: if not self._cached(): return False - return self.__artifact.cached_buildtree() + return self._get_artifact().cached_buildtree() # _buildtree_exists() # @@ -2223,11 +2270,11 @@ def _cached_buildtree(self): # (bool): True if artifact was created with buildtree, False if # element not cached or not created with a buildtree. # - def _buildtree_exists(self): + def _buildtree_exists(self) -> bool: if not self._cached(): return False - return self.__artifact.buildtree_exists() + return self._get_artifact().buildtree_exists() # _cached_buildroot() # @@ -2241,11 +2288,11 @@ def _buildtree_exists(self): # Note this only confirms if a buildroot is present, # not its contents. # - def _cached_buildroot(self): + def _cached_buildroot(self) -> bool: if not self._cached(): return False - return self.__artifact.cached_buildroot() + return self._get_artifact().cached_buildroot() # _buildroot_exists() # @@ -2256,11 +2303,11 @@ def _cached_buildroot(self): # (bool): True if artifact was created with buildroot, False if # element not cached or not created with a buildroot. # - def _buildroot_exists(self): + def _buildroot_exists(self) -> bool: if not self._cached(): return False - return self.__artifact.buildroot_exists() + return self._get_artifact().buildroot_exists() # _cached_logs() # @@ -2270,8 +2317,8 @@ def _buildroot_exists(self): # (bool): True if artifact is cached with logs, False if # element not cached or missing logs. # - def _cached_logs(self): - return self.__artifact.cached_logs() + def _cached_logs(self) -> bool: + return self._get_artifact().cached_logs() # _fetch() # @@ -2280,7 +2327,7 @@ def _cached_logs(self): # Raises: # SourceError: If one of the element sources has an error # - def _fetch(self, fetch_original=False): + def _fetch(self, fetch_original: bool = False) -> None: if fetch_original: self.__sources.fetch_sources(fetch_original=True) @@ -2299,9 +2346,9 @@ def _fetch(self, fetch_original=False): # # Calculates the cache key # + # # Args: - # dependencies (List[List[str]]): list of dependencies with project name, - # element name and optional cache key + # key_strength (_KeyStrength): The stength of the key to calculate # weak_cache_key (Optional[str]): the weak cache key, required for calculating the # strict and strong cache keys # @@ -2310,10 +2357,33 @@ def _fetch(self, fetch_original=False): # # None is returned if information for the cache key is missing. # - def _calculate_cache_key(self, dependencies, weak_cache_key=None): - # No cache keys for dependencies which have no cache keys - if any(not all(dep) for dep in dependencies): - return None + def _calculate_cache_key( + self, key_strength: _KeyStrength = _KeyStrength.STRONG, weak_cache_key: Optional[str] = None, scope: _Scope = _Scope.BUILD + ) -> str | None: + assert self.__sandbox_config, "Element should have a sandbox config to calculate cache key" + assert self.__public, "Element should have public data to calculate cache key" + + # Strict or Strong Dependency must have: Project Name, element name and cache key + StrictOrStrongDep: TypeAlias = tuple[str, str, str] + # Weak Dependency must have: Project Name, element name + WeakDep: TypeAlias = tuple[str, str] + + dependencies: list[StrictOrStrongDep | WeakDep] = [] + strict_or_strong: bool = key_strength in [_KeyStrength.STRONG, _KeyStrength.STRICT] or self.BST_STRICT_REBUILD + for dep_element in self._dependencies(scope): + # We need a key for the dependency if we are calculating a strong or strict key + # or we are calculating a weak key and it is a strict rebuild dependency. + if strict_or_strong or dep_element in self.__strict_dependencies: + dep_key = dep_element._get_cache_key(key_strength) + if dep_key is None: + # Abort calculating a key for this element as a dependency doesn't have a required key of correct strength + return None + dependencies.append((dep_element.project_name, dep_element.name, dep_key)) + else: + # This case should only be triggered where key_strength is _KeyStrength.WEAK + # and it is not a strict build dependency + # so we don't need a key for the dependency + dependencies.append((dep_element.project_name, dep_element.name)) # Generate dict that is used as base for all cache keys if self.__cache_key_dict is None: @@ -2352,7 +2422,7 @@ def _calculate_cache_key(self, dependencies, weak_cache_key=None): # Returns: # (bool): True if the element sources are in CAS # - def _cached_sources(self): + def _cached_sources(self) -> bool: return self.__sources.cached() # _has_all_sources_resolved() @@ -2362,7 +2432,7 @@ def _cached_sources(self): # Returns: # (bool): True if all element sources are resolved # - def _has_all_sources_resolved(self): + def _has_all_sources_resolved(self) -> bool: return self.__sources.is_resolved() # _fetch_needed(): @@ -2372,7 +2442,7 @@ def _has_all_sources_resolved(self): # Returns: # (bool): True if one or more element sources need to be fetched # - def _fetch_needed(self): + def _fetch_needed(self) -> bool: return not self.__sources.cached() and not self.__sources.cached_original() # _should_fetch(): @@ -2385,7 +2455,7 @@ def _fetch_needed(self): # Returns: # (bool): True if a fetch job is required # - def _should_fetch(self, fetch_original=False): + def _should_fetch(self, fetch_original: bool = False) -> bool: if fetch_original: return not self.__sources.cached_original() return not self.__sources.cached() @@ -2403,7 +2473,7 @@ def _should_fetch(self, fetch_original=False): # Args: # callback (callable) - The callback function # - def _set_required_callback(self, callback): + def _set_required_callback(self, callback: Callable) -> None: self.__required_callback = callback # _set_can_query_cache_callback() @@ -2421,7 +2491,7 @@ def _set_required_callback(self, callback): # Args: # callback (callable) - The callback function # - def _set_can_query_cache_callback(self, callback): + def _set_can_query_cache_callback(self, callback: Callable) -> None: self.__can_query_cache_callback = callback # _set_buildable_callback() @@ -2437,7 +2507,7 @@ def _set_can_query_cache_callback(self, callback): # Args: # callback (callable) - The callback function # - def _set_buildable_callback(self, callback): + def _set_buildable_callback(self, callback: Callable) -> None: self.__buildable_callback = callback # _set_depth() @@ -2447,7 +2517,7 @@ def _set_buildable_callback(self, callback): # The depth represents the position of the Element within the current # session's dependency graph. A depth of zero represents the bottommost element. # - def _set_depth(self, depth): + def _set_depth(self, depth: int) -> None: self._depth = depth # _update_ready_for_runtime_and_cached() @@ -2465,7 +2535,7 @@ def _set_depth(self, depth): # runtime dependencies and the reverse build dependencies of the element, decrementing # the appropriate counters. # - def _update_ready_for_runtime_and_cached(self): + def _update_ready_for_runtime_and_cached(self) -> None: assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" if not self.__ready_for_runtime_and_cached: @@ -2474,6 +2544,7 @@ def _update_ready_for_runtime_and_cached(self): # Notify reverse dependencies for rdep in self.__reverse_runtime_deps: + assert rdep.__runtime_deps_uncached, "We should have a number here" rdep.__runtime_deps_uncached -= 1 assert not rdep.__runtime_deps_uncached < 0 @@ -2482,6 +2553,7 @@ def _update_ready_for_runtime_and_cached(self): rdep._update_ready_for_runtime_and_cached() for rdep in self.__reverse_build_deps: + assert rdep.__build_deps_uncached, "We should also have a number here" rdep.__build_deps_uncached -= 1 assert not rdep.__build_deps_uncached < 0 @@ -2499,7 +2571,7 @@ def _update_ready_for_runtime_and_cached(self): # Returns: # (Artifact): The Artifact object of the Element # - def _get_artifact(self): + def _get_artifact(self) -> Artifact: assert self.__artifact, "{}: has no Artifact object".format(self.name) return self.__artifact @@ -2511,7 +2583,7 @@ def _get_artifact(self): # from a loaded artifact, or after pulling the artifact from # a remote. # - def _mimic_artifact(self): + def _mimic_artifact(self) -> None: artifact = self._get_artifact() # Load bits which have been stored on the artifact @@ -2532,7 +2604,7 @@ def _mimic_artifact(self): # Args: # (Element): The Element to add as a build dependency # - def _add_build_dependency(self, dependency): + def _add_build_dependency(self, dependency: Element) -> None: self.__build_dependencies.append(dependency) # _file_is_whitelisted() @@ -2548,7 +2620,9 @@ def _add_build_dependency(self, dependency): # Returns: # (bool): True of the specified `path` is whitelisted # - def _file_is_whitelisted(self, path): + def _file_is_whitelisted(self, path: str) -> bool: + assert self.__variables, "{}: has no Variables object".format(self.name) + # Considered storing the whitelist regex for re-use, but public data # can be altered mid-build. # Public data is not guaranteed to stay the same for the duration of @@ -2556,11 +2630,12 @@ def _file_is_whitelisted(self, path): # If this ever changes, things will go wrong unexpectedly. if not self.__whitelist_regex: bstdata = self.get_public_data("bst") - whitelist = bstdata.get_sequence("overlap-whitelist", default=[]) + assert bstdata, "We should have a bstdata section to get the whitelist" + whitelist: SequenceNode = bstdata.get_sequence("overlap-whitelist", []) whitelist_expressions = [utils._glob2re(self.__variables.subst(node)) for node in whitelist] expression = "^(?:" + "|".join(whitelist_expressions) + ")$" self.__whitelist_regex = re.compile(expression, re.MULTILINE | re.DOTALL) - return self.__whitelist_regex.match(os.path.join(os.sep, path)) + return self.__whitelist_regex.match(os.path.join(os.sep, path)) is not None # _get_logs() # @@ -2570,7 +2645,7 @@ def _file_is_whitelisted(self, path): # A list of log file paths # def _get_logs(self) -> List[str]: - return cast(Artifact, self.__artifact).get_logs() + return self._get_artifact().get_logs() ############################################################# # Private Local Methods # @@ -2591,9 +2666,9 @@ def _get_logs(self) -> List[str]: # Returns: # (ElementProxy): An ElementProxy to self, for owner. # - def __get_proxy(self, owner: "Element") -> ElementProxy: - with suppress(KeyError): - return self.__proxies[owner] + def __get_proxy(self, owner: Element) -> ElementProxy: + if (proxy := self.__proxies.get(owner)) is not None: + return proxy proxy = ElementProxy(owner, self) self.__proxies[owner] = proxy @@ -2603,7 +2678,7 @@ def __get_proxy(self, owner: "Element") -> ElementProxy: # # Load the Source objects from the LoadElement # - def __load_sources(self, load_element): + def __load_sources(self, load_element: LoadElement) -> None: project = self._get_project() workspace = self._get_workspace() meta_sources = [] @@ -2626,7 +2701,7 @@ def __load_sources(self, load_element): ) meta_sources.append(meta) else: - sources = load_element.node.get_sequence(Symbol.SOURCES, default=[]) + sources: SequenceNode = load_element.node.get_sequence(Symbol.SOURCES, []) for index, source in enumerate(sources): kind = source.get_scalar(Symbol.KIND) @@ -2650,6 +2725,9 @@ def __load_sources(self, load_element): def source_provenance_attribute_check(provenance_node=provenance_node): try: + assert ( + project.source_provenance_attributes + ), "Project should have source provenance atrributes before processing elements" provenance_node.validate_keys(project.source_provenance_attributes.keys()) except LoadError as E: raise LoadError( @@ -2700,11 +2778,20 @@ def source_provenance_attribute_check(provenance_node=provenance_node): # Returns: # (list [str]): A list of refs of all dependencies in staging order. # - def __get_dependency_artifact_names(self): - return [ - os.path.join(dep.project_name, _get_normal_name(dep.name), dep._get_cache_key()) - for dep in self._dependencies(_Scope.BUILD) - ] + def __get_dependency_artifact_names(self) -> list[str]: + refs = [] + for dep in self._dependencies(_Scope.BUILD): + key = dep._get_cache_key() + if key is None: + key = "" + refs.append( + os.path.join( + dep.project_name, + _get_normal_name(dep.name), + key, + ) + ) + return refs # __get_last_build_artifact() # @@ -2714,7 +2801,7 @@ def __get_dependency_artifact_names(self): # Returns: # (Artifact): The Artifact of the previous build or None # - def __get_last_build_artifact(self): + def __get_last_build_artifact(self) -> Optional[Artifact]: workspace = self._get_workspace() if not workspace: # Currently incremental builds are only supported for workspaces @@ -2748,7 +2835,7 @@ def __get_last_build_artifact(self): # # Internal method for calling public abstract configure_sandbox() method. # - def __configure_sandbox(self, sandbox): + def __configure_sandbox(self, sandbox: Sandbox) -> None: self.configure_sandbox(sandbox) @@ -2756,7 +2843,7 @@ def __configure_sandbox(self, sandbox): # # Internal method for calling public abstract stage() method. # - def __stage(self, sandbox): + def __stage(self, sandbox: Sandbox) -> None: # Enable the overlap collector during the staging process with self.__collect_overlaps(sandbox): @@ -2767,7 +2854,7 @@ def __stage(self, sandbox): # A internal wrapper for calling the abstract preflight() method on # the element and its sources. # - def __preflight(self): + def __preflight(self) -> None: if self.BST_FORBID_RDEPENDS and self.BST_FORBID_BDEPENDS: if any(self._dependencies(_Scope.RUN, recurse=False)) or any( @@ -2813,7 +2900,7 @@ def __preflight(self): # is the part of the cache key which is element instance # specific and automatically generated by BuildStream core. # - def __get_base_key(self): + def __get_base_key(self) -> dict[str, str | None]: return { "build-root": self.get_variable("build-root"), } @@ -2822,7 +2909,7 @@ def __get_base_key(self): # # Raises an error if the artifact is not cached. # - def __assert_cached(self): + def __assert_cached(self) -> None: assert self._cached(), "{}: Missing artifact {}".format(self, self._get_display_key().brief) # __get_tainted(): @@ -2839,18 +2926,18 @@ def __assert_cached(self): # This method should only be called after the element's # artifact is present in the local artifact cache. # - def __get_tainted(self, recalculate=False): + def __get_tainted(self, recalculate: bool = False) -> bool: + artifact = self._get_artifact() if recalculate or self.__tainted is None: # Whether this artifact has a workspace - workspaced = self.__artifact.get_metadata_workspaced() + workspaced = artifact.get_metadata_workspaced() # Whether this artifact's dependencies have workspaces - workspaced_dependencies = self.__artifact.get_metadata_workspaced_dependencies() + workspaced_dependencies = artifact.get_metadata_workspaced_dependencies() # Other conditions should be or-ed - self.__tainted = workspaced or workspaced_dependencies - + self.__tainted = workspaced or bool(workspaced_dependencies) return self.__tainted # __collect_overlaps(): @@ -2862,7 +2949,7 @@ def __get_tainted(self, recalculate=False): # this context manager. # @contextmanager - def __collect_overlaps(self, sandbox): + def __collect_overlaps(self, sandbox: Sandbox) -> Generator: self._overlap_collectors[sandbox] = OverlapCollector(self) try: yield @@ -2886,7 +2973,13 @@ def __collect_overlaps(self, sandbox): # (Sandbox): A usable sandbox # @contextmanager - def __sandbox(self, stdout=None, stderr=None, config=None, allow_remote=True): + def __sandbox( + self, + stdout: Optional[TextIO] = None, + stderr: Optional[TextIO] = None, + config: Optional[SandboxConfig] = None, + allow_remote: bool = True, + ) -> Generator[Sandbox]: context = self._get_context() project = self._get_project() platform = context.platform @@ -2927,7 +3020,7 @@ def __sandbox(self, stdout=None, stderr=None, config=None, allow_remote=True): # # Normal element initialization procedure. # - def __initialize_from_yaml(self, load_element: "LoadElement", plugin_conf: Optional[str]): + def __initialize_from_yaml(self, load_element: LoadElement, plugin_conf: Optional[str]) -> None: context = self._get_context() project = self._get_project() @@ -2971,7 +3064,7 @@ def __initialize_from_yaml(self, load_element: "LoadElement", plugin_conf: Optio # # Initialize the element state from an artifact key # - def __initialize_from_artifact_key(self, key: str): + def __initialize_from_artifact_key(self, key: str) -> None: # At this point we only know the key which was specified on the command line, # so we will pretend all keys are equal. # @@ -2996,7 +3089,7 @@ def __initialize_from_artifact_key(self, key: str): self._load_artifact_done() @classmethod - def __compose_default_splits(cls, project, defaults, first_pass): + def __compose_default_splits(cls, project: Project, defaults: MappingNode, first_pass: bool) -> None: element_public = defaults.get_mapping(Symbol.PUBLIC, default={}) element_bst = element_public.get_mapping("bst", default={}) @@ -3016,7 +3109,7 @@ def __compose_default_splits(cls, project, defaults, first_pass): defaults[Symbol.PUBLIC] = element_public @classmethod - def __init_defaults(cls, project, plugin_conf, kind, first_pass): + def __init_defaults(cls, project: Project, plugin_conf: Optional[str], kind: str, first_pass: bool) -> None: # Defaults are loaded once per class and then reused # if cls.__defaults is None: @@ -3051,7 +3144,9 @@ def __init_defaults(cls, project, plugin_conf, kind, first_pass): # creating sandboxes for this element # @classmethod - def __extract_environment(cls, project, load_element): + def __extract_environment(cls, project: Project, load_element: LoadElement) -> MappingNode: + assert cls.__defaults is not None, "Need defaults for element" + default_env = cls.__defaults.get_mapping(Symbol.ENVIRONMENT, default={}) element_env = load_element.node.get_mapping(Symbol.ENVIRONMENT, default={}) or Node.from_dict({}) @@ -3067,7 +3162,9 @@ def __extract_environment(cls, project, load_element): return environment @classmethod - def __extract_env_nocache(cls, project, load_element): + def __extract_env_nocache(cls, project: Project, load_element: LoadElement) -> list[str]: + assert cls.__defaults is not None, "Need defaults for element" + if load_element.first_pass: project_nocache = [] else: @@ -3087,14 +3184,19 @@ def __extract_env_nocache(cls, project, load_element): # substituting command strings to be run in the sandbox # @classmethod - def __extract_variables(cls, project, load_element): + def __extract_variables(cls, project: Project, load_element: LoadElement) -> MappingNode: + assert cls.__defaults is not None, "Need defaults for element" + default_vars = cls.__defaults.get_mapping(Symbol.VARIABLES, default={}) element_vars = load_element.node.get_mapping(Symbol.VARIABLES, default={}) or Node.from_dict({}) if load_element.first_pass: - variables = project.first_pass_config.base_variables.clone() + base_variables = project.first_pass_config.base_variables else: - variables = project.base_variables.clone() + base_variables = project.base_variables + + assert base_variables, "base variables should be ready to go at this point" + variables = base_variables.clone() default_vars._composite(variables) element_vars._composite(variables) @@ -3119,7 +3221,9 @@ def __extract_variables(cls, project, load_element): # off to element.configure() # @classmethod - def __extract_config(cls, load_element): + def __extract_config(cls, load_element: LoadElement) -> MappingNode: + assert cls.__defaults is not None, "Element should have defaults" + element_config = load_element.node.get_mapping(Symbol.CONFIG, default={}) or Node.from_dict({}) # The default config is already composited with the project overrides @@ -3134,12 +3238,15 @@ def __extract_config(cls, load_element): # Sandbox-specific configuration data, to be passed to the sandbox's constructor. # @classmethod - def __extract_sandbox_config(cls, project, load_element): + def __extract_sandbox_config(cls, project: Project, load_element: LoadElement) -> MappingNode: + assert cls.__defaults is not None, "Elements should have defaults" + element_sandbox = load_element.node.get_mapping(Symbol.SANDBOX, default={}) or Node.from_dict({}) if load_element.first_pass: sandbox_config = Node.from_dict({}) else: + assert project.sandbox, "Project should have a sandbox config for this" sandbox_config = project.sandbox.clone() # The default config is already composited with the project overrides @@ -3156,8 +3263,10 @@ def __extract_sandbox_config(cls, project, load_element): # elements may extend but whos defaults are defined in the project. # @classmethod - def __extract_public(cls, load_element): - element_public = load_element.node.get_mapping(Symbol.PUBLIC, default={}) or Node.from_dict({}) + def __extract_public(cls, load_element: LoadElement) -> MappingNode: + assert cls.__defaults is not None, "Element should have defaults" + + element_public: MappingNode = load_element.node.get_mapping(Symbol.PUBLIC, default={}) base_public = cls.__defaults.get_mapping(Symbol.PUBLIC, default={}) base_public = base_public.clone() @@ -3166,8 +3275,8 @@ def __extract_public(cls, load_element): base_splits = base_bst.get_mapping("split-rules", default={}) element_public = element_public.clone() - element_bst = element_public.get_mapping("bst", default={}) - element_splits = element_bst.get_mapping("split-rules", default={}) + element_bst: MappingNode = element_public.get_mapping("bst", default={}) + element_splits: MappingNode = element_bst.get_mapping("split-rules", default={}) # Allow elements to extend the default splits defined in their project or # element specific defaults @@ -3180,8 +3289,9 @@ def __extract_public(cls, load_element): return element_public - def __init_splits(self): + def __init_splits(self) -> None: bstdata = self.get_public_data("bst") + assert bstdata, "bstdata required to load splits" splits = bstdata.get_mapping("split-rules") self.__splits = { domain: re.compile( @@ -3206,7 +3316,11 @@ def __init_splits(self): # Returns: # (bool): Whether to include the specified file # - def __split_filter(self, element_domains, include, exclude, orphans, path): + def __split_filter( + self, element_domains: list[str], include: list[str], exclude: list[str], orphans: bool, path: str + ) -> bool: + assert self.__splits is not None, "Splits required to filter splits" + # Absolute path is required for matching filename = os.path.join(os.sep, path) @@ -3241,13 +3355,17 @@ def __split_filter(self, element_domains, include, exclude, orphans, path): # (callable): Filter callback that returns True if the file is included # in the specified split domains. # - def __split_filter_func(self, include=None, exclude=None, orphans=True): + def __split_filter_func( + self, include: Optional[list[str]] = None, exclude: Optional[list[str]] = None, orphans: bool = True + ) -> Optional[Callable[[str], bool]]: + # No splitting requested, no filter needed if orphans and not (include or exclude): return None if not self.__splits: self.__init_splits() + assert self.__splits is not None, "We just ran __init_splits so this should never fail" element_domains = list(self.__splits.keys()) if not include: @@ -3265,10 +3383,13 @@ def __split_filter_func(self, include=None, exclude=None, orphans=True): # the required callback signature: a single `path` parameter. return partial(self.__split_filter, element_domains, include, exclude, orphans) - def __compute_splits(self, include=None, exclude=None, orphans=True): + def __compute_splits( + self, include: Optional[list[str]] = None, exclude: Optional[list[str]] = None, orphans: bool = True + ) -> Iterable[str]: + filter_func = self.__split_filter_func(include=include, exclude=exclude, orphans=orphans) - files_vdir = self.__artifact.get_files() + files_vdir = self._get_artifact().get_files() element_files = files_vdir.list_relative_paths() @@ -3284,17 +3405,17 @@ def __compute_splits(self, include=None, exclude=None, orphans=True): # # Loads the public data from the cached artifact # - def __load_public_data(self): + def __load_public_data(self) -> None: self.__assert_cached() - assert self.__dynamic_public is None + assert self.__dynamic_public is None, "Element has already loaded it's dynamic public data" - self.__dynamic_public = self.__artifact.load_public_data() + self.__dynamic_public = self._get_artifact().load_public_data() - def __load_build_result(self): + def __load_build_result(self) -> None: self.__assert_cached() - assert self.__build_result is None + assert self.__build_result is None, "Element has already loaded it's build results" - self.__build_result = self.__artifact.load_build_result() + self.__build_result = self._get_artifact().load_build_result() # __update_cache_keys() # @@ -3320,7 +3441,7 @@ def __load_build_result(self): # The strict cache key is a cache key that changes if any dependencies # in Scope.BUILD has changed in any way. # - def __update_cache_keys(self): + def __update_cache_keys(self) -> None: assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" if self.__strict_cache_key is not None: @@ -3338,30 +3459,12 @@ def __update_cache_keys(self): # so let's ensure we only ever calculate the weak key once, even though we need # to resolve it before we can resolve the strict key. if self.__weak_cache_key is None: - # Weak cache key includes names of direct build dependencies - # so as to only trigger rebuilds when the shape of the - # dependencies change. - # - # Some conditions cause dependencies to be strict, such - # that this element will be rebuilt anyway if the dependency - # changes even in non strict mode, for these cases we just - # encode the dependency's weak cache key instead of it's name. - # - dependencies = [ - ( - [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)] - if self.BST_STRICT_REBUILD or e in self.__strict_dependencies - else [e.project_name, e.name] - ) - for e in self._dependencies(_Scope.BUILD) - ] - self.__weak_cache_key = self._calculate_cache_key(dependencies) + self.__weak_cache_key = self._calculate_cache_key(_KeyStrength.WEAK) context = self._get_context() # Calculate the strict cache key - dependencies = [[e.project_name, e.name, e.__strict_cache_key] for e in self._dependencies(_Scope.BUILD)] - self.__strict_cache_key = self._calculate_cache_key(dependencies, self.__weak_cache_key) + self.__strict_cache_key = self._calculate_cache_key(_KeyStrength.STRICT, self.__weak_cache_key) if self.__strict_cache_key is None: # Cache keys cannot be calculated yet as a build dependency doesn't @@ -3392,9 +3495,9 @@ def __update_cache_keys(self): # as the cache key can be loaded from the cache (possibly pulling from # a remote cache). # - def __update_cache_key_non_strict(self): + def __update_cache_key_non_strict(self) -> None: assert utils._is_in_main_thread(), "This has an impact on all elements and must be run in the main thread" - + artifact = self._get_artifact() # The final cache key can be None here only in non-strict mode if self.__cache_key is None: if self._pull_pending(): @@ -3402,14 +3505,12 @@ def __update_cache_key_non_strict(self): pass elif self._cached(): # Load the strong cache key from the artifact - strong_key, _, _ = self.__artifact.get_metadata_keys() + strong_key, _, _ = artifact.get_metadata_keys() self.__cache_key = strong_key elif self.__assemble_scheduled or self.__assemble_done: # Artifact will or has been built, not downloaded assert self.__weak_cache_key is not None - - dependencies = [[e.project_name, e.name, e._get_cache_key()] for e in self._dependencies(_Scope.BUILD)] - self.__cache_key = self._calculate_cache_key(dependencies, self.__weak_cache_key) + self.__cache_key = self._calculate_cache_key(_KeyStrength.STRONG, self.__weak_cache_key) if self.__cache_key is None: # Strong cache key could not be calculated yet @@ -3420,7 +3521,7 @@ def __update_cache_key_non_strict(self): self._update_ready_for_runtime_and_cached() # Now we have the strong cache key, update the Artifact - self.__artifact._cache_key = self.__cache_key + artifact._cache_key = self.__cache_key # Update the message kwargs in use for this plugin to dispatch messages with self._message_kwargs["element_key"] = self._get_display_key() @@ -3437,7 +3538,7 @@ def __update_cache_key_non_strict(self): # Returns: # (str): The normalised element name # -def _get_normal_name(element_name): +def _get_normal_name(element_name: str) -> str: return os.path.splitext(element_name.replace(os.sep, "-"))[0] @@ -3453,7 +3554,7 @@ def _get_normal_name(element_name): # Returns: # (str): The constructed artifact name path # -def _compose_artifact_name(project_name, normal_name, cache_key): +def _compose_artifact_name(project_name: str, normal_name: str, cache_key: str) -> str: valid_chars = string.digits + string.ascii_letters + "-._" normal_name = "".join([x if x in valid_chars else "_" for x in normal_name]) diff --git a/src/buildstream/node.pyi b/src/buildstream/node.pyi index 6fcb3719e..956c36205 100644 --- a/src/buildstream/node.pyi +++ b/src/buildstream/node.pyi @@ -32,7 +32,27 @@ from ._project import Project TNode = TypeVar("TNode", bound="Node") TValidNodeValue = TypeVar("TValidNodeValue", int, str, bool, Mapping, Sequence) -class ProvenanceInformation: ... +class ProvenanceInformation: + """Represents the location of a YAML node in a file. + + This can effectively be used as a pretty print to display those information in + errors consistently. + + You can retrieve this information for a :class:`Node` with + :func:`Node.get_provenance() ` + """ + + _node: MappingNode + _filename: str + _shortname: str + _displayname: str + _line: int + _col: int + _toplevel: MappingNode | None + _project: Project | None + _is_synthetic: bool + + def __init__(self, nodeish: Node): ... class Node: def clone(self) -> "Node": ... @@ -50,6 +70,8 @@ class Node: class ScalarNode(Node): def as_str(self) -> str: ... def clone(self) -> "ScalarNode": ... + def is_none(self) -> bool: ... + def as_bool(self) -> bool: ... class SequenceNode(Node, Generic[TNode]): def __iter__(self) -> "SequenceNode": ... @@ -60,11 +82,12 @@ class SequenceNode(Node, Generic[TNode]): class MappingNode(Node, Generic[TNode]): def __init__(self, file_index: int, line: int, column: int, value: Mapping[str, TValidNodeValue]) -> None: ... def __contains__(self, what: Any) -> bool: ... + def _find(self, target: Node) -> list[Node]: ... def clone(self) -> MappingNode[TNode]: ... def keys(self) -> Iterable[str]: ... def items(self) -> Iterable[Tuple[str, Any]]: ... def safe_del(self, key: str) -> None: ... - def validate_keys(self, valid_keys: List[str]): ... + def validate_keys(self, valid_keys: Iterable[str]): ... def values(self) -> List[Node]: ... @overload def get_scalar(self, key: str) -> ScalarNode: ... @@ -105,6 +128,8 @@ class MappingNode(Node, Generic[TNode]): self, key: str, default: Union["MappingNode", Dict[str, Any], None] ) -> Optional["MappingNode"]: ... @overload + def get_sequence(self, key: str, default: Optional[List[Any]]) -> SequenceNode: ... + @overload def get_sequence(self, key: str, *, allowed_types: Optional[List[Type[Node]]]) -> SequenceNode: ... @overload def get_sequence( @@ -117,6 +142,8 @@ class MappingNode(Node, Generic[TNode]): @overload def get_node(self, key: str) -> Node: ... @overload + def get_node(self, key: str, allow_none: bool) -> Optional[Node]: ... + @overload def get_node(self, key: str, allowed_types: Optional[List[Type[Node]]]) -> Node: ... @overload def get_node(self, key: str, allowed_types: Optional[List[Type[Node]]], allow_none: bool) -> Optional[Node]: ... @@ -124,8 +151,13 @@ class MappingNode(Node, Generic[TNode]): # Private # def _composite(self, target: "MappingNode") -> None: ... + def _assert_fully_composited(self) -> None: ... + def __setitem__(self, key: str, value: Any) -> None: ... def _assert_symbol_name( - symbol_name: str, purpose: str, *, ref_node: Optional[Node], allow_dashes: bool = True + symbol_name: str, purpose: str, *, ref_node: Optional[Node] = None, allow_dashes: bool = True ) -> None: ... def _new_synthetic_file(filename: str, project: Optional[Project]) -> MappingNode[TNode]: ... +def _reset_global_state(): ... + +_SYNTHETIC_FILE_INDEX = -1 diff --git a/src/buildstream/plugin.py b/src/buildstream/plugin.py index eb5679453..b613a806c 100644 --- a/src/buildstream/plugin.py +++ b/src/buildstream/plugin.py @@ -135,7 +135,7 @@ from . import utils, _signals from ._exceptions import PluginError, ImplError from ._message import Message, MessageType -from .node import Node, MappingNode +from .node import Node, MappingNode, ProvenanceInformation from .types import CoreWarnings, SourceRef if TYPE_CHECKING: @@ -322,7 +322,7 @@ def __init__( # reference to the Project, it keeps the plugin factory alive. If the # factory were to be GC'd then we would see undefined behaviour. Make # sure to test plugin pickling if this reference is to be removed. - self.__project = project # The Project object + self.__project: Project = project # The Project object self.__provenance_node = provenance_node # The originating YAML node self.__type_tag = type_tag # The type of plugin (element or source) @@ -838,21 +838,21 @@ def _lookup(cls, unique_id): # # Fetches the invocation context # - def _get_context(self): + def _get_context(self) -> "Context": return self.__context # _get_project() # # Fetches the project object associated with this plugin # - def _get_project(self): + def _get_project(self) -> "Project": return self.__project # _get_provenance(): # # Fetch bst file, line and column of the entity # - def _get_provenance(self): + def _get_provenance(self) -> ProvenanceInformation: return self.__provenance_node.get_provenance() # Context manager for getting the open file handle to this diff --git a/src/buildstream/scriptelement.py b/src/buildstream/scriptelement.py index a24ddcee8..b8452a639 100644 --- a/src/buildstream/scriptelement.py +++ b/src/buildstream/scriptelement.py @@ -36,15 +36,15 @@ from .element import Element if TYPE_CHECKING: - from typing import Dict, Tuple + pass class ScriptElement(Element): __install_root = "/" __cwd = "/" __root_read_only = False - __commands = None # type: OrderedDict[str, List[str]] - __layout = {} # type: Dict[str, List[Tuple[Element, str]]] + __commands: Optional[OrderedDict[str, list[str]]] = None + __layout: dict[str, list[tuple[Element, str]]] = {} # The compose element's output is its dependencies, so # we must rebuild if the dependencies change even when @@ -243,6 +243,7 @@ def stage(self, sandbox): def assemble(self, sandbox): with sandbox.batch(root_read_only=self.__root_read_only, collect=self.__install_root): + assert self.__commands, "Commands are required in a script element" for groupname, commands in self.__commands.items(): with sandbox.batch(root_read_only=self.__root_read_only, label="Running '{}'".format(groupname)): for cmd in commands: diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 704d54c55..2a6740ece 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -777,7 +777,7 @@ class Source(Plugin): """ # The defaults from the project - __defaults: Optional[Dict[str, Any]] = None + __defaults: Optional[MappingNode] = None BST_CUSTOM_SOURCE_PROVENANCE = False """Whether multiple sources' provenance information are provided @@ -1186,6 +1186,7 @@ def get_mirror_directory(self) -> str: if self.__mirror_directory is None: # Create the directory if it doesnt exist context = self._get_context() + assert context.sourcedir, "Must have a source dir to get mirror directory" directory = os.path.join(context.sourcedir, self.get_kind()) os.makedirs(directory, exist_ok=True) self.__mirror_directory = directory @@ -1239,7 +1240,7 @@ def translate_url( url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1) project_alias_url = project.get_alias_url(url_alias, first_pass=self.__first_pass) - + assert project_alias_url, "project get alias url should have returns something" if self.__alias_override is not None: override_alias = self.__alias_override[0] override_subst = self.__alias_override[1] @@ -1361,6 +1362,7 @@ def get_project_directory(self) -> str: The project base directory """ project = self._get_project() + assert project.directory, "Project must have a directory" return project.directory @contextmanager @@ -1433,6 +1435,9 @@ def create_source_info( if provenance_node is not None: # Ensure provenance node keys are valid and values are all strings try: + assert ( + project.source_provenance_attributes + ), "must have source_provenance_attributes from project for source" provenance_node.validate_keys(project.source_provenance_attributes.keys()) except LoadError as E: raise LoadError( @@ -1683,7 +1688,7 @@ def do_load_ref(node): # Raises: # (SourceError): In the case we encounter errors saving a file to disk # - def _set_ref(self, new_ref, *, save): + def _set_ref(self, new_ref: None | int | str | list[Any] | dict[str, Any], *, save: bool): context = self._get_context() project = self._get_project() @@ -1697,13 +1702,15 @@ def _set_ref(self, new_ref, *, save): # # Step 1 - Obtain the node # - node = {} + node: MappingNode | None = None if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS: node = toplevel_refs.lookup_ref(project.name, element_name, element_idx, write=True) if project is toplevel and not node: node = provenance._node + assert node, "Node should now be set, or we have a problem" + # # Step 2 - Set the ref in memory, and determine changed state # @@ -1716,7 +1723,7 @@ def _set_ref(self, new_ref, *, save): # In the following add/del/mod merge algorithm we are working with # dictionaries, but the plugin API calls for a MappingNode. # - modify = node.clone() + modify: MappingNode = node.clone() self.set_ref(new_ref, modify) to_modify = modify.strip_node_info() @@ -1786,7 +1793,7 @@ def process_value(action, container, path, key, new_value): else: assert False, "BUG: Unknown action: {}".format(action) - roundtrip_cache = {} + roundtrip_cache: dict[str, Any] = {} for key, action in actions.items(): # Obtain the top level node and its file if action == "add": @@ -1795,6 +1802,7 @@ def process_value(action, container, path, key, new_value): provenance = node.get_node(key).get_provenance() toplevel_node = provenance._toplevel + assert toplevel_node is not None, "Must have a top level node for the provenance" # Get the path to whatever changed if action == "add": @@ -1891,7 +1899,7 @@ def _get_source_name(self): def _get_brief_display_key(self): context = self._get_context() key = self._key - + assert context.log_key_length, "Need log key length" length = min(len(key), context.log_key_length) return key[:length] @@ -2030,7 +2038,8 @@ def __do_fetch(self, **kwargs): else: # No break occurred, raise the last detected error - raise last_error + if last_error: + raise last_error # Default codepath is to reinstantiate the Source # @@ -2056,7 +2065,8 @@ def __do_fetch(self, **kwargs): return # Re raise the last detected error - raise last_error + if last_error: + raise last_error # Tries to call track for every mirror, stopping once it succeeds def __do_track(self, **kwargs): @@ -2080,11 +2090,11 @@ def __do_track(self, **kwargs): continue return ref - - raise last_error + if last_error: + raise last_error @classmethod - def __init_defaults(cls, project, meta): + def __init_defaults(cls, project: "Project", meta: MetaSource): if cls.__defaults is None: if meta.first_pass: sources = project.first_pass_config.source_overrides @@ -2096,7 +2106,8 @@ def __init_defaults(cls, project, meta): # off to source.configure() # @classmethod - def __extract_config(cls, meta): + def __extract_config(cls, meta: MetaSource) -> MappingNode: + assert cls.__defaults, "Need source plugin defaults here" config = cls.__defaults.get_mapping("config", default={}) config = config.clone() @@ -2108,7 +2119,7 @@ def __extract_config(cls, meta): def _extract_alias(url): parts = url.split(utils._ALIAS_SEPARATOR, 1) - if len(parts) > 1 and not parts[0].lower() in utils._URI_SCHEMES: + if len(parts) > 1 and parts[0].lower() not in utils._URI_SCHEMES: return parts[0] else: return "" diff --git a/src/buildstream/types.py b/src/buildstream/types.py index bd69dd21e..cbf60c6a8 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -211,6 +211,10 @@ class _KeyStrength(FastEnum): # cache keys of dependencies. WEAK = 2 + # The strict cache key is a cache key that changes if any dependencies + # in Scope.BUILD has changed in any way. + STRICT = 3 + # _DisplayKey(): # diff --git a/src/buildstream/types.pyi b/src/buildstream/types.pyi index 8a75b9f13..0d36a31c6 100644 --- a/src/buildstream/types.pyi +++ b/src/buildstream/types.pyi @@ -47,7 +47,7 @@ class OverlapAction(Enum): from ._types import MetaFastEnum as MetaFastEnum from .node import MappingNode as MappingNode, SequenceNode as SequenceNode from _typeshed import Incomplete -from typing import Any +from typing import Any, Literal from enum import Enum class FastEnum(metaclass=MetaFastEnum): @@ -83,11 +83,11 @@ class CoreWarnings: Some common warnings which are raised by core functionalities within BuildStream are found in this class. """ - OVERLAPS: str - UNSTAGED_FILES: str - REF_NOT_IN_TRACK: str - UNALIASED_URL: str - UNAVAILABLE_SOURCE_INFO: str + OVERLAPS = "overlaps" + UNSTAGED_FILES = "unstaged-files" + REF_NOT_IN_TRACK = "ref-not-in-track" + UNALIASED_URL = "unaliased-url" + UNAVAILABLE_SOURCE_INFO = "unavailable-source-info" class OverlapAction(Enum): """OverlapAction() @@ -111,19 +111,20 @@ class OverlapAction(Enum): as a :ref:`fatal warning `. """ - ERROR: str - WARNING: str - IGNORE: str + ERROR = "error" + WARNING = "warning" + IGNORE = "ignore" class _Scope(Enum): - ALL: int - BUILD: int - RUN: int - NONE: int + ALL = 1 + BUILD = 2 + RUN = 3 + NONE = 4 class _KeyStrength(Enum): - STRONG: int - WEAK: int + STRONG = 1 + WEAK = 2 + STRICT = 3 class _DisplayKey: full: str @@ -132,27 +133,27 @@ class _DisplayKey: def __init__(self, full: str, brief: str, strict: bool) -> None: ... class _SchedulerErrorAction(Enum): - CONTINUE: str - QUIT: str - TERMINATE: str + CONTINUE = "continue" + QUIT = "quit" + TERMINATE = "terminate" class _CacheBuildTrees(Enum): - ALWAYS: str - AUTO: str - NEVER: str + ALWAYS = "always" + AUTO = "auto" + NEVER = "never" class _SourceUriPolicy(Enum): - ALL: str - ALIASES: str - MIRRORS: str - USER: str + ALL = "all" + ALIASES = "aliases" + MIRRORS = "mirrors" + USER = "user" class _PipelineSelection(Enum): - NONE: str - REDIRECT: str - ALL: str - BUILD: str - RUN: str + NONE = "none" + REDIRECT = "redirect" + ALL = "all" + BUILD = "build" + RUN = "run" class _ProjectInformation: project: Incomplete diff --git a/tests/integration/shell.py b/tests/integration/shell.py index 63c7af8b3..d918f5d05 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -15,13 +15,17 @@ # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name + import os +from typing import List, Tuple import uuid import pytest from buildstream import _yaml +from buildstream.node import MappingNode from buildstream._testing import cli_integration as cli # pylint: disable=unused-import +from buildstream._testing.runcli import Result, CliIntegration from buildstream._testing._utils.site import HAVE_SANDBOX, BUILDBOX_RUN from buildstream.exceptions import ErrorDomain from buildstream import utils @@ -46,8 +50,22 @@ # mount (tuple): A (host, target) tuple for the `--mount` option # element (str): The element to build and run a shell with # isolate (bool): Whether to pass --isolate to `bst shell` -# -def execute_shell(cli, project, command, *, config=None, mount=None, element="base.bst", isolate=False): +# other_elements (list(str)): Other elements to stage in the sandbox +def execute_shell( + cli: CliIntegration, + project: str, + command: List[str], + *, + config: dict | MappingNode | None = None, + mount: Tuple[str, str] | None = None, + element: str = "base.bst", + isolate: bool = False, + build: bool = False, +) -> Result: + # Ensure config is a mapping node + if isinstance(config, dict): + config = MappingNode.from_dict(config) + # Ensure the element is built result = cli.run_project_config(project=project, project_config=config, args=["build", element]) assert result.exit_code == 0 @@ -55,11 +73,13 @@ def execute_shell(cli, project, command, *, config=None, mount=None, element="ba args = ["shell"] if isolate: args += ["--isolate"] + if build: + args += ["--build"] if mount is not None: host_path, target_path = mount args += ["--mount", host_path, target_path] args += [element, "--", *command] - + # cli.verbose = True return cli.run_project_config(project=project, project_config=config, args=args) @@ -83,7 +103,14 @@ def test_executable(cli, datafiles): result = execute_shell(cli, project, ["/bin/echo", "Horseys!"]) assert result.exit_code == 0 - assert result.output == "Horseys!\n" + assert result.output == "Horseys!\n", "echo should be present and working from base.bst" + + # Running executable directly from a build dependency + result = execute_shell(cli, project, ["/bin/echo", "Horseys!"], element="build-shell/buildtree.bst", build=True) + assert result.exit_code == 0 + assert ( + result.output == "Horseys!\n" + ), "echo should be present and working from base.bst as a dependency of build-shell/buildtree.bst" # Test shell environment variable explicit assignments @@ -96,7 +123,10 @@ def test_env_assign(cli, datafiles, animal): expected = animal + "\n" result = execute_shell( - cli, project, ["/bin/sh", "-c", "echo ${ANIMAL}"], config={"shell": {"environment": {"ANIMAL": animal}}} + cli, + project, + ["/bin/sh", "-c", "echo ${ANIMAL}"], + config=MappingNode.from_dict({"shell": {"environment": {"ANIMAL": animal}}}), ) assert result.exit_code == 0 @@ -237,6 +267,7 @@ def test_isolated_no_mount(cli, datafiles, path): config={"shell": {"host-files": [{"host_path": ponyfile, "path": path}]}}, ) assert result.exit_code != 0 + assert result.stderr assert path in result.stderr assert "No such file or directory" in result.stderr @@ -261,7 +292,7 @@ def test_host_files_missing(cli, datafiles, optional): ) assert result.exit_code == 0 assert result.output == "Hello\n" - + assert result.stderr if option: # Assert that there was no warning about the mount assert ponyfile not in result.stderr diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py index 8a5f94e65..b5a5da55c 100644 --- a/tests/integration/workspace.py +++ b/tests/integration/workspace.py @@ -360,7 +360,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles): files = res.output.splitlines() assert "./prepared" in files - assert not "./prepared-again" in files + assert "./prepared-again" not in files # Test that rebuilding an already built workspaced element does