diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 546d91187e05e..d2a22f3a135ed 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -310,7 +310,7 @@ struct CollectRegionConstraintsResult<'tcx> { deferred_closure_requirements: DeferredClosureRequirements<'tcx>, deferred_opaque_type_errors: Vec>, polonius_facts: Option>, - polonius_context: Option, + polonius_context: Option>, } /// Start borrow checking by collecting the region constraints for @@ -798,7 +798,7 @@ pub(crate) struct MirBorrowckCtxt<'a, 'diag, 'tcx> { /// Results of Polonius analysis. polonius_output: Option<&'a PoloniusOutput>, /// When using `-Zpolonius=next`: the data used to compute errors and diagnostics. - polonius_context: Option<&'a PoloniusContext>, + polonius_context: Option<&'a PoloniusContext<'tcx>>, } // Check that: diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs index 1a328f62fc73e..fed7fdd97ac4d 100644 --- a/compiler/rustc_borrowck/src/nll.rs +++ b/compiler/rustc_borrowck/src/nll.rs @@ -46,7 +46,7 @@ pub(crate) struct NllOutput<'tcx> { /// When using `-Zpolonius=next`: the data used to compute errors and diagnostics, e.g. /// localized typeck and liveness constraints. - pub polonius_context: Option, + pub polonius_context: Option>, } /// Rewrites the regions in the MIR to use NLL variables, also scraping out the set of universal @@ -121,7 +121,7 @@ pub(crate) fn compute_regions<'tcx>( universal_region_relations: Frozen>, constraints: MirTypeckRegionConstraints<'tcx>, mut polonius_facts: Option>, - mut polonius_context: Option, + mut polonius_context: Option>, ) -> NllOutput<'tcx> { let polonius_output = root_cx.consumer.as_ref().map_or(false, |c| c.polonius_output()) || infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled(); @@ -144,20 +144,20 @@ pub(crate) fn compute_regions<'tcx>( &lowered_constraints, ); - let num_points = location_map.num_points(); - // If requested for `-Zpolonius=next`, compute loan liveness information. // This is done prior to `RegionInferenceContext::new`, because we may add // additional liveness constraints. if let Some(polonius_context) = polonius_context.as_mut() { let _timer = infcx.tcx.prof.generic_activity("borrowck_polonius_loan_liveness"); polonius_context.compute_loan_liveness( + infcx, &mut lowered_constraints.liveness_constraints, lowered_constraints.outlives_constraints.outlives().iter().copied(), &universal_region_relations.universal_regions, body, + move_data, + &location_map, borrow_set, - num_points, ); } diff --git a/compiler/rustc_borrowck/src/polonius/constraints.rs b/compiler/rustc_borrowck/src/polonius/constraints.rs index 637068a9799d7..1b819e3afcb67 100644 --- a/compiler/rustc_borrowck/src/polonius/constraints.rs +++ b/compiler/rustc_borrowck/src/polonius/constraints.rs @@ -1,15 +1,14 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; -use rustc_index::interval::SparseIntervalMatrix; +use rustc_index::interval::{IntervalSet, SparseIntervalMatrix}; use rustc_middle::mir::{Body, Location}; use rustc_middle::ty::RegionVid; -use rustc_mir_dataflow::points::PointIndex; +use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; use tracing::debug; use crate::BorrowSet; use crate::constraints::OutlivesConstraint; use crate::dataflow::BorrowIndex; use crate::polonius::{ConstraintDirection, LiveRegionVariances}; -use crate::region_infer::values::LivenessValues; use crate::type_check::Locations; use crate::universal_regions::UniversalRegions; @@ -50,11 +49,51 @@ pub(super) struct LocalizedConstraintGraph { logical_edges: FxHashMap>, } +/// For a given region, the relevant liveness and variance information. +pub(super) struct RegionLiveness<'a> { + region: RegionVid, + direction: ConstraintDirection, + live_points: Option<&'a IntervalSet>, +} + +impl<'a> RegionLiveness<'a> { + pub(super) fn new( + region: RegionVid, + live_region_variances: &LiveRegionVariances, + live_points: &'a SparseIntervalMatrix, + ) -> Self { + // Note: there currently are cases related to promoted and const generics, where we don't yet + // have variance information (possibly about temporary regions created when typeck sanitizes the + // promoteds). Until that is done, we conservatively fallback to maximizing reachability by + // adding a bidirectional edge here. This will not limit traversal whatsoever, and thus + // propagate liveness when needed. + // + // FIXME: add the missing variance information and remove this fallback bidirectional edge. + let direction = live_region_variances + .get(region) + .copied() + .flatten() + .unwrap_or(ConstraintDirection::Bidirectional); + let live_points = live_points.row(region); + Self { region, direction, live_points } + } + + fn is_live_at(&self, point: PointIndex) -> bool { + self.live_points.map_or(false, |points| points.contains(point)) + } +} + +/// The source of liveness information for a given region. +pub(super) trait LivenessSource<'loc> { + fn liveness_for_region(&mut self, region: RegionVid) -> RegionLiveness<'_>; + fn location_map(&self) -> &'loc DenseLocationMap; +} + /// The visitor interface when traversing a `LocalizedConstraintGraph`. pub(super) trait LocalizedConstraintGraphVisitor { /// Callback called when traversing a given `loan` encounters a localized `node` it hasn't /// visited before. - fn on_node_traversed(&mut self, _loan: BorrowIndex, _node: LocalizedNode) {} + fn on_node_traversed(&mut self, _loan: BorrowIndex, _node: LocalizedNode, _is_live: bool) {} /// Callback called when discovering a new `successor` node for the `current_node`. fn on_successor_discovered(&mut self, _current_node: LocalizedNode, _successor: LocalizedNode) { @@ -64,7 +103,7 @@ pub(super) trait LocalizedConstraintGraphVisitor { impl LocalizedConstraintGraph { /// Traverses the constraints and returns the indexed graph of edges per node. pub(super) fn new<'tcx>( - liveness: &LivenessValues, + location_map: &DenseLocationMap, outlives_constraints: impl Iterator>, ) -> Self { let mut edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default(); @@ -82,7 +121,7 @@ impl LocalizedConstraintGraph { Locations::Single(location) => { let node = LocalizedNode { region: outlives_constraint.sup, - point: liveness.point_from_location(location), + point: location_map.point_from_location(location), }; edges.entry(node).or_default().insert(outlives_constraint.sub); } @@ -94,20 +133,19 @@ impl LocalizedConstraintGraph { /// Traverses the localized constraint graph per-loan, and notifies the `visitor` of discovered /// nodes and successors. - pub(super) fn traverse<'tcx>( + pub(super) fn traverse<'tcx, 'loc>( &self, body: &Body<'tcx>, - liveness: &LivenessValues, - live_region_variances: &LiveRegionVariances, universal_regions: &UniversalRegions<'tcx>, borrow_set: &BorrowSet<'tcx>, + liveness_source: &mut impl LivenessSource<'loc>, visitor: &mut impl LocalizedConstraintGraphVisitor, ) { - let live_regions = liveness.points(); - let mut visited = FxHashSet::default(); let mut stack = Vec::new(); + let location_map = liveness_source.location_map(); + // Compute reachability per loan by traversing each loan's subgraph starting from where it // is introduced. for (loan_idx, loan) in borrow_set.iter_enumerated() { @@ -116,7 +154,7 @@ impl LocalizedConstraintGraph { let start_node = LocalizedNode { region: loan.region, - point: liveness.point_from_location(loan.reserve_location), + point: location_map.point_from_location(loan.reserve_location), }; stack.push(start_node); @@ -125,9 +163,10 @@ impl LocalizedConstraintGraph { continue; } + let liveness = liveness_source.liveness_for_region(node.region); // We've reached a node we haven't visited before. - let location = liveness.location_from_point(node.point); - visitor.on_node_traversed(loan_idx, node); + let location = location_map.to_location(node.point); + visitor.on_node_traversed(loan_idx, node, liveness.is_live_at(node.point)); // When we find a _new_ successor, we'd like to // - visit it eventually, @@ -162,13 +201,9 @@ impl LocalizedConstraintGraph { // Intra-block edges, straight line constraints from each point to its successor // within the same block. let next_point = node.point + 1; - if let Some(succ) = compute_forward_successor( - node.region, - next_point, - live_regions, - live_region_variances, - is_universal_region, - ) { + if let Some(succ) = + compute_forward_successor(&liveness, next_point, is_universal_region) + { successor_found(succ); } } else { @@ -176,14 +211,10 @@ impl LocalizedConstraintGraph { // entry point. for successor_block in body[location.block].terminator().successors() { let next_location = Location { block: successor_block, statement_index: 0 }; - let next_point = liveness.point_from_location(next_location); - if let Some(succ) = compute_forward_successor( - node.region, - next_point, - live_regions, - live_region_variances, - is_universal_region, - ) { + let next_point = location_map.point_from_location(next_location); + if let Some(succ) = + compute_forward_successor(&liveness, next_point, is_universal_region) + { successor_found(succ); } } @@ -195,13 +226,9 @@ impl LocalizedConstraintGraph { if location.statement_index > 0 { // Backward edges to the predecessor point in the same block. let previous_point = PointIndex::from(node.point.as_usize() - 1); - if let Some(succ) = compute_backward_successor( - node.region, - node.point, - previous_point, - live_regions, - live_region_variances, - ) { + if let Some(succ) = + compute_backward_successor(&liveness, node.point, previous_point) + { successor_found(succ); } } else { @@ -213,14 +240,11 @@ impl LocalizedConstraintGraph { block: pred_block, statement_index: body[pred_block].statements.len(), }; - let previous_point = liveness.point_from_location(previous_location); - if let Some(succ) = compute_backward_successor( - node.region, - node.point, - previous_point, - live_regions, - live_region_variances, - ) { + let previous_point = + location_map.point_from_location(previous_location); + if let Some(succ) = + compute_backward_successor(&liveness, node.point, previous_point) + { successor_found(succ); } } @@ -240,12 +264,12 @@ impl LocalizedConstraintGraph { /// Returns the successor for the current region/point node when propagating a loan through forward /// edges, if applicable, according to liveness and variance. fn compute_forward_successor( - region: RegionVid, + liveness: &RegionLiveness<'_>, next_point: PointIndex, - live_regions: &SparseIntervalMatrix, - live_region_variances: &LiveRegionVariances, is_universal_region: bool, ) -> Option { + let region = liveness.region; + // 1. Universal regions are semantically live at all points. if is_universal_region { let succ = LocalizedNode { region, point: next_point }; @@ -253,7 +277,7 @@ fn compute_forward_successor( } // 2. Otherwise, gather the edges due to explicit region liveness, when applicable. - if !live_regions.contains(region, next_point) { + if !liveness.is_live_at(next_point) { debug!(?region, ?next_point, "region isn't live at successor"); return None; } @@ -261,22 +285,9 @@ fn compute_forward_successor( // Here, `region` could be live at the current point, and is live at the next point: add a // constraint between them, according to variance. - // Note: there currently are cases related to promoted and const generics, where we don't yet - // have variance information (possibly about temporary regions created when typeck sanitizes the - // promoteds). Until that is done, we conservatively fallback to maximizing reachability by - // adding a bidirectional edge here. This will not limit traversal whatsoever, and thus - // propagate liveness when needed. - // - // FIXME: add the missing variance information and remove this fallback bidirectional edge. - let direction = live_region_variances - .get(region) - .copied() - .flatten() - .unwrap_or(ConstraintDirection::Bidirectional); - - debug!(?direction); - - match direction { + debug!(?liveness.direction); + + match liveness.direction { ConstraintDirection::Backward => { // Contravariant cases: loans flow in the inverse direction, but we're only interested // in forward successors and there are none here. @@ -295,28 +306,20 @@ fn compute_forward_successor( /// Returns the successor for the current region/point node when propagating a loan through backward /// edges, if applicable, according to liveness and variance. fn compute_backward_successor( - region: RegionVid, + liveness: &RegionLiveness<'_>, current_point: PointIndex, previous_point: PointIndex, - live_regions: &SparseIntervalMatrix, - live_region_variances: &LiveRegionVariances, ) -> Option { + let region = liveness.region; + // Liveness flows into the regions live at the next point. So, in a backwards view, we'll link // the region from the current point, if it's live there, to the previous point. - if !live_regions.contains(region, current_point) { + if !liveness.is_live_at(current_point) { debug!(?region, ?current_point, "region isn't live at current point"); return None; } - // FIXME: add the missing variance information and remove this fallback bidirectional edge. See - // the same comment in `compute_forward_successor`. - let direction = live_region_variances - .get(region) - .copied() - .flatten() - .unwrap_or(ConstraintDirection::Bidirectional); - - match direction { + match liveness.direction { ConstraintDirection::Forward => { // Covariant cases: loans flow in the regular direction, but we're only interested in // backward successors and there are none here. diff --git a/compiler/rustc_borrowck/src/polonius/dump.rs b/compiler/rustc_borrowck/src/polonius/dump.rs index 0c63b00ce5461..5d88f71dec97c 100644 --- a/compiler/rustc_borrowck/src/polonius/dump.rs +++ b/compiler/rustc_borrowck/src/polonius/dump.rs @@ -5,13 +5,16 @@ use rustc_index::IndexVec; use rustc_middle::mir::pretty::{MirDumper, PassWhere, PrettyPrintMirOptions}; use rustc_middle::mir::{Body, Location}; use rustc_middle::ty::{RegionVid, TyCtxt}; -use rustc_mir_dataflow::points::PointIndex; +use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; use rustc_session::config::MirIncludeSpans; use crate::borrow_set::BorrowSet; use crate::constraints::OutlivesConstraint; use crate::dataflow::BorrowIndex; -use crate::polonius::{LocalizedConstraintGraphVisitor, LocalizedNode, PoloniusContext}; +use crate::polonius::{ + LiveRegionVariances, LivenessSource, LocalizedConstraintGraphVisitor, LocalizedNode, + PoloniusContext, RegionLiveness, +}; use crate::region_infer::values::LivenessValues; use crate::type_check::Locations; use crate::{BorrowckInferCtxt, ClosureRegionRequirements, RegionInferenceContext}; @@ -20,6 +23,21 @@ use crate::{BorrowckInferCtxt, ClosureRegionRequirements, RegionInferenceContext /// sections to be replaced by real contents. const TEMPLATE: &str = include_str!("./dump/polonius-mir-dump.template.html"); +/// A `LivenessSource` for already-existing liveness and variance data. +struct CachedLivenessSource<'a> { + live_region_variances: &'a LiveRegionVariances, + liveness: &'a LivenessValues, +} + +impl<'a> LivenessSource<'a> for CachedLivenessSource<'a> { + fn liveness_for_region(&mut self, region: RegionVid) -> RegionLiveness<'_> { + RegionLiveness::new(region, self.live_region_variances, self.liveness.points()) + } + fn location_map(&self) -> &'a DenseLocationMap { + self.liveness.location_map() + } +} + /// `-Zdump-mir=polonius` dumps MIR annotated with NLL and polonius specific information. pub(crate) fn dump_polonius_mir<'tcx>( infcx: &BorrowckInferCtxt<'tcx>, @@ -27,7 +45,7 @@ pub(crate) fn dump_polonius_mir<'tcx>( regioncx: &RegionInferenceContext<'tcx>, closure_region_requirements: &Option>, borrow_set: &BorrowSet<'tcx>, - polonius_context: Option<&PoloniusContext>, + polonius_context: Option<&PoloniusContext<'tcx>>, ) { let tcx = infcx.tcx; if !tcx.sess.opts.unstable_opts.polonius.is_next_enabled() { @@ -41,14 +59,17 @@ pub(crate) fn dump_polonius_mir<'tcx>( // If we have a polonius graph to dump along the rest of the MIR and NLL info, we extract its // constraints here. + let mut liveness_source = CachedLivenessSource { + live_region_variances: &polonius_context.live_region_variances, + liveness: regioncx.liveness_constraints(), + }; let mut collector = MirDumpCollector::default(); if let Some(graph) = &polonius_context.graph { graph.traverse( body, - regioncx.liveness_constraints(), - &polonius_context.live_region_variances, regioncx.universal_regions(), borrow_set, + &mut liveness_source, &mut collector, ); } @@ -98,7 +119,7 @@ struct MirDumpCollector { } impl LocalizedConstraintGraphVisitor for MirDumpCollector { - fn on_node_traversed(&mut self, loan: BorrowIndex, node: LocalizedNode) { + fn on_node_traversed(&mut self, loan: BorrowIndex, node: LocalizedNode, _is_live: bool) { self.reachability.entry(loan).or_default().push(node); } diff --git a/compiler/rustc_borrowck/src/polonius/liveness.rs b/compiler/rustc_borrowck/src/polonius/liveness.rs new file mode 100644 index 0000000000000..9faf01129fdda --- /dev/null +++ b/compiler/rustc_borrowck/src/polonius/liveness.rs @@ -0,0 +1,66 @@ +use rustc_index::IndexVec; +use rustc_middle::mir::Local; +use rustc_middle::ty::{GenericArg, RegionVid, Ty}; + +use crate::BorrowckInferCtxt; +use crate::universal_regions::UniversalRegions; + +#[derive(Default)] +pub(crate) struct DeferredLocals<'tcx> { + /// For each region, the local whose liveness is deferred. + /// + /// Importantly, because of MIR renumbering, this will always be a 1:1 relationship. + by_region: IndexVec>, + + /// For each deferred local, gets the regions contained within that local at use and drop. + drop_args_by_local: IndexVec>>>, +} + +impl<'tcx> DeferredLocals<'tcx> { + pub(crate) fn defer_local( + &mut self, + infcx: &BorrowckInferCtxt<'tcx>, + universal_regions: &UniversalRegions<'tcx>, + local: Local, + local_ty: Ty<'tcx>, + dropck_kinds: &[GenericArg<'tcx>], + ) { + let tcx = infcx.tcx; + + // We already have drop data for this local, because we need to register + // region constraints eagerly. So, we'll store this so we don't need to + // recompute. + self.drop_args_by_local.insert(local, dropck_kinds.to_vec()); + + // Then, we want to map all the regions contained within this local to + // the local itself. Later, when asked for liveness of a given region, + // we can trace liveness for the local containing it. + let by_region = &mut self.by_region; + tcx.for_each_free_region(&local_ty, |region| { + // See note in [`VarianceExtractor::record_variance`]. + if region.is_bound() || region.is_erased() { + return; + } + let vid = universal_regions.to_region_vid(region); + // Because of MIR renumbering, we should always have a 1:1 mapping + // between a region and a local. + let previous = by_region.insert(vid, local); + debug_assert!( + previous.is_none(), + "{vid:?} is in the type of both {previous:?} and {local:?}, but \ + MIR renumbering should ensure that this is impossible.", + ); + }); + } + + /// For a given region, return the local whose liveness is deferred, and + /// the regions within that local at use and drop. + pub(crate) fn use_deferred_local( + &mut self, + region: RegionVid, + ) -> Option<(Local, Vec>)> { + let local = self.by_region.remove(region)?; + let drop_args = self.drop_args_by_local.remove(local)?; + Some((local, drop_args)) + } +} diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs index 0735aa6120c37..0c7abf42a7631 100644 --- a/compiler/rustc_borrowck/src/polonius/mod.rs +++ b/compiler/rustc_borrowck/src/polonius/mod.rs @@ -36,6 +36,7 @@ mod constraints; mod dump; pub(crate) mod legacy; +mod liveness; mod liveness_constraints; use rustc_data_structures::fx::FxHashSet; @@ -43,16 +44,19 @@ use rustc_index::IndexVec; use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::{Body, Local}; use rustc_middle::ty::RegionVid; -use rustc_mir_dataflow::points::PointIndex; +use rustc_mir_dataflow::move_paths::MoveData; +use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex}; pub(self) use self::constraints::*; pub(crate) use self::dump::dump_polonius_mir; pub(crate) use self::liveness_constraints::record_live_region_variance; -use crate::BorrowSet; use crate::constraints::OutlivesConstraint; use crate::dataflow::BorrowIndex; +pub(crate) use crate::polonius::liveness::DeferredLocals; use crate::region_infer::values::LivenessValues; +use crate::type_check::liveness::{LivenessComputation, LocalUseMap}; use crate::universal_regions::UniversalRegions; +use crate::{BorrowSet, BorrowckInferCtxt}; pub(crate) type LiveRegionVariances = IndexVec>; @@ -84,7 +88,7 @@ impl LiveLoans { /// polonius localized constraints, during NLL region inference as well as MIR dumping, /// - data needed by the borrowck error computation and diagnostics. #[derive(Default)] -pub(crate) struct PoloniusContext { +pub(crate) struct PoloniusContext<'tcx> { /// The graph from which we extract the localized outlives constraints. graph: Option, @@ -97,6 +101,10 @@ pub(crate) struct PoloniusContext { /// currently has more boring locals than NLLs so we record the latter to use in errors and /// diagnostics, to focus on the locals we consider relevant and match NLL diagnostics. pub(crate) boring_nll_locals: FxHashSet, + + pub(crate) deferred_locals_for_liveness: DeferredLocals<'tcx>, + + pub(crate) local_use_map: Option, } /// The direction a constraint can flow into. Used to create liveness constraints according to @@ -113,7 +121,7 @@ pub(crate) enum ConstraintDirection { Bidirectional, } -impl PoloniusContext { +impl<'tcx> PoloniusContext<'tcx> { /// Computes live loans using the set of loans model for `-Zpolonius=next`. /// /// First, creates a constraint graph combining regions and CFG points, by: @@ -124,14 +132,16 @@ impl PoloniusContext { /// loan scope and active loans computations. /// /// The constraint data will be used to compute errors and diagnostics. - pub(crate) fn compute_loan_liveness<'tcx>( + pub(crate) fn compute_loan_liveness( &mut self, + infcx: &BorrowckInferCtxt<'tcx>, liveness: &mut LivenessValues, outlives_constraints: impl Iterator>, universal_regions: &UniversalRegions<'tcx>, body: &Body<'tcx>, + move_data: &MoveData<'tcx>, + location_map: &DenseLocationMap, borrow_set: &BorrowSet<'tcx>, - num_points: usize, ) { // We don't need to prepare the graph (index NLL constraints, etc.) if we have no loans to // trace throughout localized constraints. @@ -139,18 +149,27 @@ impl PoloniusContext { // From the outlives constraints, liveness, and variances, we can compute reachability // on the lazy localized constraint graph to trace the liveness of loans, for the next // step in the chain (the NLL loan scope and active loans computations). - let graph = LocalizedConstraintGraph::new(liveness, outlives_constraints); + let graph = + LocalizedConstraintGraph::new(liveness.location_map(), outlives_constraints); - let mut live_loans = LiveLoans::new(num_points, borrow_set.len()); - let mut visitor = LoanLivenessVisitor { liveness, live_loans: &mut live_loans }; - graph.traverse( - body, + let local_use_map = self + .local_use_map + .as_ref() + .expect("local use map should be computed before loan liveness"); + let deferred_locals_for_liveness = + std::mem::take(&mut self.deferred_locals_for_liveness); + let mut live_loans = LiveLoans::new(location_map.num_points(), borrow_set.len()); + let comp = + LivenessComputation::new(infcx, body, location_map, move_data, &local_use_map); + let mut liveness_source = DeferredLivenessSource { liveness, - &self.live_region_variances, + live_region_variances: &mut self.live_region_variances, universal_regions, - borrow_set, - &mut visitor, - ); + deferred_locals_for_liveness, + comp, + }; + let mut visitor = LoanLivenessVisitor { live_loans: &mut live_loans }; + graph.traverse(body, universal_regions, borrow_set, &mut liveness_source, &mut visitor); liveness.record_live_loans(live_loans); // The graph can be traversed again during MIR dumping, so we store it here. @@ -159,14 +178,43 @@ impl PoloniusContext { } } +struct DeferredLivenessSource<'a, 'tcx> { + liveness: &'a mut LivenessValues, + live_region_variances: &'a mut LiveRegionVariances, + universal_regions: &'a UniversalRegions<'tcx>, + deferred_locals_for_liveness: DeferredLocals<'tcx>, + comp: LivenessComputation<'a, 'tcx>, +} + +impl<'a> LivenessSource<'a> for DeferredLivenessSource<'a, '_> { + fn liveness_for_region(&mut self, region: RegionVid) -> RegionLiveness<'_> { + if let Some((local, drop_args)) = + self.deferred_locals_for_liveness.use_deferred_local(region) + { + self.comp.compute( + local, + self.universal_regions, + Some(&mut self.live_region_variances), + &mut self.liveness, + || &drop_args, + ); + } + + RegionLiveness::new(region, self.live_region_variances, self.liveness.points()) + } + + fn location_map(&self) -> &'a rustc_mir_dataflow::points::DenseLocationMap { + self.comp.location_map + } +} + /// Visitor to record loan liveness when traversing the localized constraint graph. struct LoanLivenessVisitor<'a> { - liveness: &'a LivenessValues, live_loans: &'a mut LiveLoans, } impl LocalizedConstraintGraphVisitor for LoanLivenessVisitor<'_> { - fn on_node_traversed(&mut self, loan: BorrowIndex, node: LocalizedNode) { + fn on_node_traversed(&mut self, loan: BorrowIndex, node: LocalizedNode, is_live: bool) { // Record the loan as being live on entry to this point if it reaches a live region // there. // @@ -208,7 +256,7 @@ impl LocalizedConstraintGraphVisitor for LoanLivenessVisitor<'_> { // // FIXME: analyze potential unsoundness, possibly in concert with a borrowck // implementation in a-mir-formality, fuzzing, or manually crafting counter-examples. - if self.liveness.is_live_at_point(node.region, node.point) { + if is_live { self.live_loans.insert(node.point, loan); } } diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs index 2f03fec0d2245..c7f4e520d3fad 100644 --- a/compiler/rustc_borrowck/src/region_infer/values.rs +++ b/compiler/rustc_borrowck/src/region_infer/values.rs @@ -200,6 +200,10 @@ impl LivenessValues { self.location_map.to_location(point) } + pub(crate) fn location_map(&self) -> &Rc { + &self.location_map + } + /// When using `-Zpolonius=next`, records the given live loans for the loan scopes and active /// loans dataflow computations. pub(crate) fn record_live_loans(&mut self, live_loans: LiveLoans) { diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs index c2fa8ab8637af..bdaee84119497 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs @@ -21,6 +21,9 @@ use crate::universal_regions::UniversalRegions; mod local_use_map; mod trace; +pub(crate) use local_use_map::LocalUseMap; +pub(crate) use trace::LivenessComputation; + /// Combines liveness analysis with initialization analysis to /// determine which variables are live at which points, both due to /// ordinary uses and drops. Returns a set of (ty, location) pairs @@ -42,35 +45,57 @@ pub(super) fn generate<'tcx>( typeck.constraints.liveness_constraints.add_all_points(region); } - let mut free_regions = regions_that_outlive_free_regions( + let free_regions = regions_that_outlive_free_regions( typeck.infcx.num_region_vars(), &typeck.universal_regions, &typeck.constraints.outlives_constraints, ); - // NLLs can avoid computing some liveness data here because its constraints are - // location-insensitive, but that doesn't work in polonius: locals whose type contains a region - // that outlives a free region are not necessarily live everywhere in a flow-sensitive setting, - // unlike NLLs. - // We do record these regions in the polonius context, since they're used to differentiate - // relevant and boring locals, which is a key distinction used later in diagnostics. - // This additional liveness information is ultimately used for *loan* liveness, - // so we don't need to compute it when there are no loans. - // FIXME: this NLL optimization idea, to reduce work to relevant locals only, still makes sense - // for polonius, and should be investigated to improve liveness performance. - if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() - && typeck.borrow_set.len() > 0 - { - let (_, boring_locals) = - compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body); - typeck.polonius_context.as_mut().unwrap().boring_nll_locals = - boring_locals.into_iter().collect(); - free_regions = typeck.universal_regions.universal_regions_iter().collect(); - } let (relevant_live_locals, boring_locals) = compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body); - trace::trace(typeck, location_map, move_data, &relevant_live_locals, &boring_locals); + // Under Polonius Alpha, a larger set of locals are considered relevant: specifically, + // locals containing regions *outliving* universal regions are relevant and only + // locals containing solely universal regions are considered boring. + // + // However, we don't actually need liveness information for *all* these locals, + // only when actually computing loans. So, we can defer computing the liveness + // until we try to compute the loan, which is gated on `LocalizedConstraintGraph` + // traversal. + // + // Potentially in theory, we could defer computing liveness for *all* locals, + // but that's a much bigger refactor (many things rely on liveness of + // NLL-relevant locals). So, we only defer NLL-boring/Polonius-relevant locals + // for now. + let deferred_locals = 'deferred: { + // If we aren't going to be using the additional liveness information, + // don't even bother computing the larger relevant set. + // Similarly, since this liveness information is ultimately used for *loan* + // liveness, we don't need to compute it when there are no loans. + if typeck.polonius_context.is_none() || typeck.borrow_set.len() == 0 { + break 'deferred vec![]; + } + + let free_regions: FxHashSet = + typeck.universal_regions.universal_regions_iter().collect(); + let (polonius_relevant, _) = + compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body); + + let boring: FxHashSet = boring_locals.iter().copied().collect(); + let deferred = + polonius_relevant.into_iter().filter(|local| boring.contains(local)).collect(); + typeck.polonius_context.as_mut().unwrap().boring_nll_locals = boring; + deferred + }; + + trace::trace( + typeck, + location_map, + move_data, + &relevant_live_locals, + &boring_locals, + &deferred_locals, + ); // Mark regions that should be live where they appear within rvalues or within a call: like // args, regions, and types. @@ -155,7 +180,7 @@ fn record_regular_live_regions<'tcx>( tcx: TyCtxt<'tcx>, liveness_constraints: &mut LivenessValues, universal_regions: &UniversalRegions<'tcx>, - polonius_context: &mut Option, + polonius_context: &mut Option>, body: &Body<'tcx>, ) { let mut visitor = @@ -170,7 +195,7 @@ struct LiveVariablesVisitor<'a, 'tcx> { tcx: TyCtxt<'tcx>, liveness_constraints: &'a mut LivenessValues, universal_regions: &'a UniversalRegions<'tcx>, - polonius_context: &'a mut Option, + polonius_context: &'a mut Option>, } impl<'a, 'tcx> Visitor<'tcx> for LiveVariablesVisitor<'a, 'tcx> { diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs index d0302faa513b2..3d2035535062b 100644 --- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs +++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs @@ -18,7 +18,7 @@ use rustc_trait_selection::traits::query::dropck_outlives; use rustc_trait_selection::traits::query::type_op::{DropckOutlives, TypeOpOutput}; use tracing::debug; -use crate::polonius::{LiveRegionVariances, record_live_region_variance}; +use crate::polonius::{DeferredLocals, LiveRegionVariances, record_live_region_variance}; use crate::region_infer::values::LivenessValues; use crate::type_check::liveness::local_use_map::LocalUseMap; use crate::type_check::liveness::make_all_regions_live; @@ -46,10 +46,14 @@ pub(super) fn trace<'tcx>( move_data: &MoveData<'tcx>, relevant_live_locals: &[Local], boring_locals: &[Local], + deferred: &[Local], ) { let _timer = typeck.tcx().prof.generic_activity("borrowck_liveness_trace"); - let local_use_map = LocalUseMap::build(&relevant_live_locals, location_map, typeck.body); + // The use map must also cover the deferred locals: their liveness is computed later, from + // this same map, when the loan liveness traversal first reaches one of their regions. + let use_map_locals: Vec = relevant_live_locals.iter().chain(deferred).copied().collect(); + let local_use_map = LocalUseMap::build(&use_map_locals, location_map, typeck.body); let comp = LivenessComputation::new( typeck.infcx, typeck.body, @@ -60,11 +64,19 @@ pub(super) fn trace<'tcx>( let mut results = LivenessResults::new(typeck, comp); - results.record_legacy_polonius_drop_facts(relevant_live_locals); + let deferred: FxIndexSet = deferred.iter().copied().collect(); + let mut deferred_locals = DeferredLocals::default(); + + results.record_legacy_polonius_drop_facts(relevant_live_locals, &deferred); results.compute_for_all_locals(relevant_live_locals); - results.dropck_boring_locals(boring_locals); + results.dropck_boring_locals(boring_locals, &deferred, &mut deferred_locals); + + if let Some(polonius_context) = &mut typeck.polonius_context { + polonius_context.deferred_locals_for_liveness = deferred_locals; + polonius_context.local_use_map = Some(local_use_map); + } } pub(crate) struct LivenessComputation<'a, 'tcx> { @@ -73,7 +85,7 @@ pub(crate) struct LivenessComputation<'a, 'tcx> { pub(crate) body: &'a Body<'tcx>, /// Defines the `PointIndex` mapping - location_map: &'a DenseLocationMap, + pub(crate) location_map: &'a DenseLocationMap, /// Mapping to/from the various indices used for initialization tracking. move_data: &'a MoveData<'tcx>, @@ -183,19 +195,94 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { /// These are all the locals which do not potentially reference a region local /// to this body. Locals which only reference free regions are always drop-live /// and can therefore safely be dropped. - fn dropck_boring_locals(&mut self, boring_locals: &[Local]) { + fn dropck_boring_locals( + &mut self, + boring_locals: &[Local], + deferred: &FxIndexSet, + deferred_locals: &mut DeferredLocals<'tcx>, + ) { for &local in boring_locals { - let local_ty = self.comp.body.local_decls[local].ty; - let local_span = self.comp.body.local_decls[local].source_info.span; - dropck_local(&self.typeck.infcx, &mut self.drop_data, local_ty, local_span); + self.dropck_boring_local(local, deferred, deferred_locals); } } + fn dropck_boring_local( + &mut self, + local: Local, + deferred: &FxIndexSet, + deferred_locals: &mut DeferredLocals<'tcx>, + ) { + let typeck = &mut *self.typeck; + let local_ty = self.comp.body.local_decls[local].ty; + let local_span = self.comp.body.local_decls[local].source_info.span; + + // If we had treated this as "relevant", we would have run `compute_for_local`. This + // in turn would have skipped calculating dropck *at all* for locals without drop-liveness. + // Calculating drop-liveness is expensive, but we can skip it when we know that there + // are *no* drops (which is relatively cheap). + if deferred.contains(&local) && self.comp.local_use_map.drops(local).next().is_none() { + deferred_locals.defer_local( + typeck.infcx, + typeck.universal_regions, + local, + local_ty, + &[], + ); + return; + } + + // We need to compute dropck for *all* boring locals because we report overflows. + // + // FIXME: there is an argument to be made that we don't need to do this for boring locals + // without drop-liveness, because we skip it for *relevant* locals without drop-liveness. + // But, this is preexisting even on NLL, so leaving it for now. + let drop_data = dropck_local(&typeck.infcx, &mut self.drop_data, local_ty, local_span); + + // We are done with *truly* boring locals. + if !deferred.contains(&local) { + return; + } + + // If this local is deferred and has drop region constraints, we need to register + // them, but *only if the local is drop-live*. + // It doesn't really make sense to only check drop-liveness but defer use-liveness, + // so we just treat this as eager. + if drop_data.region_constraint_data.is_some() { + self.compute_for_local(local); + return; + } + + // The only other thing we need to do *eagerly* for deferred locals is to register + // legacy drop facts (because these facts are on `typeck`). + for &kind in &drop_data.dropck_result.kinds { + polonius::legacy::emit_drop_facts( + typeck.tcx(), + local, + &kind, + typeck.universal_regions, + typeck.polonius_facts, + ); + } + + // Finally, we mark that this local is deferred, including the drop kinds. + deferred_locals.defer_local( + typeck.infcx, + typeck.universal_regions, + local, + local_ty, + &drop_data.dropck_result.kinds, + ); + } + /// Add extra drop facts needed for Polonius Legacy. /// /// Add facts for all locals with free regions, since regions may outlive /// the function body only at certain nodes in the CFG. - fn record_legacy_polonius_drop_facts(&mut self, relevant_live_locals: &[Local]) { + fn record_legacy_polonius_drop_facts( + &mut self, + relevant_live_locals: &[Local], + deferred: &FxIndexSet, + ) { // This is *all wonky* because this used to call a shared // `add_drop_live_facts_for` function that was also used for regular // relevant locals. Presumably, this can be cleaned up quite a bit. @@ -214,7 +301,10 @@ impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> { .iter() .filter_map(|&(local, location_index)| { let local_ty = self.comp.body.local_decls[local].ty; - if relevant_live_locals.contains(&local) || !local_ty.has_free_regions() { + if relevant_live_locals.contains(&local) + || deferred.contains(&local) + || !local_ty.has_free_regions() + { return None; } @@ -299,7 +389,7 @@ impl<'a, 'tcx> LivenessComputation<'a, 'tcx> { } /// Compute for a given local the use- and drop-live points - fn compute<'drop_data>( + pub(crate) fn compute<'drop_data>( &mut self, local: Local, universal_regions: &UniversalRegions<'tcx>, diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 6418c73173df0..eb489bdc3da89 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -247,7 +247,7 @@ struct TypeChecker<'a, 'tcx> { constraints: &'a mut MirTypeckRegionConstraints<'tcx>, deferred_closure_requirements: &'a mut DeferredClosureRequirements<'tcx>, /// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints. - polonius_context: Option, + polonius_context: Option>, } /// Holder struct for passing results from MIR typeck to the rest of the non-lexical regions @@ -258,7 +258,7 @@ pub(crate) struct MirTypeckResults<'tcx> { pub(crate) region_bound_pairs: Frozen>, pub(crate) known_type_outlives_obligations: Frozen>>, pub(crate) deferred_closure_requirements: DeferredClosureRequirements<'tcx>, - pub(crate) polonius_context: Option, + pub(crate) polonius_context: Option>, } /// A collection of region constraints that must be satisfied for the