Safe Transmute: Refactor error handling and Answer type

- Create `Answer` type that is not just a type alias of `Result`
- Remove a usage of `map_layouts` to make the code easier to read
- Don't hide errors related to Unknown Layout when computing transmutability

Author: bryangarza

compiler/rustc_trait_selection/src/solve/eval_ctxt.rs

@@ -668,18 +668,16 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         scope: Ty<'tcx>,
         assume: rustc_transmute::Assume,
     ) -> Result<Certainty, NoSolution> {
+        use rustc_transmute::Answer;
         // FIXME(transmutability): This really should be returning nested goals for `Answer::If*`
         match rustc_transmute::TransmuteTypeEnv::new(self.infcx).is_transmutable(
             ObligationCause::dummy(),
             src_and_dst,
             scope,
             assume,
         ) {
-            Ok(None) => Ok(Certainty::Yes),
-            Err(_)
-            | Ok(Some(rustc_transmute::Condition::IfTransmutable { .. }))
-            | Ok(Some(rustc_transmute::Condition::IfAll(_)))
-            | Ok(Some(rustc_transmute::Condition::IfAny(_))) => Err(NoSolution),
+            Answer::Yes => Ok(Certainty::Yes),
+            Answer::No(_) | Answer::If(_) => Err(NoSolution),
         }
     }
 }

compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs

@@ -65,6 +65,11 @@ pub struct ImplCandidate<'tcx> {
     pub similarity: CandidateSimilarity,
 }
 
+enum GetSafeTransmuteErrorAndReason {
+    Silent,
+    Error { err_msg: String, safe_transmute_explanation: String },
+}
+
 pub trait InferCtxtExt<'tcx> {
     /// Given some node representing a fn-like thing in the HIR map,
     /// returns a span and `ArgKind` information that describes the
@@ -724,11 +729,17 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             == self.tcx.lang_items().transmute_trait()
                         {
                             // Recompute the safe transmute reason and use that for the error reporting
-                            self.get_safe_transmute_error_and_reason(
+                            match self.get_safe_transmute_error_and_reason(
                                 obligation.clone(),
                                 trait_ref,
                                 span,
-                            )
+                            ) {
+                                GetSafeTransmuteErrorAndReason::Silent => return,
+                                GetSafeTransmuteErrorAndReason::Error {
+                                    err_msg,
+                                    safe_transmute_explanation,
+                                } => (err_msg, Some(safe_transmute_explanation)),
+                            }
                         } else {
                             (err_msg, None)
                         };
@@ -1292,7 +1303,7 @@ trait InferCtxtPrivExt<'tcx> {
         obligation: PredicateObligation<'tcx>,
         trait_ref: ty::PolyTraitRef<'tcx>,
         span: Span,
-    ) -> (String, Option<String>);
+    ) -> GetSafeTransmuteErrorAndReason;
 
     fn add_tuple_trait_message(
         &self,
@@ -2738,7 +2749,9 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         obligation: PredicateObligation<'tcx>,
         trait_ref: ty::PolyTraitRef<'tcx>,
         span: Span,
-    ) -> (String, Option<String>) {
+    ) -> GetSafeTransmuteErrorAndReason {
+        use rustc_transmute::Answer;
+
         // Erase regions because layout code doesn't particularly care about regions.
         let trait_ref = self.tcx.erase_regions(self.tcx.erase_late_bound_regions(trait_ref));
 
@@ -2758,13 +2771,13 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             scope,
             assume,
         ) {
-            Err(reason) => {
+            Answer::No(reason) => {
                 let dst = trait_ref.substs.type_at(0);
                 let src = trait_ref.substs.type_at(1);
-                let custom_err_msg = format!(
+                let err_msg = format!(
                     "`{src}` cannot be safely transmuted into `{dst}` in the defining scope of `{scope}`"
                 );
-                let reason_msg = match reason {
+                let safe_transmute_explanation = match reason {
                     rustc_transmute::Reason::SrcIsUnspecified => {
                         format!("`{src}` does not have a well-specified layout")
                     }
@@ -2794,11 +2807,21 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     rustc_transmute::Reason::DstIsMoreUnique => {
                         format!("`{src}` is a shared reference, but `{dst}` is a unique reference")
                     }
+                    // Already reported by rustc
+                    rustc_transmute::Reason::TypeError => {
+                        return GetSafeTransmuteErrorAndReason::Silent;
+                    }
+                    rustc_transmute::Reason::SrcLayoutUnknown => {
+                        format!("`{src}` has an unknown layout")
+                    }
+                    rustc_transmute::Reason::DstLayoutUnknown => {
+                        format!("`{dst}` has an unknown layout")
+                    }
                 };
-                (custom_err_msg, Some(reason_msg))
+                GetSafeTransmuteErrorAndReason::Error { err_msg, safe_transmute_explanation }
             }
             // Should never get a Yes at this point! We already ran it before, and did not get a Yes.
-            Ok(None) => span_bug!(
+            Answer::Yes => span_bug!(
                 span,
                 "Inconsistent rustc_transmute::is_transmutable(...) result, got Yes",
             ),

compiler/rustc_trait_selection/src/traits/select/confirmation.rs

@@ -285,28 +285,22 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         &mut self,
         obligation: &TraitObligation<'tcx>,
     ) -> Result<ImplSourceBuiltinData<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
+        use rustc_transmute::{Answer, Condition};
         #[instrument(level = "debug", skip(tcx, obligation, predicate))]
         fn flatten_answer_tree<'tcx>(
             tcx: TyCtxt<'tcx>,
             obligation: &TraitObligation<'tcx>,
             predicate: TraitPredicate<'tcx>,
-            answer: rustc_transmute::Condition<rustc_transmute::layout::rustc::Ref<'tcx>>,
+            cond: Condition<rustc_transmute::layout::rustc::Ref<'tcx>>,
         ) -> Vec<PredicateObligation<'tcx>> {
-            match answer {
+            match cond {
                 // FIXME(bryangarza): Add separate `IfAny` case, instead of treating as `IfAll`
                 // Not possible until the trait solver supports disjunctions of obligations
-                rustc_transmute::Condition::IfAll(answers)
-                | rustc_transmute::Condition::IfAny(answers) => {
-                    let mut nested = vec![];
-                    for flattened in answers
-                        .into_iter()
-                        .map(|answer| flatten_answer_tree(tcx, obligation, predicate, answer))
-                    {
-                        nested.extend(flattened);
-                    }
-                    nested
-                }
-                rustc_transmute::Condition::IfTransmutable { src, dst } => {
+                Condition::IfAll(conds) | Condition::IfAny(conds) => conds
+                    .into_iter()
+                    .flat_map(|cond| flatten_answer_tree(tcx, obligation, predicate, cond))
+                    .collect(),
+                Condition::IfTransmutable { src, dst } => {
                     let trait_def_id = obligation.predicate.def_id();
                     let scope = predicate.trait_ref.substs.type_at(2);
                     let assume_const = predicate.trait_ref.substs.const_at(3);
@@ -333,11 +327,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     // If Dst is mutable, check bidirectionally.
                     // For example, transmuting bool -> u8 is OK as long as you can't update that u8
                     // to be > 1, because you could later transmute the u8 back to a bool and get UB.
-                    let mut obligations = vec![make_obl(src.ty, dst.ty)];
-                    if dst.mutability == Mutability::Mut {
-                        obligations.push(make_obl(dst.ty, src.ty));
+                    match dst.mutability {
+                        Mutability::Not => vec![make_obl(src.ty, dst.ty)],
+                        Mutability::Mut => vec![make_obl(src.ty, dst.ty), make_obl(dst.ty, src.ty)],
                     }
-                    obligations
                 }
             }
         }
@@ -370,9 +363,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         );
 
         let fully_flattened = match maybe_transmutable {
-            Err(_) => Err(Unimplemented)?,
-            Ok(Some(mt)) => flatten_answer_tree(self.tcx(), obligation, predicate, mt),
-            Ok(None) => vec![],
+            Answer::No(_) => Err(Unimplemented)?,
+            Answer::If(cond) => flatten_answer_tree(self.tcx(), obligation, predicate, cond),
+            Answer::Yes => vec![],
         };
 
         debug!(?fully_flattened);

compiler/rustc_transmute/src/layout/tree.rs

@@ -188,14 +188,14 @@ pub(crate) mod rustc {
         /// The layout of the type is unspecified.
         Unspecified,
         /// This error will be surfaced elsewhere by rustc, so don't surface it.
-        Unknown,
+        UnknownLayout,
         TypeError(ErrorGuaranteed),
     }
 
     impl<'tcx> From<LayoutError<'tcx>> for Err {
         fn from(err: LayoutError<'tcx>) -> Self {
             match err {
-                LayoutError::Unknown(..) => Self::Unknown,
+                LayoutError::Unknown(..) => Self::UnknownLayout,
                 err => unimplemented!("{:?}", err),
             }
         }

compiler/rustc_transmute/src/lib.rs

@@ -19,10 +19,16 @@ pub struct Assume {
     pub validity: bool,
 }
 
-/// Either we have an error, or we have an optional Condition that must hold.
-pub type Answer<R> = Result<Option<Condition<R>>, Reason>;
+/// Either we have an error, transmutation is allowed, or we have an optional
+/// Condition that must hold.
+#[derive(Debug, Hash, Eq, PartialEq, Clone)]
+pub enum Answer<R> {
+    Yes,
+    No(Reason),
+    If(Condition<R>),
+}
 
-/// A condition which must hold for safe transmutation to be possible
+/// A condition which must hold for safe transmutation to be possible.
 #[derive(Debug, Hash, Eq, PartialEq, Clone)]
 pub enum Condition<R> {
     /// `Src` is transmutable into `Dst`, if `src` is transmutable into `dst`.
@@ -35,7 +41,7 @@ pub enum Condition<R> {
     IfAny(Vec<Condition<R>>),
 }
 
-/// Answers: Why wasn't the source type transmutable into the destination type?
+/// Answers "why wasn't the source type transmutable into the destination type?"
 #[derive(Debug, Hash, Eq, PartialEq, PartialOrd, Ord, Clone)]
 pub enum Reason {
     /// The layout of the source type is unspecified.
@@ -52,6 +58,12 @@ pub enum Reason {
     DstHasStricterAlignment { src_min_align: usize, dst_min_align: usize },
     /// Can't go from shared pointer to unique pointer
     DstIsMoreUnique,
+    /// Encountered a type error
+    TypeError,
+    /// The layout of src is unknown
+    SrcLayoutUnknown,
+    /// The layout of dst is unknown
+    DstLayoutUnknown,
 }
 
 #[cfg(feature = "rustc")]