Auto merge of #135742 - RalfJung:miri-sync, r=RalfJung

Miri subtree update

r? `@ghost`

Author: bors

Diff for: src/tools/miri/.github/workflows/ci.yml

@@ -126,7 +126,7 @@ jobs:
         with:
           fetch-depth: 256 # get a bit more of the history
       - name: install josh-proxy
-        run: RUSTFLAGS="--cap-lints warn" cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r23.12.04
+        run: cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r24.10.04
       - name: setup bot git name and email
         run: |
           git config --global user.name 'The Miri Cronjob Bot'

Diff for: src/tools/miri/CONTRIBUTING.md

@@ -290,7 +290,7 @@ We use the [`josh` proxy](https://github.com/josh-project/josh) to transmit chan
 rustc and Miri repositories. You can install it as follows:
 
 ```sh
-RUSTFLAGS="--cap-lints=warn" cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r23.12.04
+cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r24.10.04
 ```
 
 Josh will automatically be started and stopped by `./miri`.

Diff for: src/tools/miri/README.md

@@ -217,8 +217,8 @@ degree documented below):
 - For every other target with OS `linux`, `macos`, or `windows`, Miri should generally work, but we
   make no promises and we don't run tests for such targets.
 - We have unofficial support (not maintained by the Miri team itself) for some further operating systems.
-  - `solaris` / `illumos`: maintained by @devnexen. Supports `std::{env, thread, sync}`, but not `std::fs`.
-  - `freebsd`: **maintainer wanted**. Supports `std::env` and parts of `std::{thread, fs}`, but not `std::sync`.
+  - `solaris` / `illumos`: maintained by @devnexen. Supports the entire test suite.
+  - `freebsd`: maintained by @YohDeadfall. Supports `std::env` and parts of `std::{thread, fs}`, but not `std::sync`.
   - `android`: **maintainer wanted**. Support very incomplete, but a basic "hello world" works.
   - `wasi`: **maintainer wanted**. Support very incomplete, not even standard output works, but an empty `main` function works.
 - For targets on other operating systems, Miri might fail before even reaching the `main` function.

Diff for: src/tools/miri/ci/ci.sh

@@ -147,13 +147,14 @@ case $HOST_TARGET in
     # Extra tier 2
     TEST_TARGET=arm-unknown-linux-gnueabi run_tests
     TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture of choice
+    # Not officially supported tier 2
+    TEST_TARGET=x86_64-unknown-illumos run_tests
+    TEST_TARGET=x86_64-pc-solaris run_tests
     # Partially supported targets (tier 2)
     BASIC="empty_main integer heap_alloc libc-mem vec string btreemap" # ensures we have the basics: pre-main code, system allocator
     UNIX="hello panic/panic panic/unwind concurrency/simple atomic libc-mem libc-misc libc-random env num_cpus" # the things that are very similar across all Unixes, and hence easily supported there
     TEST_TARGET=x86_64-unknown-freebsd run_tests_minimal $BASIC $UNIX time hashmap random threadname pthread fs libc-pipe
     TEST_TARGET=i686-unknown-freebsd   run_tests_minimal $BASIC $UNIX time hashmap random threadname pthread fs libc-pipe
-    TEST_TARGET=x86_64-unknown-illumos run_tests_minimal $BASIC $UNIX time hashmap random thread sync available-parallelism tls libc-pipe fs
-    TEST_TARGET=x86_64-pc-solaris      run_tests_minimal $BASIC $UNIX time hashmap random thread sync available-parallelism tls libc-pipe fs
     TEST_TARGET=aarch64-linux-android  run_tests_minimal $BASIC $UNIX time hashmap random sync threadname pthread epoll eventfd
     TEST_TARGET=wasm32-wasip2          run_tests_minimal $BASIC wasm
     TEST_TARGET=wasm32-unknown-unknown run_tests_minimal no_std empty_main wasm # this target doesn't really have std

Diff for: src/tools/miri/etc/rust_analyzer_vscode.json

@@ -5,21 +5,19 @@
         "cargo-miri/Cargo.toml",
         "miri-script/Cargo.toml",
     ],
-    "rust-analyzer.check.invocationLocation": "root",
     "rust-analyzer.check.invocationStrategy": "once",
     "rust-analyzer.check.overrideCommand": [
-        "env",
-        "MIRI_AUTO_OPS=no",
         "./miri",
         "clippy", // make this `check` when working with a locally built rustc
         "--message-format=json",
     ],
+    "rust-analyzer.cargo.extraEnv": {
+        "MIRI_AUTO_OPS": "no",
+        "MIRI_IN_RA": "1",
+    },
     // Contrary to what the name suggests, this also affects proc macros.
-    "rust-analyzer.cargo.buildScripts.invocationLocation": "root",
     "rust-analyzer.cargo.buildScripts.invocationStrategy": "once",
     "rust-analyzer.cargo.buildScripts.overrideCommand": [
-        "env",
-        "MIRI_AUTO_OPS=no",
         "./miri",
         "check",
         "--message-format=json",

Diff for: src/tools/miri/miri

@@ -3,12 +3,16 @@ set -e
 # We want to call the binary directly, so we need to know where it ends up.
 ROOT_DIR="$(dirname "$0")"
 MIRI_SCRIPT_TARGET_DIR="$ROOT_DIR"/miri-script/target
-# If stdout is not a terminal and we are not on CI, assume that we are being invoked by RA, and use JSON output.
-if ! [ -t 1 ] && [ -z "$CI" ]; then
+TOOLCHAIN="+nightly"
+# If we are being invoked for RA, use JSON output and the default toolchain (to make proc-macros
+# work in RA). This needs a different target dir to avoid mixing up the builds.
+if [ -n "$MIRI_IN_RA" ]; then
   MESSAGE_FORMAT="--message-format=json"
+  TOOLCHAIN=""
+  MIRI_SCRIPT_TARGET_DIR="$MIRI_SCRIPT_TARGET_DIR"/ra
 fi
 # We need a nightly toolchain, for `-Zroot-dir`.
-cargo +nightly build $CARGO_EXTRA_FLAGS --manifest-path "$ROOT_DIR"/miri-script/Cargo.toml \
+cargo $TOOLCHAIN build $CARGO_EXTRA_FLAGS --manifest-path "$ROOT_DIR"/miri-script/Cargo.toml \
   -Zroot-dir="$ROOT_DIR" \
   -q --target-dir "$MIRI_SCRIPT_TARGET_DIR" $MESSAGE_FORMAT || \
   ( echo "Failed to build miri-script. Is the 'nightly' toolchain installed?"; exit 1 )

Diff for: src/tools/miri/miri-script/src/commands.rs

@@ -423,7 +423,7 @@ impl Command {
                 .map(|path| path.into_os_string().into_string().unwrap())
                 .collect()
         } else {
-            benches.into_iter().map(Into::into).collect()
+            benches.into_iter().collect()
         };
         let target_flag = if let Some(target) = target {
             let mut flag = OsString::from("--target=");
@@ -564,6 +564,10 @@ impl Command {
         if bless {
             e.sh.set_var("RUSTC_BLESS", "Gesundheit");
         }
+        if e.sh.var("MIRI_TEST_TARGET").is_ok() {
+            // Avoid trouble due to an incorrectly set env var.
+            bail!("MIRI_TEST_TARGET must not be set when invoking `./miri test`");
+        }
         if let Some(target) = target {
             // Tell the harness which target to test.
             e.sh.set_var("MIRI_TEST_TARGET", target);

Diff for: src/tools/miri/miri-script/src/main.rs

@@ -111,6 +111,7 @@ pub enum Command {
     /// `rustup-toolchain-install-master` must be installed for this to work.
     Toolchain {
         /// Flags that are passed through to `rustup-toolchain-install-master`.
+        #[arg(trailing_var_arg = true, allow_hyphen_values = true)]
         flags: Vec<String>,
     },
     /// Pull and merge Miri changes from the rustc repo.

Diff for: src/tools/miri/rust-version

@@ -1 +1 @@
-13170cd787cb733ed24842ee825bcbd98dc01476
+01706e1a34c87656fcbfce198608f4cd2ac6461a

Diff for: src/tools/miri/src/concurrency/cpu_affinity.rs

@@ -54,17 +54,17 @@ impl CpuAffinityMask {
                 let chunk = self.0[start..].first_chunk_mut::<4>().unwrap();
                 let offset = cpu % 32;
                 *chunk = match target.options.endian {
-                    Endian::Little => (u32::from_le_bytes(*chunk) | 1 << offset).to_le_bytes(),
-                    Endian::Big => (u32::from_be_bytes(*chunk) | 1 << offset).to_be_bytes(),
+                    Endian::Little => (u32::from_le_bytes(*chunk) | (1 << offset)).to_le_bytes(),
+                    Endian::Big => (u32::from_be_bytes(*chunk) | (1 << offset)).to_be_bytes(),
                 };
             }
             8 => {
                 let start = cpu / 64 * 8; // first byte of the correct u64
                 let chunk = self.0[start..].first_chunk_mut::<8>().unwrap();
                 let offset = cpu % 64;
                 *chunk = match target.options.endian {
-                    Endian::Little => (u64::from_le_bytes(*chunk) | 1 << offset).to_le_bytes(),
-                    Endian::Big => (u64::from_be_bytes(*chunk) | 1 << offset).to_be_bytes(),
+                    Endian::Little => (u64::from_le_bytes(*chunk) | (1 << offset)).to_le_bytes(),
+                    Endian::Big => (u64::from_be_bytes(*chunk) | (1 << offset)).to_be_bytes(),
                 };
             }
             other => bug!("chunk size not supported: {other}"),

Diff for: src/tools/miri/src/concurrency/sync.rs

@@ -422,7 +422,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                     mutex_ref: MutexRef,
                     retval_dest: Option<(Scalar, MPlaceTy<'tcx>)>,
                 }
-                @unblock = |this| {
+                |this, unblock: UnblockKind| {
+                    assert_eq!(unblock, UnblockKind::Ready);
+
                     assert!(!this.mutex_is_locked(&mutex_ref));
                     this.mutex_lock(&mutex_ref);
 
@@ -538,7 +540,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                     retval: Scalar,
                     dest: MPlaceTy<'tcx>,
                 }
-                @unblock = |this| {
+                |this, unblock: UnblockKind| {
+                    assert_eq!(unblock, UnblockKind::Ready);
                     this.rwlock_reader_lock(id);
                     this.write_scalar(retval, &dest)?;
                     interp_ok(())
@@ -623,7 +626,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                     retval: Scalar,
                     dest: MPlaceTy<'tcx>,
                 }
-                @unblock = |this| {
+                |this, unblock: UnblockKind| {
+                    assert_eq!(unblock, UnblockKind::Ready);
                     this.rwlock_writer_lock(id);
                     this.write_scalar(retval, &dest)?;
                     interp_ok(())
@@ -677,25 +681,29 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                     retval_timeout: Scalar,
                     dest: MPlaceTy<'tcx>,
                 }
-                @unblock = |this| {
-                    // The condvar was signaled. Make sure we get the clock for that.
-                    if let Some(data_race) = &this.machine.data_race {
-                        data_race.acquire_clock(
-                            &this.machine.sync.condvars[condvar].clock,
-                            &this.machine.threads,
-                        );
+                |this, unblock: UnblockKind| {
+                    match unblock {
+                        UnblockKind::Ready => {
+                            // The condvar was signaled. Make sure we get the clock for that.
+                            if let Some(data_race) = &this.machine.data_race {
+                                data_race.acquire_clock(
+                                    &this.machine.sync.condvars[condvar].clock,
+                                    &this.machine.threads,
+                                );
+                            }
+                            // Try to acquire the mutex.
+                            // The timeout only applies to the first wait (until the signal), not for mutex acquisition.
+                            this.condvar_reacquire_mutex(&mutex_ref, retval_succ, dest)
+                        }
+                        UnblockKind::TimedOut => {
+                            // We have to remove the waiter from the queue again.
+                            let thread = this.active_thread();
+                            let waiters = &mut this.machine.sync.condvars[condvar].waiters;
+                            waiters.retain(|waiter| *waiter != thread);
+                            // Now get back the lock.
+                            this.condvar_reacquire_mutex(&mutex_ref, retval_timeout, dest)
+                        }
                     }
-                    // Try to acquire the mutex.
-                    // The timeout only applies to the first wait (until the signal), not for mutex acquisition.
-                    this.condvar_reacquire_mutex(&mutex_ref, retval_succ, dest)
-                }
-                @timeout = |this| {
-                    // We have to remove the waiter from the queue again.
-                    let thread = this.active_thread();
-                    let waiters = &mut this.machine.sync.condvars[condvar].waiters;
-                    waiters.retain(|waiter| *waiter != thread);
-                    // Now get back the lock.
-                    this.condvar_reacquire_mutex(&mutex_ref, retval_timeout, dest)
                 }
             ),
         );
@@ -752,25 +760,29 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
                     dest: MPlaceTy<'tcx>,
                     errno_timeout: IoError,
                 }
-                @unblock = |this| {
-                    let futex = futex_ref.0.borrow();
-                    // Acquire the clock of the futex.
-                    if let Some(data_race) = &this.machine.data_race {
-                        data_race.acquire_clock(&futex.clock, &this.machine.threads);
+                |this, unblock: UnblockKind| {
+                    match unblock {
+                        UnblockKind::Ready => {
+                            let futex = futex_ref.0.borrow();
+                            // Acquire the clock of the futex.
+                            if let Some(data_race) = &this.machine.data_race {
+                                data_race.acquire_clock(&futex.clock, &this.machine.threads);
+                            }
+                            // Write the return value.
+                            this.write_scalar(retval_succ, &dest)?;
+                            interp_ok(())
+                        },
+                        UnblockKind::TimedOut => {
+                            // Remove the waiter from the futex.
+                            let thread = this.active_thread();
+                            let mut futex = futex_ref.0.borrow_mut();
+                            futex.waiters.retain(|waiter| waiter.thread != thread);
+                            // Set errno and write return value.
+                            this.set_last_error(errno_timeout)?;
+                            this.write_scalar(retval_timeout, &dest)?;
+                            interp_ok(())
+                        },
                     }
-                    // Write the return value.
-                    this.write_scalar(retval_succ, &dest)?;
-                    interp_ok(())
-                }
-                @timeout = |this| {
-                    // Remove the waiter from the futex.
-                    let thread = this.active_thread();
-                    let mut futex = futex_ref.0.borrow_mut();
-                    futex.waiters.retain(|waiter| waiter.thread != thread);
-                    // Set errno and write return value.
-                    this.set_last_error(errno_timeout)?;
-                    this.write_scalar(retval_timeout, &dest)?;
-                    interp_ok(())
                 }
             ),
         );

Diff for: src/tools/miri/src/concurrency/thread.rs

@@ -19,7 +19,7 @@ use crate::concurrency::data_race;
 use crate::shims::tls;
 use crate::*;
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq)]
 enum SchedulingAction {
     /// Execute step on the active thread.
     ExecuteStep,
@@ -30,6 +30,7 @@ enum SchedulingAction {
 }
 
 /// What to do with TLS allocations from terminated threads
+#[derive(Clone, Copy, Debug, PartialEq)]
 pub enum TlsAllocAction {
     /// Deallocate backing memory of thread-local statics as usual
     Deallocate,
@@ -38,71 +39,18 @@ pub enum TlsAllocAction {
     Leak,
 }
 
-/// Trait for callbacks that are executed when a thread gets unblocked.
-pub trait UnblockCallback<'tcx>: VisitProvenance {
-    /// Will be invoked when the thread was unblocked the "regular" way,
-    /// i.e. whatever event it was blocking on has happened.
-    fn unblock(self: Box<Self>, ecx: &mut InterpCx<'tcx, MiriMachine<'tcx>>) -> InterpResult<'tcx>;
-
-    /// Will be invoked when the timeout ellapsed without the event the
-    /// thread was blocking on having occurred.
-    fn timeout(self: Box<Self>, _ecx: &mut InterpCx<'tcx, MiriMachine<'tcx>>)
-    -> InterpResult<'tcx>;
+/// The argument type for the "unblock" callback, indicating why the thread got unblocked.
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum UnblockKind {
+    /// Operation completed successfully, thread continues normal execution.
+    Ready,
+    /// The operation did not complete within its specified duration.
+    TimedOut,
 }
-pub type DynUnblockCallback<'tcx> = Box<dyn UnblockCallback<'tcx> + 'tcx>;
-
-#[macro_export]
-macro_rules! callback {
-    (
-        @capture<$tcx:lifetime $(,)? $($lft:lifetime),*> { $($name:ident: $type:ty),* $(,)? }
-        @unblock = |$this:ident| $unblock:block
-    ) => {
-        callback!(
-            @capture<$tcx, $($lft),*> { $($name: $type),* }
-            @unblock = |$this| $unblock
-            @timeout = |_this| {
-                unreachable!(
-                    "timeout on a thread that was blocked without a timeout (or someone forgot to overwrite this method)"
-                )
-            }
-        )
-    };
-    (
-        @capture<$tcx:lifetime $(,)? $($lft:lifetime),*> { $($name:ident: $type:ty),* $(,)? }
-        @unblock = |$this:ident| $unblock:block
-        @timeout = |$this_timeout:ident| $timeout:block
-    ) => {{
-        struct Callback<$tcx, $($lft),*> {
-            $($name: $type,)*
-            _phantom: std::marker::PhantomData<&$tcx ()>,
-        }
-
-        impl<$tcx, $($lft),*> VisitProvenance for Callback<$tcx, $($lft),*> {
-            #[allow(unused_variables)]
-            fn visit_provenance(&self, visit: &mut VisitWith<'_>) {
-                $(
-                    self.$name.visit_provenance(visit);
-                )*
-            }
-        }
-
-        impl<$tcx, $($lft),*> UnblockCallback<$tcx> for Callback<$tcx, $($lft),*> {
-            fn unblock(self: Box<Self>, $this: &mut MiriInterpCx<$tcx>) -> InterpResult<$tcx> {
-                #[allow(unused_variables)]
-                let Callback { $($name,)* _phantom } = *self;
-                $unblock
-            }
 
-            fn timeout(self: Box<Self>, $this_timeout: &mut MiriInterpCx<$tcx>) -> InterpResult<$tcx> {
-                #[allow(unused_variables)]
-                let Callback { $($name,)* _phantom } = *self;
-                $timeout
-            }
-        }
-
-        Box::new(Callback { $($name,)* _phantom: std::marker::PhantomData })
-    }}
-}
+/// Type alias for unblock callbacks, i.e. machine callbacks invoked when
+/// a thread gets unblocked.
+pub type DynUnblockCallback<'tcx> = DynMachineCallback<'tcx, UnblockKind>;
 
 /// A thread identifier.
 #[derive(Clone, Copy, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)]
@@ -656,7 +604,8 @@ impl<'tcx> ThreadManager<'tcx> {
                     @capture<'tcx> {
                         joined_thread_id: ThreadId,
                     }
-                    @unblock = |this| {
+                    |this, unblock: UnblockKind| {
+                        assert_eq!(unblock, UnblockKind::Ready);
                         if let Some(data_race) = &mut this.machine.data_race {
                             data_race.thread_joined(&this.machine.threads, joined_thread_id);
                         }
@@ -842,7 +791,7 @@ trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> {
             // 2. Make the scheduler the only place that can change the active
             //    thread.
             let old_thread = this.machine.threads.set_active_thread_id(thread);
-            callback.timeout(this)?;
+            callback.call(this, UnblockKind::TimedOut)?;
             this.machine.threads.set_active_thread_id(old_thread);
         }
         // found_callback can remain None if the computer's clock
@@ -1084,7 +1033,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
         };
         // The callback must be executed in the previously blocked thread.
         let old_thread = this.machine.threads.set_active_thread_id(thread);
-        callback.unblock(this)?;
+        callback.call(this, UnblockKind::Ready)?;
         this.machine.threads.set_active_thread_id(old_thread);
         interp_ok(())
     }