Merge branch 'daniel/byte_operation' of https://github.com/danielhumanmod/verify-rust-std into daniel/byte_operation

Author: danielhumanmod

.github/workflows/kani.yml

@@ -1,6 +1,5 @@
-# This workflow is responsible for verifying the standard library with Kani.
-
 name: Kani
+
 on:
   workflow_dispatch:
   pull_request:
@@ -9,30 +8,60 @@ on:
     paths:
       - 'library/**'
       - '.github/workflows/kani.yml'
-      - 'scripts/check_kani.sh'
+      - 'scripts/run-kani.sh'
 
 defaults:
   run:
     shell: bash
 
 jobs:
-  build:
+  check-kani-on-std:
+    name: Verify std library
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        # Kani does not support windows.
         os: [ubuntu-latest, macos-latest]
         include:
           - os: ubuntu-latest
             base: ubuntu
           - os: macos-latest
             base: macos
     steps:
-      - name: Checkout Library
+      # Step 1: Check out the repository
+      - name: Checkout Repository
         uses: actions/checkout@v4
         with:
           path: head
           submodules: true
 
-      - name: Run Kani Script
-        run: bash ./head/scripts/check_kani.sh ${{github.workspace}}/head
+      # Step 2: Run Kani on the std library (default configuration)
+      - name: Run Kani Verification
+        run: head/scripts/run-kani.sh --path ${{github.workspace}}/head
+
+  test-kani-script:
+    name: Test Kani script
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        include:
+          - os: ubuntu-latest
+            base: ubuntu
+          - os: macos-latest
+            base: macos
+    steps:
+      # Step 1: Check out the repository
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          path: head
+          submodules: true
+
+      # Step 2: Test Kani verification script with specific arguments
+      - name: Test Kani script (Custom Args)
+        run: head/scripts/run-kani.sh -p ${{github.workspace}}/head --kani-args --harness ptr --output-format=terse
+
+      # Step 3: Test Kani verification script in the repository directory
+      - name: Test Kani script (In Repo Directory)
+        working-directory: ${{github.workspace}}/head
+        run: scripts/run-kani.sh --kani-args --harness ptr::verify::check_read_u128 --harness ptr --output-format=terse

.gitignore

@@ -19,6 +19,7 @@ Session.vim
 ## Build
 /book/
 /build/
+/kani_build/
 /target
 library/target
 *.rlib

doc/src/SUMMARY.md

@@ -19,6 +19,7 @@
   - [4: Memory safety of BTreeMap's `btree::node` module](./challenges/0004-btree-node.md)
   - [5: Verify functions iterating over inductive data type: `linked_list`](./challenges/0005-linked-list.md)
   - [6: Safety of `NonNull`](./challenges/0006-nonnull.md)
+  - [7: Safety of Methods for Atomic Types and `ReentrantLock`](./challenges/0007-atomic-types.md)
   - [8: Contracts for SmallSort](./challenges/0008-smallsort.md)
   - [9: Safe abstractions for `core::time::Duration`](./challenges/0009-duration.md)
   - [10: Memory safety of String](./challenges/0010-string.md)

doc/src/challenges/0007-atomic-types.md

@@ -0,0 +1,128 @@
+# Challenge 7: Safety of Methods for Atomic Types & Atomic Intrinsics
+
+- **Status:** Open
+- **Tracking Issue:** [#83](https://github.com/model-checking/verify-rust-std/issues/83)
+- **Start date:** *2024-10-30*
+- **End date:** *2024-12-10*
+
+-------------------
+
+## Goal
+
+[`core::sync::atomic`](https://doc.rust-lang.org/std/sync/atomic/index.html) provides methods that operate on atomic types.
+For example, `AtomicBool::store(&self, val: bool, order: Ordering)` stores `val` in the atomic boolean referenced by `self` according to the specified [atomic memory ordering](https://doc.rust-lang.org/std/sync/atomic/enum.Ordering.html).
+
+The goal of this challenge is to verify that these methods are safe.[^1]
+
+### Success Criteria
+
+#### Part 1: Unsafe Methods
+
+First, verify that the unsafe `from_ptr` methods are safe, given that their safety preconditions are met.
+
+Write safety contracts for each of the `from_ptr` methods:
+
+- `AtomicBool::from_ptr`
+- `AtomicPtr::from_ptr`
+- `AtomicI8::from_ptr`
+- `AtomicU8::from_ptr`
+- `AtomicI16::from_ptr`
+- `AtomicU16::from_ptr`
+- `AtomicI32::from_ptr`
+- `AtomicU32::from_ptr`
+- `AtomicI64::from_ptr`
+- `AtomicU64::from_ptr`
+- `AtomicI128::from_ptr`
+- `AtomicU128::from_ptr`
+
+Specifically, encode the conditions about `ptr`'s alignment and validity (marked `#Safety` in the methods' documentation) as preconditions.
+Then, verify that the methods are safe for all possible values for the type that `ptr` points to, given that `ptr` satisfies those preconditions.
+
+For example, `AtomicI8::from_ptr` is defined as:
+```rust
+/// `ptr` must be [valid] ... (comment continues; omitted for brevity)
+pub const unsafe fn from_ptr<'a>(ptr: *mut i8) -> &'a AtomicI8 {
+    // SAFETY: guaranteed by the caller
+    unsafe { &*ptr.cast() }
+}
+```
+
+To verify this method, first encode the safety comments (e.g., about pointer validity) as preconditions, then verify the absence of undefined behavior for all possible `i8` values.
+
+For the `AtomicPtr` case only, we do not require that you verify safety for all possible types.
+Concretely, below is the type signature for `AtomicPtr::from_ptr`:
+
+```rust
+pub const unsafe fn from_ptr<'a>(ptr: *mut *mut T) -> &'a AtomicPtr<T>
+```
+
+The type pointed to is a `*mut T`.
+Verify that for any arbitrary value for this `*mut T` (i.e., any arbitrary memory address), the method is safe.
+However, you need not verify the method for all possible instantiations of `T`.
+It suffices to verify this method for `T` of byte sizes 0, 1, 2, 4, and at least one non-power of two.
+
+#### Part 2: Safe Abstractions
+
+The atomic types expose safe abstractions for atomic operations.
+In this part, you will work toward verifying that these abstractions are indeed safe by writing and verifying safety contracts for the unsafe code in their bodies.
+
+For example, `AtomicBool::store` is the (public) safe method that atomically updates the boolean's value.
+This method invokes the unsafe function `atomic_store`, which in turn calls `intrinsics::atomic_store_relaxed`, `intrinsics::atomic_store_release`, or `intrinsics::atomic_store_seqcst`, depending on the provided ordering.
+
+Write and verify safety contracts for the unsafe functions:
+
+- `atomic_store`
+- `atomic_load`
+- `atomic_swap`
+- `atomic_add`
+- `atomic_sub`
+- `atomic_compare_exchange`
+- `atomic_compare_exchange_weak`
+- `atomic_and`
+- `atomic_nand`
+- `atomic_or`
+- `atomic_xor`
+- `atomic_max`
+- `atomic_umax`
+- `atomic_umin`
+
+Then, for each of the safe abstractions that invoke the unsafe functions listed above, write contracts that ensure that they are not invoked with `order`s that would cause panics.
+
+For example, `atomic_store` panics if invoked with `Acquire` or `AcqRel` ordering.
+In this case, you would write contracts on the safe `store` methods that enforce that they are not called with either of those `order`s.
+
+#### Part 3: Atomic Intrinsics
+
+Write and verify safety contracts for the intrinsics invoked by the unsafe functions from Part 2 (in `core::intrinsics`):
+
+| Operations            |  Functions |
+|-----------------------|-------------|
+| Store                 | `atomic_store_relaxed`, `atomic_store_release`, `atomic_store_seqcst` |
+| Load                  | `atomic_load_relaxed`, `atomic_load_acquire`, `atomic_load_seqcst` |
+| Swap                  | `atomic_xchg_relaxed`, `atomic_xchg_acquire`, `atomic_xchg_release`, `atomic_xchg_acqrel`, `atomic_xchg_seqcst` |
+| Add                   | `atomic_xadd_relaxed`, `atomic_xadd_acquire`, `atomic_xadd_release`, `atomic_xadd_acqrel`, `atomic_xadd_seqcst` |
+| Sub                   | `atomic_xsub_relaxed`, `atomic_xsub_acquire`, `atomic_xsub_release`, `atomic_xsub_acqrel`, `atomic_xsub_seqcst` |
+| Compare Exchange      | `atomic_cxchg_relaxed_relaxed`, `atomic_cxchg_relaxed_acquire`, `atomic_cxchg_relaxed_seqcst`, `atomic_cxchg_acquire_relaxed`, `atomic_cxchg_acquire_acquire`, `atomic_cxchg_acquire_seqcst`, `atomic_cxchg_release_relaxed`, `atomic_cxchg_release_acquire`, `atomic_cxchg_release_seqcst`, `atomic_cxchg_acqrel_relaxed`, `atomic_cxchg_acqrel_acquire`, `atomic_cxchg_acqrel_seqcst`, `atomic_cxchg_seqcst_relaxed`, `atomic_cxchg_seqcst_acquire`, `atomic_cxchg_seqcst_seqcst` |
+| Compare Exchange Weak | `atomic_cxchgweak_relaxed_relaxed`, `atomic_cxchgweak_relaxed_acquire`, `atomic_cxchgweak_relaxed_seqcst`, `atomic_cxchgweak_acquire_relaxed`, `atomic_cxchgweak_acquire_acquire`, `atomic_cxchgweak_acquire_seqcst`, `atomic_cxchgweak_release_relaxed`, `atomic_cxchgweak_release_acquire`, `atomic_cxchgweak_release_seqcst`, `atomic_cxchgweak_acqrel_relaxed`, `atomic_cxchgweak_acqrel_acquire`, `atomic_cxchgweak_acqrel_seqcst`, `atomic_cxchgweak_seqcst_relaxed`, `atomic_cxchgweak_seqcst_acquire`, `atomic_cxchgweak_seqcst_seqcst` |
+| And                   | `atomic_and_relaxed`, `atomic_and_acquire`, `atomic_and_release`, `atomic_and_acqrel`, `atomic_and_seqcst` |
+| Nand                  | `atomic_nand_relaxed`, `atomic_nand_acquire`, `atomic_nand_release`, `atomic_nand_acqrel`, `atomic_nand_seqcst` |
+| Or                    | `atomic_or_seqcst`, `atomic_or_acquire`, `atomic_or_release`, `atomic_or_acqrel`, `atomic_or_relaxed` |
+| Xor                   | `atomic_xor_seqcst`, `atomic_xor_acquire`, `atomic_xor_release`, `atomic_xor_acqrel`, `atomic_xor_relaxed` |
+| Max                   | `atomic_max_relaxed`, `atomic_max_acquire`, `atomic_max_release`, `atomic_max_acqrel`, `atomic_max_seqcst` |
+| Min                   | `atomic_min_relaxed`, `atomic_min_acquire`, `atomic_min_release`, `atomic_min_acqrel`, `atomic_min_seqcst` |
+| Umax                  | `atomic_umax_relaxed`, `atomic_umax_acquire`, `atomic_umax_release`, `atomic_umax_acqrel`, `atomic_umax_seqcst` |
+| Umin                  | `atomic_umin_relaxed`, `atomic_umin_acquire`, `atomic_umin_release`, `atomic_umin_acqrel`, `atomic_umin_seqcst` |
+
+## List of UBs
+
+In addition to any safety properties mentioned in the API documentation, all proofs must automatically ensure the absence of the following [undefined behaviors](https://github.com/rust-lang/reference/blob/142b2ed77d33f37a9973772bd95e6144ed9dce43/src/behavior-considered-undefined.md):
+
+* Data races.
+* Accessing (loading from or storing to) a place that is dangling or based on a misaligned pointer.
+* Reading from uninitialized memory.
+* Invoking undefined behavior via compiler intrinsics.
+* Producing an invalid value.
+
+Note: All solutions to verification challenges need to satisfy the criteria established in the [challenge book](../general-rules.md) in addition to the ones listed above.
+
+[^1]: Throughout this challenge, when we say "safe", it is identical to saying "does not exhibit undefined behavior".

doc/src/general-rules.md

@@ -5,7 +5,9 @@
 **Verification Target:** [Our repository](https://github.com/model-checking/verify-rust-std) is a fork of the original Rust repository,
 and we kept a copy of the Rust standard library inside the `library/` folder that shall be used as the verification target for all our challenges.
 We will periodically update the `library/` folder to track newer versions of the [official Rust standard library](https://github.com/rust-lang/rust/).
-NOTE: This work is not officially affiliated, or endorsed by the Rust project or Rust Foundation.
+
+**NOTE:** This work is not officially affiliated, or endorsed by the Rust project or Rust Foundation.
+
 **Challenges:** Each individual verification effort will have a
 tracking issue where contributors can add comments and ask clarification questions.
 You can find the list of [open challenges here](https://github.com/model-checking/verify-rust-std/labels/Challenge).

doc/src/tools/kani.md

@@ -59,7 +59,8 @@ Create a local copy of the [model-checking fork](https://github.com/model-checki
 `assert`, `assume`, `proof` and [function-contracts](https://github.com/model-checking/kani/blob/main/rfc/src/rfcs/0009-function-contracts.md) such as `modifies`, `requires` and `ensures`) directly.
 
 
-For example, insert this module into an existing file in the core library, like `library/core/src/hint.rs` or `library/core/src/error.rs` in your copy of the library. This is just for the purpose of getting started, so you can insert in any existing file in the core library if you have other preferences.
+For example, insert this module into an existing file in the core library, like `library/core/src/hint.rs` or `library/core/src/error.rs` in your copy of the library.
+This is just for the purpose of getting started, so you can insert it in a different (existing) file in the core library instead.
 
 ``` rust
 #[cfg(kani)]
@@ -84,22 +85,24 @@ pub mod verify {
 }
 ```
 
-### Step 2 - Run the Kani verify-std subcommand
+### Step 2 - Run the Kani script on the std library
 
-To aid the Rust Standard Library verification effort, Kani provides a sub-command out of the box to help you get started.
-Run the following command in your local terminal (Replace "/path/to/library" and "/path/to/target" with your local paths) from the verify repository root:
+To aid the Rust Standard Library verification effort, Kani provides a script out of the box to help you get started.
+Run the following command in your local terminal from the verify repository root:
 
 ```
-kani verify-std -Z unstable-options "/path/to/library" --target-dir "/path/to/target" -Z function-contracts -Z mem-predicates
+./scripts/run-kani.sh --path .
 ```
 
-The command `kani verify-std` is a sub-command of the `kani`. This specific sub-command is used to verify the Rust Standard Library with the following arguments.
+To pass kani arguments such as `--harness`, you can run the script with `--kani-args` and continue passing in all the necessary arguments:
 
-- `"path/to/library"`: This argument specifies the path to the modified Rust Standard Library that was prepared earlier in the script. For example, `./library` or `/home/ubuntu/verify-rust-std/library`
-- `--target-dir "path/to/target"`: This optional argument sets the target directory where Kani will store its output and intermediate files. For example, `/tmp` or `/tmp/verify-std`
+```
+./scripts/run-kani.sh --path . --kani-args --harness alloc::layout::verify::check_array_i32 --output-format=terse
+```
+
+The script `run-kani` installs the right version of Kani for you, builds it and then finally runs the verify-std sub-command of the `kani` with some default flags.
 
-Apart from these, you can use your regular `kani-args` such as `-Z function-contracts`, `-Z stubbing` and `-Z mem-predicates` depending on your verification needs. If you run into a Kani error that says `Use of unstable feature`, add the corresponding feature with `-Z` to the command line.
-For more details on Kani's features, refer to [the features section in the Kani Book](https://model-checking.github.io/kani/reference/attributes.html)
+**NOTE:** This script may crash due to linking issues. If the script fails with an error message related to linking, link the new CBMC version, delete the `./kani_build` directory and re-run.
 
 ### Step 3 - Check verification result
 
@@ -122,7 +125,7 @@ You can specify a specific harness to be verified using the `--harness` flag.
 For example, in your local copy of the verify repo, run the following command.
 
 ```
-kani verify-std --harness harness_introduction -Z unstable-options "./library" --target-dir "/tmp" -Z function-contracts -Z mem-predicates
+./scripts/run-kani.sh --kani-args --harness harness_introduction
 ```
 
 This gives you the verification result for just `harness_introduction` from the aforementioned blob.
@@ -144,13 +147,16 @@ Verification Time: 0.01885804s
 Complete - 1 successfully verified harnesses, 0 failures, 1 total.
 ```
 
-Now you can write proof harnesses to verify specific functions in the library. 
-The current convention is to keep proofs in the same module file of the verification target. 
-To run Kani for an individual proof, use `--harness [harness_function_name]`. 
-Note that Kani will batch run all proofs in the library folder if you do not supply the `--harness` flag. 
-If Kani returns the error `no harnesses matched the harness filter`, you can give the full name of the harness. 
-For example, to run the proof harness named `check_new` in `library/core/src/ptr/unique.rs`, use 
-`--harness ptr::unique::verify::check_new`. To run all proofs in `unique.rs`, use `--harness ptr::unique::verify`. 
+Now you can write proof harnesses to verify specific functions in the library.
+The current convention is to keep proofs in the same module file of the verification target.
+
+To run Kani for an individual proof, use `--harness [harness_function_name]`.
+Note that Kani will batch run all proofs in the library folder if you do not supply the `--harness` flag.
+
+If Kani returns the error `no harnesses matched the harness filter`, you can give the full name of the harness.
+For example, to run the proof harness named `check_new` in `library/core/src/ptr/unique.rs`, use
+`--harness ptr::unique::verify::check_new`. To run all proofs in `unique.rs`, use `--harness ptr::unique::verify`.
+
 To find the full name of a harness, check the Kani output and find the line starting with `Checking harness [harness full name]`.
 
 ## More details

library/contracts/safety/src/kani.rs

@@ -1,6 +1,6 @@
-use proc_macro::{TokenStream};
-use quote::{quote, format_ident};
-use syn::{ItemFn, parse_macro_input};
+use proc_macro::TokenStream;
+use quote::{format_ident, quote};
+use syn::{parse_macro_input, ItemFn, Stmt};
 
 pub(crate) fn requires(attr: TokenStream, item: TokenStream) -> TokenStream {
     rewrite_attr(attr, item, "requires")
@@ -10,12 +10,28 @@ pub(crate) fn ensures(attr: TokenStream, item: TokenStream) -> TokenStream {
     rewrite_attr(attr, item, "ensures")
 }
 
+pub(crate) fn loop_invariant(attr: TokenStream, stmt: TokenStream) -> TokenStream {
+    rewrite_stmt_attr(attr, stmt, "loop_invariant")
+}
+
+fn rewrite_stmt_attr(attr: TokenStream, stmt_stream: TokenStream, name: &str) -> TokenStream {
+    let args = proc_macro2::TokenStream::from(attr);
+    let stmt = parse_macro_input!(stmt_stream as Stmt);
+    let attribute = format_ident!("{}", name);
+    quote!(
+        #[kani_core::#attribute(#args)]
+        #stmt
+    )
+    .into()
+}
+
 fn rewrite_attr(attr: TokenStream, item: TokenStream, name: &str) -> TokenStream {
     let args = proc_macro2::TokenStream::from(attr);
     let fn_item = parse_macro_input!(item as ItemFn);
     let attribute = format_ident!("{}", name);
     quote!(
         #[kani_core::#attribute(#args)]
         #fn_item
-    ).into()
+    )
+    .into()
 }