Move models into crates_io_database crate (#10597)

... and some `utils` for now until we can separate them properly 😅

Author: Turbo87

Cargo.lock

@@ -132,7 +132,6 @@ tracing = "=0.1.41"
 tracing-subscriber = { version = "=0.3.19", features = ["env-filter", "json"] }
 typomania = { version = "=0.1.2", default-features = false }
 url = "=2.5.4"
-unicode-xid = "=0.2.6"
 utoipa = { version = "=5.3.1", features = ["chrono"] }
 utoipa-axum = "=0.2.0"
 

Cargo.toml

@@ -8,10 +8,28 @@ edition = "2021"
 workspace = true
 
 [dependencies]
-diesel = "=2.2.7"
+bon = "=3.3.2"
+chrono = { version = "=0.4.39", default-features = false, features = ["serde"] }
+crates_io_diesel_helpers = { path = "../crates_io_diesel_helpers" }
+crates_io_index = { path = "../crates_io_index" }
+diesel = { version = "=2.2.7", features = ["serde_json", "chrono", "numeric"] }
+diesel-async = "=0.5.2"
 diesel_full_text_search = "=2.2.0"
+futures-util = "=0.3.31"
+rand = "=0.9.0"
+secrecy = "=0.10.3"
+semver = { version = "=1.0.25", features = ["serde"] }
+serde = { version = "=1.0.217", features = ["derive"] }
+serde_json = "=1.0.138"
+sha2 = "=0.10.8"
+thiserror = "=2.0.11"
+tracing = "=0.1.41"
+unicode-xid = "=0.2.6"
 
 [dev-dependencies]
+claims = "=0.8.0"
 crates_io_test_db = { path = "../crates_io_test_db" }
 diesel-async = { version = "=0.5.2", features = ["postgres"] }
+googletest = "=0.13.0"
+insta = "=1.42.1"
 tokio = { version = "=1.43.0", features = ["macros", "rt"] }

crates/crates_io_database/Cargo.toml

@@ -1,3 +1,5 @@
 #![doc = include_str!("../README.md")]
 
+pub mod models;
 pub mod schema;
+pub mod utils;

crates/crates_io_database/src/lib.rs

@@ -1,15 +1,12 @@
+use crate::models::Crate;
+use crate::schema::*;
 use chrono::NaiveDateTime;
-use diesel::{
-    delete, dsl, insert_into, sql_query, ExpressionMethods, QueryDsl, QueryResult,
-    TextExpressionMethods,
-};
+use diesel::dsl;
+use diesel::prelude::*;
 use diesel_async::scoped_futures::ScopedFutureExt;
 use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl};
 use std::future::Future;
 
-use crate::models::Crate;
-use crate::schema::*;
-
 #[derive(Clone, Identifiable, Queryable, QueryableByName, Debug, Selectable)]
 #[diesel(table_name = categories, check_for_backend(diesel::pg::Pg))]
 pub struct Category {
@@ -21,7 +18,7 @@ pub struct Category {
     pub created_at: NaiveDateTime,
 }
 
-type WithSlug<'a> = diesel::dsl::Eq<categories::slug, crates_io_diesel_helpers::lower<&'a str>>;
+type WithSlug<'a> = dsl::Eq<categories::slug, crates_io_diesel_helpers::lower<&'a str>>;
 
 #[derive(Associations, Insertable, Identifiable, Debug, Clone, Copy)]
 #[diesel(
@@ -73,12 +70,12 @@ impl Category {
                     })
                     .collect::<Vec<_>>();
 
-                delete(crates_categories::table)
+                diesel::delete(crates_categories::table)
                     .filter(crates_categories::crate_id.eq(crate_id))
                     .execute(conn)
                     .await?;
 
-                insert_into(crates_categories::table)
+                diesel::insert_into(crates_categories::table)
                     .values(&crate_categories)
                     .execute(conn)
                     .await?;
@@ -113,7 +110,7 @@ impl Category {
 
         // Collect all the top-level categories and sum up the crates_cnt of
         // the crates in all subcategories
-        sql_query(format!(include_str!("toplevel.sql"), sort_sql))
+        diesel::sql_query(format!(include_str!("toplevel.sql"), sort_sql))
             .bind::<Int8, _>(limit)
             .bind::<Int8, _>(offset)
             .load(conn)
@@ -122,7 +119,7 @@ impl Category {
     pub async fn subcategories(&self, conn: &mut AsyncPgConnection) -> QueryResult<Vec<Category>> {
         use diesel::sql_types::Text;
 
-        sql_query(include_str!("../subcategories.sql"))
+        diesel::sql_query(include_str!("subcategories.sql"))
             .bind::<Text, _>(&self.category)
             .load(conn)
             .await
@@ -138,7 +135,7 @@ impl Category {
     ) -> QueryResult<Vec<Category>> {
         use diesel::sql_types::Text;
 
-        sql_query(include_str!("../parent_categories.sql"))
+        diesel::sql_query(include_str!("parent_categories.sql"))
             .bind::<Text, _>(&self.slug)
             .load(conn)
             .await
@@ -168,7 +165,7 @@ mod tests {
         let test_db = TestDatabase::new();
         let mut conn = test_db.async_connect().await;
 
-        insert_into(categories::table)
+        diesel::insert_into(categories::table)
             .values(&vec![
                 (
                     categories::category.eq("Cat 2"),
@@ -212,7 +209,7 @@ mod tests {
         let test_db = TestDatabase::new();
         let mut conn = test_db.async_connect().await;
 
-        insert_into(categories::table)
+        diesel::insert_into(categories::table)
             .values(&vec![
                 new_cat("Cat 1", "cat1", 0),
                 new_cat("Cat 2", "cat2", 2),
@@ -243,7 +240,7 @@ mod tests {
         let test_db = TestDatabase::new();
         let mut conn = test_db.async_connect().await;
 
-        insert_into(categories::table)
+        diesel::insert_into(categories::table)
             .values(&vec![
                 (
                     categories::category.eq("Cat 1"),
@@ -292,7 +289,7 @@ mod tests {
         let test_db = TestDatabase::new();
         let mut conn = test_db.async_connect().await;
 
-        insert_into(categories::table)
+        diesel::insert_into(categories::table)
             .values(&vec![
                 new_cat("Cat 1", "cat1", 1),
                 new_cat("Cat 1::sub", "cat1::sub", 2),
@@ -334,7 +331,7 @@ mod tests {
         let test_db = TestDatabase::new();
         let mut conn = test_db.async_connect().await;
 
-        insert_into(categories::table)
+        diesel::insert_into(categories::table)
             .values(&vec![
                 new_cat("Cat 1", "cat1", 1),
                 new_cat("Cat 1::sub", "cat1::sub", 2),
@@ -381,7 +378,7 @@ mod tests {
         let test_db = TestDatabase::new();
         let mut conn = test_db.async_connect().await;
 
-        insert_into(categories::table)
+        diesel::insert_into(categories::table)
             .values(&vec![
                 new_cat("Cat 1", "cat1", 1),
                 new_cat("Cat 1::sub1", "cat1::sub1", 2),

src/models/action.rs renamed to crates/crates_io_database/src/models/action.rs

@@ -2,6 +2,7 @@ use crate::schema::{default_versions, versions};
 use crates_io_diesel_helpers::SemverVersion;
 use diesel::prelude::*;
 use diesel_async::{AsyncPgConnection, RunQueryDsl};
+use tracing::{debug, instrument, warn};
 
 /// A subset of the columns of the `versions` table.
 ///
@@ -137,6 +138,7 @@ async fn calculate_default_version(
 mod tests {
     use super::*;
     use crate::schema::crates;
+    use claims::assert_some;
     use crates_io_test_db::TestDatabase;
     use insta::assert_snapshot;
     use std::fmt::Write;

src/models/category.rs renamed to crates/crates_io_database/src/models/category.rs

@@ -1,9 +1,9 @@
+use crate::schema::deleted_crates;
 use bon::Builder;
 use chrono::{DateTime, Utc};
-use crates_io_database::schema::deleted_crates;
 
 /// Struct used to `INSERT` a new `deleted_crates` record into the database.
-#[derive(Insertable, Debug, Builder)]
+#[derive(diesel::Insertable, Debug, Builder)]
 #[diesel(table_name = deleted_crates, check_for_backend(diesel::pg::Pg))]
 pub struct NewDeletedCrate<'a> {
     #[builder(start_fn)]

src/models/crate_owner_invitation.rs renamed to crates/crates_io_database/src/models/crate_owner_invitation.rs

@@ -1,9 +1,9 @@
-use diesel::sql_types::{BigInt, Text};
-
 use crate::models::{Crate, Version};
 use crate::schema::*;
 use crates_io_diesel_helpers::pg_enum;
 use crates_io_index::DependencyKind as IndexDependencyKind;
+use diesel::prelude::*;
+use diesel::sql_types::{BigInt, Text};
 
 #[derive(Identifiable, Associations, Debug, Queryable, QueryableByName, Selectable)]
 #[diesel(

src/models/default_versions.rs renamed to crates/crates_io_database/src/models/default_versions.rs

@@ -2,6 +2,7 @@ use crate::models::Version as FullVersion;
 use crate::schema::{version_downloads, versions};
 use chrono::NaiveDate;
 use crates_io_diesel_helpers::SemverVersion;
+use diesel::prelude::*;
 
 #[derive(Queryable, Identifiable, Associations, Debug, Clone, Copy)]
 #[diesel(

src/models/deleted_crate.rs renamed to crates/crates_io_database/src/models/deleted_crate.rs

@@ -1,6 +1,6 @@
 use bon::Builder;
 use chrono::NaiveDateTime;
-use diesel::{OptionalExtension, QueryResult};
+use diesel::prelude::*;
 use diesel_async::{AsyncPgConnection, RunQueryDsl};
 use secrecy::SecretString;
 

src/models/dependency.rs renamed to crates/crates_io_database/src/models/dependency.rs

@@ -1,5 +1,6 @@
 use crate::models::User;
 use crate::schema::follows;
+use diesel::prelude::*;
 
 #[derive(Insertable, Queryable, Identifiable, Associations, Clone, Copy, Debug)]
 #[diesel(

src/models/download.rs renamed to crates/crates_io_database/src/models/download.rs

@@ -1,11 +1,12 @@
+use diesel::prelude::*;
 use diesel::sql_types::BigInt;
 
 #[derive(QueryableByName, Queryable, Debug)]
 pub struct WithCount<T> {
     #[diesel(embed)]
-    pub(crate) record: T,
+    pub record: T,
     #[diesel(sql_type = BigInt)]
-    pub(crate) total: i64,
+    pub total: i64,
 }
 
 pub trait WithCountExtension<T> {

src/models/email.rs renamed to crates/crates_io_database/src/models/email.rs

@@ -1,4 +1,9 @@
+use crate::models::helpers::with_count::*;
+use crate::models::version::TopVersions;
+use crate::models::{CrateOwner, Owner, OwnerKind, ReverseDependency, User, Version};
+use crate::schema::*;
 use chrono::NaiveDateTime;
+use crates_io_diesel_helpers::canon_crate_name;
 use diesel::associations::Identifiable;
 use diesel::dsl;
 use diesel::pg::Pg;
@@ -8,12 +13,7 @@ use diesel_async::scoped_futures::ScopedFutureExt;
 use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl};
 use secrecy::SecretString;
 use thiserror::Error;
-
-use crate::models::helpers::with_count::*;
-use crate::models::version::TopVersions;
-use crate::models::{CrateOwner, Owner, OwnerKind, ReverseDependency, User, Version};
-use crate::schema::*;
-use crates_io_diesel_helpers::canon_crate_name;
+use tracing::instrument;
 
 use super::Team;
 
@@ -422,7 +422,7 @@ impl Crate {
 
     /// Returns (dependency, dependent crate name, dependent crate downloads)
     #[instrument(skip_all, fields(krate.name = %self.name))]
-    pub(crate) async fn reverse_dependencies(
+    pub async fn reverse_dependencies(
         &self,
         conn: &mut AsyncPgConnection,
         offset: i64,
@@ -546,6 +546,7 @@ pub enum InvalidDependencyName {
 #[cfg(test)]
 mod tests {
     use crate::models::Crate;
+    use claims::{assert_err_eq, assert_ok};
 
     #[test]
     fn validate_crate_name() {

src/models/follow.rs renamed to crates/crates_io_database/src/models/follow.rs

@@ -7,7 +7,7 @@ use crate::schema::{crate_owners, teams};
 
 /// For now, just a Github Team. Can be upgraded to other teams
 /// later if desirable.
-#[derive(Queryable, Identifiable, Serialize, Deserialize, Debug, Selectable)]
+#[derive(Queryable, Identifiable, serde::Serialize, serde::Deserialize, Debug, Selectable)]
 pub struct Team {
     /// Unique table id
     pub id: i32,

src/models/helpers.rs renamed to crates/crates_io_database/src/models/helpers.rs

@@ -10,8 +10,8 @@ use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl};
 pub use self::scopes::{CrateScope, EndpointScope};
 use crate::models::User;
 use crate::schema::api_tokens;
-use crate::util::rfc3339;
-use crate::util::token::{HashedToken, PlainToken};
+use crate::utils::rfc3339;
+use crate::utils::token::{HashedToken, PlainToken};
 
 #[derive(Debug, Insertable, Builder)]
 #[diesel(table_name = api_tokens, check_for_backend(diesel::pg::Pg))]
@@ -39,7 +39,7 @@ impl NewApiToken {
 }
 
 /// The model representing a row in the `api_tokens` database table.
-#[derive(Debug, Identifiable, Queryable, Selectable, Associations, Serialize)]
+#[derive(Debug, Identifiable, Queryable, Selectable, Associations, serde::Serialize)]
 #[diesel(belongs_to(User))]
 pub struct ApiToken {
     pub id: i32,
@@ -99,6 +99,7 @@ impl ApiToken {
 mod tests {
     use super::*;
     use chrono::NaiveDate;
+    use claims::assert_some;
 
     #[test]
     fn api_token_serializes_to_rfc3339() {

src/models/helpers/with_count.rs renamed to crates/crates_io_database/src/models/helpers/with_count.rs

@@ -5,7 +5,7 @@ use diesel::serialize::{self, IsNull, Output, ToSql};
 use diesel::sql_types::Text;
 use std::io::Write;
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, AsExpression, Serialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, diesel::AsExpression, serde::Serialize)]
 #[diesel(sql_type = Text)]
 #[serde(rename_all = "kebab-case")]
 pub enum EndpointScope {
@@ -54,7 +54,7 @@ impl FromSql<Text, Pg> for EndpointScope {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
 #[serde(transparent)]
 pub struct CrateScope {
     pattern: String,
@@ -126,6 +126,7 @@ impl CrateScope {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use claims::assert_ok;
     use googletest::prelude::*;
 
     #[googletest::test]

src/models/keyword.rs renamed to crates/crates_io_database/src/models/keyword.rs

@@ -0,0 +1,2 @@
+pub mod rfc3339;
+pub mod token;