Shell out to crates.io for search

This is a much smaller maintenance burden and fixes a lot of our bugs.
TODO: fix tests

Author: jyn514

src/utils/github_updater.rs

@@ -1,3 +1,4 @@
+use super::APP_USER_AGENT;
 use crate::error::Result;
 use crate::{db::Pool, Config};
 use chrono::{DateTime, Utc};
@@ -13,12 +14,6 @@ use serde::Deserialize;
 use std::collections::HashSet;
 use std::sync::Arc;
 
-const APP_USER_AGENT: &str = concat!(
-    env!("CARGO_PKG_NAME"),
-    " ",
-    include_str!(concat!(env!("OUT_DIR"), "/git_version"))
-);
-
 const GRAPHQL_UPDATE: &str = "query($ids: [ID!]!) {
     nodes(ids: $ids) {
         ... on Repository {

src/utils/mod.rs

@@ -26,3 +26,9 @@ mod queue_builder;
 mod release_activity_updater;
 mod rustc_version;
 pub(crate) mod sized_buffer;
+
+pub(crate) const APP_USER_AGENT: &str = concat!(
+    env!("CARGO_PKG_NAME"),
+    " ",
+    include_str!(concat!(env!("OUT_DIR"), "/git_version"))
+);

src/web/releases.rs

@@ -14,9 +14,10 @@ use iron::{
     modifiers::Redirect,
     status, IronResult, Request, Response, Url,
 };
+use log::debug;
 use postgres::Client;
 use router::Router;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
 /// Number of release in home page
@@ -215,84 +216,84 @@ fn get_releases_by_owner(
 
 /// Get the search results for a crate search query
 ///
-/// Retrieves crates which names have a levenshtein distance of less than or equal to 3,
-/// crates who fit into or otherwise are made up of the query or crates whose descriptions
-/// match the search query.
-///
-/// * `query`: The query string, unfiltered
-/// * `page`: The page of results to show (1-indexed)
-/// * `limit`: The number of results to return
-///
-/// Returns 0 and an empty Vec when no results are found or if a database error occurs
-///
+/// This delegates to the crates.io search API.
 fn get_search_results(
     conn: &mut Client,
-    mut query: &str,
+    query: &str,
     page: i64,
     limit: i64,
-) -> Result<(i64, Vec<Release>), failure::Error> {
-    query = query.trim();
-    if query.is_empty() {
-        return Ok((0, Vec::new()));
+) -> Result<(u64, Vec<Release>), failure::Error> {
+    #[derive(Deserialize)]
+    struct CratesIoReleases {
+        crates: Vec<CratesIoRelease>,
+        meta: CratesIoMeta,
+    }
+    #[derive(Deserialize, Debug)]
+    struct CratesIoRelease {
+        name: String,
+        max_version: String,
+        description: Option<String>,
+        updated_at: DateTime<Utc>,
+    }
+    #[derive(Deserialize)]
+    struct CratesIoMeta {
+        total: u64,
     }
-    let offset = (page - 1) * limit;
-
-    let statement = "
-        SELECT
-            crates.name AS name,
-            releases.version AS version,
-            releases.description AS description,
-            releases.target_name AS target_name,
-            releases.release_time AS release_time,
-            releases.rustdoc_status AS rustdoc_status,
-            github_repos.stars AS github_stars,
-            COUNT(*) OVER() as total
-        FROM crates
-        INNER JOIN (
-            SELECT releases.id, releases.crate_id
-            FROM (
-                SELECT
-                    releases.id,
-                    releases.crate_id,
-                    RANK() OVER (PARTITION BY crate_id ORDER BY release_time DESC) as rank
-                FROM releases
-                WHERE releases.rustdoc_status AND NOT releases.yanked
-            ) AS releases
-            WHERE releases.rank = 1
-        ) AS latest_release ON latest_release.crate_id = crates.id
-        INNER JOIN releases ON latest_release.id = releases.id
-        LEFT JOIN github_repos ON releases.github_repo = github_repos.id
-        WHERE
-            ((char_length($1)::float - levenshtein(crates.name, $1)::float) / char_length($1)::float) >= 0.65
-            OR crates.name ILIKE CONCAT('%', $1, '%')
-        GROUP BY crates.id, releases.id, github_repos.stars
-        ORDER BY
-            levenshtein(crates.name, $1) ASC,
-            crates.name ILIKE CONCAT('%', $1, '%'),
-            releases.downloads DESC
-        LIMIT $2 OFFSET $3";
-
-    let rows = conn.query(statement, &[&query, &limit, &offset])?;
 
-    // Each row contains the total number of possible/valid results, just get it once
-    let total_results = rows
-        .get(0)
-        .map(|row| row.get::<_, i64>("total"))
-        .unwrap_or_default();
-    let packages: Vec<Release> = rows
+    use crate::utils::APP_USER_AGENT;
+    use once_cell::sync::Lazy;
+    use reqwest::blocking::Client as HttpClient;
+    use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, USER_AGENT};
+
+    static HTTP_CLIENT: Lazy<HttpClient> = Lazy::new(|| {
+        let mut headers = HeaderMap::new();
+        headers.insert(USER_AGENT, HeaderValue::from_static(APP_USER_AGENT));
+        headers.insert(ACCEPT, HeaderValue::from_static("application/json"));
+        HttpClient::builder()
+            .default_headers(headers)
+            .build()
+            .unwrap()
+    });
+
+    let url = format!(
+        "https://crates.io/api/v1/crates?page={page}&per_page={limit}&q={query}",
+        page = page,
+        limit = limit,
+        query = query
+    );
+    debug!("fetching search results from {}", url);
+    let releases: CratesIoReleases = HTTP_CLIENT.get(&url).send()?.json()?;
+    let query = conn.prepare(
+        "SELECT github_repos.stars, releases.target_name, releases.rustdoc_status
+        FROM crates INNER JOIN releases ON crates.id = releases.crate_id
+                    LEFT JOIN github_repos ON releases.github_repo = github_repos.id
+        WHERE crates.name = $1 AND releases.version = $2",
+    )?;
+    let crates = releases
+        .crates
         .into_iter()
-        .map(|row| Release {
-            name: row.get("name"),
-            version: row.get("version"),
-            description: row.get("description"),
-            target_name: row.get("target_name"),
-            release_time: row.get("release_time"),
-            rustdoc_status: row.get("rustdoc_status"),
-            stars: row.get::<_, Option<i32>>("github_stars").unwrap_or(0),
+        .flat_map(|krate| {
+            let rows = match conn.query(&query, &[&krate.name, &krate.max_version]) {
+                Err(e) => return Some(Err(e)),
+                Ok(rows) => rows,
+            };
+            debug!("looking up results for {:?}", krate);
+            // crates.io could have a release that hasn't yet been added to the database.
+            // If so, just skip it.
+            let row = rows.get(0)?;
+            let stars: Option<_> = row.get("stars");
+            Some(Result::<_, postgres::Error>::Ok(Release {
+                name: krate.name,
+                version: krate.max_version,
+                description: krate.description,
+                release_time: krate.updated_at,
+                target_name: row.get("target_name"),
+                rustdoc_status: row.get("rustdoc_status"),
+                stars: stars.unwrap_or(0),
+            }))
         })
-        .collect();
-
-    Ok((total_results, packages))
+        .collect::<Result<_, _>>()?;
+    Ok((releases.meta.total, crates))
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]