Split single sitemap into index and sub-sitemaps per starting character

Author: syphar

src/web/routes.rs

@@ -16,7 +16,11 @@ pub(super) fn build_routes() -> Routes {
     //   https://support.google.com/webmasters/answer/183668?hl=en
     routes.static_resource("/robots.txt", PermanentRedirect("/-/static/robots.txt"));
     routes.static_resource("/favicon.ico", PermanentRedirect("/-/static/favicon.ico"));
-    routes.static_resource("/sitemap.xml", super::sitemap::sitemap_handler);
+    routes.static_resource("/sitemap.xml", super::sitemap::sitemapindex_handler);
+    routes.static_resource(
+        "/-/sitemap/:which/sitemap.xml",
+        super::sitemap::sitemap_handler,
+    );
 
     // This should not need to be served from the root as we reference the inner path in links,
     // but clients might have cached the url and need to update it.

src/web/sitemap.rs

@@ -5,9 +5,27 @@ use iron::{
     mime::{Mime, SubLevel, TopLevel},
     IronResult, Request, Response,
 };
+use router::Router;
 use serde::Serialize;
 use serde_json::Value;
 
+/// sitemap index
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+struct SitemapIndexXml {
+    sitemaps: Vec<char>,
+}
+
+impl_webpage! {
+    SitemapIndexXml   = "core/sitemapindex.xml",
+    content_type = ContentType(Mime(TopLevel::Application, SubLevel::Xml, vec![])),
+}
+
+pub fn sitemapindex_handler(req: &mut Request) -> IronResult<Response> {
+    let sitemaps: Vec<char> = (b'a'..=b'z').map(char::from).collect();
+
+    SitemapIndexXml { sitemaps }.into_response(req)
+}
+
 /// The sitemap
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 struct SitemapXml {
@@ -21,16 +39,30 @@ impl_webpage! {
 }
 
 pub fn sitemap_handler(req: &mut Request) -> IronResult<Response> {
+    let router = extension!(req, Router);
+    let which = cexpect!(req, router.find("which")).to_lowercase();
+
     let mut conn = extension!(req, Pool).get()?;
     let query = conn
         .query(
-            "SELECT DISTINCT ON (crates.name)
-                    crates.name,
-                    releases.release_time
+            "SELECT crates.name,
+                    MAX(releases.release_time) as release_time
              FROM crates
              INNER JOIN releases ON releases.crate_id = crates.id
-             WHERE rustdoc_status = true",
-            &[],
+             WHERE 
+                rustdoc_status = true AND 
+                ( 
+                    crates.name like $1 OR 
+                    crates.name like $2
+                )
+             GROUP BY crates.name
+             ",
+            &[
+                // this LIKE pattern has the '%' only at the end,
+                // so postgres can use the index on `name`
+                &format!("{}%", which),
+                &format!("{}%", which.to_uppercase()),
+            ],
         )
         .unwrap();
 
@@ -127,13 +159,15 @@ mod tests {
         wrapper(|env| {
             let web = env.frontend();
             assert_success("/sitemap.xml", web)?;
+            assert_success("/-/sitemap/s/sitemap.xml", web)?;
 
             env.fake_release().name("some_random_crate").create()?;
             env.fake_release()
                 .name("some_random_crate_that_failed")
                 .build_result_successful(false)
                 .create()?;
-            assert_success("/sitemap.xml", web)
+            assert_success("/sitemap.xml", web)?;
+            assert_success("/-/sitemap/s/sitemap.xml", web)
         })
     }
 

templates/core/sitemapindex.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+    {% for which in sitemaps -%}
+        <sitemap>
+            <loc>https://docs.rs/-/sitemap/{{ which }}/sitemap.xml</loc>
+        </sitemap>
+    {%- endfor %}
+</sitemapindex>