New datagen API (#3386)

Author: robertbastian

Cargo.lock

@@ -38,6 +38,7 @@ log = { version = "0.4", optional = true }
 
 [dev-dependencies]
 icu_locid = { path = "../../components/locid", features = ["serde"] }
+icu_datagen = { path = "../datagen" }
 
 [features]
 std = ["icu_provider/std"]

provider/blob/Cargo.toml

@@ -57,7 +57,6 @@ impl DataExporter for BlobExporter<'_> {
         locale: &DataLocale,
         payload: &DataPayload<ExportMarker>,
     ) -> Result<(), DataError> {
-        log::trace!("Adding: {}/{}", key, locale);
         let mut serializer = postcard::Serializer {
             output: AllocVec::new(),
         };

provider/blob/src/export/blob_exporter.rs

@@ -2,60 +2,59 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
-//! Data generation for [`BlobDataProvider`](crate::BlobDataProvider) data. See the `icu_datagen` crate.
+//! Data exporter for [`BlobDataProvider`](crate::BlobDataProvider).
+//!
+//! This module can be used as a target for the `icu_datagen` crate.
 //!
 //! # Examples
 //!
 //! ```
-//! use icu_provider::datagen::DataExporter;
-//! use icu_provider::dynutil::*;
+//! use icu_datagen::prelude::*;
+//! use icu_provider_blob::export::*;
+//!
+//! let mut blob: Vec<u8> = Vec::new();
+//!
+//! // Set up the exporter
+//! let mut exporter = BlobExporter::new_with_sink(Box::new(&mut blob));
+//!
+//! // Export something
+//! DatagenProvider::default()
+//!     .export(
+//!         [icu_provider::hello_world::HelloWorldV1Marker::KEY].into_iter().collect(),
+//!         exporter
+//!     ).unwrap();
+//!
+//! // communicate the blob to the client application (network, disk, etc.)
+//! ```
+//!
+//! The resulting blob can now be used like this:
+//!
+//! ```
+//! use icu_locid::langid;
 //! use icu_provider::hello_world::*;
 //! use icu_provider::prelude::*;
-//! use icu_provider_blob::export::BlobExporter;
 //! use icu_provider_blob::BlobDataProvider;
-//! use std::borrow::Cow;
-//! use std::io::Read;
-//! use std::rc::Rc;
 //!
-//! let mut buffer: Vec<u8> = Vec::new();
+//! // obtain the data blob
+//! # let blob = std::fs::read(concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/hello_world.postcard")).unwrap();
 //!
-//! let payload = DataPayload::<HelloWorldV1Marker>::from_owned(HelloWorldV1 {
-//!     message: Cow::Borrowed("Hi"),
-//! });
-//!
-//! // Export something
-//! {
-//!     let mut exporter = BlobExporter::new_with_sink(Box::new(&mut buffer));
-//!     exporter
-//!         .put_payload(
-//!             HelloWorldV1Marker::KEY,
-//!             &Default::default(),
-//!             &UpcastDataPayload::upcast(payload.clone()),
-//!         )
-//!         .expect("Should successfully export");
-//!     exporter
-//!         .close()
-//!         .expect("Should successfully dump to buffer");
-//! }
-//!
-//! // Create a blob provider reading from the buffer
+//! // Create a provider reading from the blob
 //! let provider =
-//!     BlobDataProvider::try_new_from_blob(buffer.into_boxed_slice())
-//!         .expect("Should successfully read from buffer");
-//!
-//! // Read the key from the filesystem and ensure it is as expected
-//! let req = DataRequest {
-//!     locale: Default::default(),
-//!     metadata: Default::default(),
-//! };
+//!     BlobDataProvider::try_new_from_blob(blob.into_boxed_slice())
+//!         .expect("Should successfully read from blob");
+//!
+//! // Read the key from the blob
 //! let response: DataPayload<HelloWorldV1Marker> = provider
 //!     .as_deserializing()
-//!     .load(req)
+//!     .load(DataRequest {
+//!         locale: &langid!("en").into(),
+//!         metadata: Default::default(),
+//!     })
 //!     .unwrap()
 //!     .take_payload()
 //!     .unwrap();
 //!
-//! assert_eq!(response.get(), payload.get(),);
+//! assert_eq!(response.get().message, "Hello World");
 //! ```
 
 mod blob_exporter;

provider/blob/src/export/mod.rs

@@ -50,13 +50,16 @@ icu_timezone = { version = "1.2.0", path = "../../components/timezone", features
 # ICU provider infrastructure
 icu_provider = { version = "1.2.0", path = "../core", features = ["std", "log_error_context", "datagen"]}
 icu_provider_adapters = { version = "1.2.0", path = "../adapters", features = ["datagen"] }
-icu_provider_blob = { version = "1.2.0", path = "../blob", features = ["export"] }
-icu_provider_fs = { version = "1.2.0", path = "../fs", features = ["export"] }
+
+# Exporters
+icu_provider_blob = { version = "1.2.0", path = "../blob", features = ["export"], optional = true }
+icu_provider_fs = { version = "1.2.0", path = "../fs", features = ["export"], optional = true }
+crlify = { version = "1.0.1", path = "../../utils/crlify", optional = true }
+databake = { version = "0.1.3", path = "../../utils/databake", optional = true}
+syn = {version = "1.0", features = ["parsing"], optional = true }
 
 # Other
 cached-path = { version = ">=0.5, <0.7", optional = true }
-crlify = { version = "1.0.1", path = "../../utils/crlify"}
-databake = { version = "0.1.3", path = "../../utils/databake"}
 displaydoc = { version = "0.2.3", default-features = false }
 elsa = "1.7"
 icu_codepointtrie_builder = { version = "0.3.4", path = "../../components/collections/codepointtrie_builder", default-features = false }
@@ -66,13 +69,10 @@ itertools = "0.10"
 lazy_static = "1"
 log = "0.4"
 ndarray = { version = "0.15.5", default-features = false }
-proc-macro2 = "1.0"
-quote = "1.0.9"
 rayon = "1.5"
 serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] }
 serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
 serde-aux = { version = "4.1.2", default-features = false }
-syn = {version = "1.0", features = ["parsing"] }
 tinystr = { version = "0.7.1", path = "../../utils/tinystr", features = ["alloc", "serde", "zerovec"], default-features = false }
 toml = "0.5"
 writeable = { version = "0.5.1", path = "../../utils/writeable" }
@@ -96,7 +96,11 @@ repodata = { path = "../../provider/repodata" }
 dhat = "0.3.0"
 
 [features]
-default = ["bin", "use_wasm", "networking"]
+default = ["bin", "use_wasm", "networking", "legacy_api"]
+provider_baked = ["dep:crlify", "dep:databake", "dep:syn"]
+provider_blob = ["dep:icu_provider_blob"]
+provider_fs = ["dep:icu_provider_fs"]
+legacy_api = ["provider_fs", "provider_blob", "provider_baked"]
 bin = ["dep:clap", "dep:eyre", "dep:simple_logger"]
 # Use wasm for building codepointtries
 use_wasm = ["icu_codepointtrie_builder/wasm"]
@@ -109,13 +113,13 @@ networking = ["dep:cached-path"]
 
 [[bin]]
 name = "icu4x-datagen"
-path = "src/bin/datagen.rs"
+path = "src/bin/datagen/mod.rs"
 required-features = ["bin"]
 
 [[test]]
 name = "icu4x-verify-zero-copy"
 path = "tests/verify-zero-copy.rs"
 
 [package.metadata.cargo-all-features]
-# Disable check-all-features, as the bin feature is purely additive.
-skip_feature_sets = [[]]
+# We don't need working CPT builders for check
+skip_feature_sets = [["use_icu4c"], ["use_wasm"]]