Merge pull request #852 from googlefonts/bulk-compile

Checkpoint: bulk compilation abilities

Cargo.toml

@@ -51,4 +51,5 @@ members = [
     "fea-rs",
     "fea-lsp",
     "otl-normalizer",
+    "fontc_crater",
 ]

README.md

@@ -52,6 +52,12 @@ Once you have them you could try building them:
 cargo run --package fontc -- ../google_fonts_sources/sources/ofl/notosanskayahli/sources/NotoSansKayahLi.designspace
 ```
 
+### Building lots of fonts at once
+
+There is an included `fontc_crater` tool that can download and compile multiple
+fonts at once; this is used for evaluating the compiler. For more information,
+see `fontc_crater/README.md`.
+
 ## Plan
 
 As of 6/4/2023 we intend to:

fea-rs/src/compile/error.rs

@@ -46,19 +46,15 @@ pub enum GlyphOrderError {
 #[derive(Debug, thiserror::Error)]
 #[allow(missing_docs)]
 pub enum CompilerError {
-    #[error("{0}")]
-    SourceLoad(
-        #[from]
-        #[source]
-        SourceLoadError,
-    ),
-    #[error("Parsing failed with {} errors\n{}", .0.messages.len(), .0.display())]
+    #[error(transparent)]
+    SourceLoad(#[from] SourceLoadError),
+    #[error("FEA parsing failed with {} errors", .0.messages.len())]
     ParseFail(DiagnosticSet),
-    #[error("Validation failed with {} errors\n{}", .0.messages.len(), .0.display())]
+    #[error("FEA validation failed with {} errors", .0.messages.len())]
     ValidationFail(DiagnosticSet),
-    #[error("Compilation failed with {} errors\n{}", .0.messages.len(), .0.display())]
+    #[error("FEA compilation failed with {} errors", .0.messages.len())]
     CompilationFail(DiagnosticSet),
-    #[error("{0}")]
+    #[error(transparent)]
     WriteFail(#[from] BuilderError),
 }
 

fontbe/src/error.rs

@@ -22,7 +22,7 @@ use write_fonts::{
 pub enum Error {
     #[error("IO failure")]
     IoError(#[from] io::Error),
-    #[error("Fea compilation failure {0}")]
+    #[error(transparent)]
     FeaCompileError(#[from] CompilerError),
     #[error("'{0}' {1}")]
     GlyphError(GlyphName, GlyphProblem),
@@ -93,6 +93,8 @@ pub enum Error {
     MissingGlyphId(GlyphName),
     #[error("Error making CMap: {0}")]
     CmapConflict(#[from] CmapConflict),
+    #[error("Progress stalled computing composite bbox: {0:?}")]
+    CompositesStalled(Vec<GlyphName>),
 }
 
 #[derive(Debug)]

fontbe/src/glyphs.rs

@@ -824,7 +824,9 @@ fn compute_composite_bboxes(context: &Context) -> Result<(), Error> {
         // Kerplode if we didn't make any progress this spin
         composites.retain(|composite| !bbox_acquired.contains_key(&composite.name));
         if pending == composites.len() {
-            panic!("Unable to make progress on composite bbox, stuck at\n{composites:?}");
+            return Err(Error::CompositesStalled(
+                composites.iter().map(|g| g.name.clone()).collect(),
+            ));
         }
     }
 

fontc/src/args.rs

@@ -118,13 +118,7 @@ impl Args {
         flags
     }
 
-    /// Manually create args for testing
-    #[cfg(test)]
-    pub fn for_test(build_dir: &std::path::Path, source: &str) -> Args {
-        use crate::testdata_dir;
-
-        let input_source = testdata_dir().join(source).canonicalize().unwrap();
-
+    pub fn new(build_dir: &std::path::Path, input_source: PathBuf) -> Args {
         Args {
             glyph_name_filter: None,
             input_source: Some(input_source),
@@ -145,6 +139,15 @@ impl Args {
         }
     }
 
+    /// Manually create args for testing
+    #[cfg(test)]
+    pub fn for_test(build_dir: &std::path::Path, source: &str) -> Args {
+        use crate::testdata_dir;
+
+        let input_source = testdata_dir().join(source).canonicalize().unwrap();
+        Self::new(build_dir, input_source)
+    }
+
     /// The input source to compile.
     pub fn source(&self) -> &Path {
         // safe to unwrap because clap ensures that the input_source is

fontc/src/error.rs

@@ -21,7 +21,7 @@ pub enum Error {
     YamlSerError(#[from] serde_yaml::Error),
     #[error(transparent)]
     TrackFile(#[from] TrackFileError),
-    #[error("Font IR error: '{0}'")]
+    #[error(transparent)]
     FontIrError(#[from] fontir::error::Error),
     #[error(transparent)]
     Backend(#[from] fontbe::error::Error),

fontc/src/lib.rs

@@ -16,8 +16,6 @@ pub use error::Error;
 pub use timing::{create_timer, JobTimer};
 use workload::Workload;
 
-use std::{fs, path::Path};
-
 use fontbe::{
     avar::create_avar_work,
     cmap::create_cmap_work,
@@ -39,17 +37,77 @@ use fontbe::{
     post::create_post_work,
     stat::create_stat_work,
 };
+use std::{
+    fs::{self, OpenOptions},
+    io::BufWriter,
+    path::Path,
+};
 
 use fontdrasil::{coords::NormalizedLocation, types::GlyphName};
-use fontir::{glyph::create_glyph_order_work, source::DeleteWork};
+use fontir::{
+    glyph::create_glyph_order_work,
+    orchestration::{Context as FeContext, Flags},
+    source::DeleteWork,
+};
 
 use fontbe::orchestration::Context as BeContext;
 use fontbe::paths::Paths as BePaths;
 use fontir::paths::Paths as IrPaths;
 
 use log::{debug, warn};
 
+/// Run the compiler with the provided arguments
+pub fn run(args: Args, mut timer: JobTimer) -> Result<(), Error> {
+    let time = create_timer(AnyWorkId::InternalTiming("Init config"), 0)
+        .queued()
+        .run();
+    let (ir_paths, be_paths) = init_paths(&args)?;
+    let config = Config::new(args)?;
+    let prev_inputs = config.init()?;
+    timer.add(time.complete());
+
+    let mut change_detector = ChangeDetector::new(
+        config.clone(),
+        ir_paths.clone(),
+        be_paths.clone(),
+        prev_inputs,
+        &mut timer,
+    )?;
+
+    let workload = create_workload(&mut change_detector, timer)?;
+
+    let fe_root = FeContext::new_root(
+        config.args.flags(),
+        ir_paths,
+        workload.current_inputs().clone(),
+    );
+    let be_root = BeContext::new_root(config.args.flags(), be_paths, &fe_root);
+    let mut timing = workload.exec(&fe_root, &be_root)?;
+
+    if config.args.flags().contains(Flags::EMIT_TIMING) {
+        let path = config.args.build_dir.join("threads.svg");
+        let out_file = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(&path)
+            .map_err(|source| Error::FileIo {
+                path: path.clone(),
+                source,
+            })?;
+        let mut buf = BufWriter::new(out_file);
+        timing
+            .write_svg(&mut buf)
+            .map_err(|source| Error::FileIo { path, source })?;
+    }
+
+    change_detector.finish_successfully()?;
+
+    write_font_file(&config.args, &be_root)
+}
+
 pub fn require_dir(dir: &Path) -> Result<(), Error> {
+    // skip empty paths
     if dir == Path::new("") {
         return Ok(());
     }

fontc/src/main.rs

@@ -1,16 +1,9 @@
-use std::{
-    fs::OpenOptions,
-    io::{BufWriter, Write},
-    time::Instant,
-};
+use std::{io::Write, time::Instant};
 
 use clap::Parser;
 
-use fontbe::orchestration::{AnyWorkId, Context as BeContext};
-use fontc::{
-    create_timer, init_paths, write_font_file, Args, ChangeDetector, Config, Error, JobTimer,
-};
-use fontir::orchestration::{Context as FeContext, Flags};
+use fontbe::orchestration::AnyWorkId;
+use fontc::{create_timer, Args, Error, JobTimer};
 
 fn main() {
     // catch and print errors manually, to avoid just seeing the Debug impls
@@ -48,52 +41,7 @@ fn run() -> Result<(), Error> {
         .init();
     timer.add(time.complete());
 
-    let time = create_timer(AnyWorkId::InternalTiming("Init config"), 0)
-        .queued()
-        .run();
-    let (ir_paths, be_paths) = init_paths(&args)?;
-    let config = Config::new(args)?;
-    let prev_inputs = config.init()?;
-    timer.add(time.complete());
-
-    let mut change_detector = ChangeDetector::new(
-        config.clone(),
-        ir_paths.clone(),
-        be_paths.clone(),
-        prev_inputs,
-        &mut timer,
-    )?;
-
-    let workload = fontc::create_workload(&mut change_detector, timer)?;
-
-    let fe_root = FeContext::new_root(
-        config.args.flags(),
-        ir_paths,
-        workload.current_inputs().clone(),
-    );
-    let be_root = BeContext::new_root(config.args.flags(), be_paths, &fe_root);
-    let mut timing = workload.exec(&fe_root, &be_root)?;
-
-    if config.args.flags().contains(Flags::EMIT_TIMING) {
-        let path = config.args.build_dir.join("threads.svg");
-        let out_file = OpenOptions::new()
-            .write(true)
-            .create(true)
-            .truncate(true)
-            .open(&path)
-            .map_err(|source| Error::FileIo {
-                path: path.clone(),
-                source,
-            })?;
-        let mut buf = BufWriter::new(out_file);
-        timing
-            .write_svg(&mut buf)
-            .map_err(|source| Error::FileIo { path, source })?;
-    }
-
-    change_detector.finish_successfully()?;
-
-    write_font_file(&config.args, &be_root)
+    fontc::run(args, timer)
 }
 
 fn print_verbose_version() -> Result<(), std::io::Error> {

fontc_crater/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "fontc_crater"
+version = "0.1.0"
+edition = "2021"
+license = "MIT/Apache-2.0"
+description = "A compiler for fonts."
+repository = "https://github.com/googlefonts/fontc"
+readme = "README.md"
+
+[dependencies]
+fontc = { version = "0.0.1", path = "../fontc" }
+
+google-fonts-sources = "0.1.0"
+write-fonts.workspace = true
+serde.workspace = true
+thiserror.workspace = true
+clap.workspace = true
+serde_yaml.workspace = true
+serde_json.workspace = true
+tempfile.workspace = true
+env_logger.workspace = true
+

fontc_crater/README.md

@@ -0,0 +1,32 @@
+# `fontc_crater`
+
+The `fontc_crater` crate (named after [rust-lang/crater]) is a tool for
+performing compilation and related actions across a large number of source
+fonts.
+
+
+
+```sh
+$ cargo run --release --p=fontc_crater -- compile FONT_CACHE --fonts-repo GOOGLE/FONTS  -o results.json
+```
+
+This is a binary for executing font compilation (and possibly other tasks) in
+bulk.
+
+Discovery of font sources is managed by a separate tool,
+[google-fonts-sources][]; this tool checks out the
+[github.com/google/fonts][google/fonts] repository and looks for fonts with
+known source repos. You can use the `--fonts-repo` argument to pass the path to
+an existing checkout of this repository, which saves time.
+
+Once sources are identified, they are checked out into the `FONT_CACHE`
+directory, where they can be reused between runs.
+
+## output
+
+For detailed output, use the `-o/--out` flag to specify a path where we should
+dump a json dictionary containing the outcome for each source.
+
+[google-fonts-sources]: https://github.com/googlefonts/google-fonts-sources
+[google/fonts]: https://github.com/google/fonts
+[rust-lang/crater]: https://github.com/rust-lang/crater

fontc_crater/src/args.rs

@@ -0,0 +1,30 @@
+//! CLI args
+
+use std::path::PathBuf;
+
+use clap::{Parser, ValueEnum};
+
+#[derive(Debug, Clone, PartialEq, Parser)]
+#[command(about = "compile multiple fonts and report the results")]
+pub(super) struct Args {
+    /// The task to perform with each font
+    pub(super) command: Tasks,
+    /// Directory to store font sources.
+    ///
+    /// Reusing this directory saves us having to clone all the repos on each run.
+    ///
+    /// This directory is also used to write cached results during repo discovery.
+    pub(super) font_cache: PathBuf,
+    /// Path to local checkout of google/fonts repository
+    #[arg(short, long)]
+    pub(super) fonts_repo: Option<PathBuf>,
+    /// Optional path to write out results (as json)
+    #[arg(short = 'o', long = "out")]
+    pub(super) out_path: Option<PathBuf>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, ValueEnum)]
+pub(super) enum Tasks {
+    Compile,
+    // this will expand to include at least 'ttx_diff'
+}

fontc_crater/src/error.rs

@@ -0,0 +1,21 @@
+use std::path::PathBuf;
+
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub(super) enum Error {
+    #[error("Failed to load input file: '{0}'")]
+    InputFile(std::io::Error),
+    #[error("Failed to write file '{path}': '{error}'")]
+    WriteFile {
+        path: PathBuf,
+        #[source]
+        error: std::io::Error,
+    },
+    #[error("Failed to parse input json: '{0}'")]
+    InputJson(#[source] serde_json::Error),
+    #[error("Failed to encode json: '{0}'")]
+    OutputJson(#[source] serde_json::Error),
+    #[error("Failed to create cache directory: '{0}'")]
+    CacheDir(std::io::Error),
+}