diff --git a/Cargo.toml b/Cargo.toml index 03619c1d8..bdcd3dbae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,4 +51,5 @@ members = [ "fea-rs", "fea-lsp", "otl-normalizer", + "fontc_crater", ] diff --git a/README.md b/README.md index 31d92b69a..44d432924 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,12 @@ Once you have them you could try building them: cargo run --package fontc -- ../google_fonts_sources/sources/ofl/notosanskayahli/sources/NotoSansKayahLi.designspace ``` +### Building lots of fonts at once + +There is an included `fontc_crater` tool that can download and compile multiple +fonts at once; this is used for evaluating the compiler. For more information, +see `fontc_crater/README.md`. + ## Plan As of 6/4/2023 we intend to: diff --git a/fea-rs/src/compile/error.rs b/fea-rs/src/compile/error.rs index 77cc118c8..57761eacb 100644 --- a/fea-rs/src/compile/error.rs +++ b/fea-rs/src/compile/error.rs @@ -46,19 +46,15 @@ pub enum GlyphOrderError { #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum CompilerError { - #[error("{0}")] - SourceLoad( - #[from] - #[source] - SourceLoadError, - ), - #[error("Parsing failed with {} errors\n{}", .0.messages.len(), .0.display())] + #[error(transparent)] + SourceLoad(#[from] SourceLoadError), + #[error("FEA parsing failed with {} errors", .0.messages.len())] ParseFail(DiagnosticSet), - #[error("Validation failed with {} errors\n{}", .0.messages.len(), .0.display())] + #[error("FEA validation failed with {} errors", .0.messages.len())] ValidationFail(DiagnosticSet), - #[error("Compilation failed with {} errors\n{}", .0.messages.len(), .0.display())] + #[error("FEA compilation failed with {} errors", .0.messages.len())] CompilationFail(DiagnosticSet), - #[error("{0}")] + #[error(transparent)] WriteFail(#[from] BuilderError), } diff --git a/fontbe/src/error.rs b/fontbe/src/error.rs index 0e34b691e..6a9793843 100644 --- a/fontbe/src/error.rs +++ b/fontbe/src/error.rs @@ -22,7 +22,7 @@ use write_fonts::{ pub enum Error { #[error("IO failure")] IoError(#[from] io::Error), - #[error("Fea compilation failure {0}")] + #[error(transparent)] FeaCompileError(#[from] CompilerError), #[error("'{0}' {1}")] GlyphError(GlyphName, GlyphProblem), @@ -93,6 +93,8 @@ pub enum Error { MissingGlyphId(GlyphName), #[error("Error making CMap: {0}")] CmapConflict(#[from] CmapConflict), + #[error("Progress stalled computing composite bbox: {0:?}")] + CompositesStalled(Vec), } #[derive(Debug)] diff --git a/fontbe/src/glyphs.rs b/fontbe/src/glyphs.rs index 34b61f271..e98472359 100644 --- a/fontbe/src/glyphs.rs +++ b/fontbe/src/glyphs.rs @@ -824,7 +824,9 @@ fn compute_composite_bboxes(context: &Context) -> Result<(), Error> { // Kerplode if we didn't make any progress this spin composites.retain(|composite| !bbox_acquired.contains_key(&composite.name)); if pending == composites.len() { - panic!("Unable to make progress on composite bbox, stuck at\n{composites:?}"); + return Err(Error::CompositesStalled( + composites.iter().map(|g| g.name.clone()).collect(), + )); } } diff --git a/fontc/src/args.rs b/fontc/src/args.rs index 514e52e5c..833411f1a 100644 --- a/fontc/src/args.rs +++ b/fontc/src/args.rs @@ -118,13 +118,7 @@ impl Args { flags } - /// Manually create args for testing - #[cfg(test)] - pub fn for_test(build_dir: &std::path::Path, source: &str) -> Args { - use crate::testdata_dir; - - let input_source = testdata_dir().join(source).canonicalize().unwrap(); - + pub fn new(build_dir: &std::path::Path, input_source: PathBuf) -> Args { Args { glyph_name_filter: None, input_source: Some(input_source), @@ -145,6 +139,15 @@ impl Args { } } + /// Manually create args for testing + #[cfg(test)] + pub fn for_test(build_dir: &std::path::Path, source: &str) -> Args { + use crate::testdata_dir; + + let input_source = testdata_dir().join(source).canonicalize().unwrap(); + Self::new(build_dir, input_source) + } + /// The input source to compile. pub fn source(&self) -> &Path { // safe to unwrap because clap ensures that the input_source is diff --git a/fontc/src/error.rs b/fontc/src/error.rs index 8fcc34726..6adb6a4e2 100644 --- a/fontc/src/error.rs +++ b/fontc/src/error.rs @@ -21,7 +21,7 @@ pub enum Error { YamlSerError(#[from] serde_yaml::Error), #[error(transparent)] TrackFile(#[from] TrackFileError), - #[error("Font IR error: '{0}'")] + #[error(transparent)] FontIrError(#[from] fontir::error::Error), #[error(transparent)] Backend(#[from] fontbe::error::Error), diff --git a/fontc/src/lib.rs b/fontc/src/lib.rs index 7c0f7997a..0007f8c98 100644 --- a/fontc/src/lib.rs +++ b/fontc/src/lib.rs @@ -16,8 +16,6 @@ pub use error::Error; pub use timing::{create_timer, JobTimer}; use workload::Workload; -use std::{fs, path::Path}; - use fontbe::{ avar::create_avar_work, cmap::create_cmap_work, @@ -39,9 +37,18 @@ use fontbe::{ post::create_post_work, stat::create_stat_work, }; +use std::{ + fs::{self, OpenOptions}, + io::BufWriter, + path::Path, +}; use fontdrasil::{coords::NormalizedLocation, types::GlyphName}; -use fontir::{glyph::create_glyph_order_work, source::DeleteWork}; +use fontir::{ + glyph::create_glyph_order_work, + orchestration::{Context as FeContext, Flags}, + source::DeleteWork, +}; use fontbe::orchestration::Context as BeContext; use fontbe::paths::Paths as BePaths; @@ -49,7 +56,58 @@ use fontir::paths::Paths as IrPaths; use log::{debug, warn}; +/// Run the compiler with the provided arguments +pub fn run(args: Args, mut timer: JobTimer) -> Result<(), Error> { + let time = create_timer(AnyWorkId::InternalTiming("Init config"), 0) + .queued() + .run(); + let (ir_paths, be_paths) = init_paths(&args)?; + let config = Config::new(args)?; + let prev_inputs = config.init()?; + timer.add(time.complete()); + + let mut change_detector = ChangeDetector::new( + config.clone(), + ir_paths.clone(), + be_paths.clone(), + prev_inputs, + &mut timer, + )?; + + let workload = create_workload(&mut change_detector, timer)?; + + let fe_root = FeContext::new_root( + config.args.flags(), + ir_paths, + workload.current_inputs().clone(), + ); + let be_root = BeContext::new_root(config.args.flags(), be_paths, &fe_root); + let mut timing = workload.exec(&fe_root, &be_root)?; + + if config.args.flags().contains(Flags::EMIT_TIMING) { + let path = config.args.build_dir.join("threads.svg"); + let out_file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&path) + .map_err(|source| Error::FileIo { + path: path.clone(), + source, + })?; + let mut buf = BufWriter::new(out_file); + timing + .write_svg(&mut buf) + .map_err(|source| Error::FileIo { path, source })?; + } + + change_detector.finish_successfully()?; + + write_font_file(&config.args, &be_root) +} + pub fn require_dir(dir: &Path) -> Result<(), Error> { + // skip empty paths if dir == Path::new("") { return Ok(()); } diff --git a/fontc/src/main.rs b/fontc/src/main.rs index cc65e3257..776e64850 100644 --- a/fontc/src/main.rs +++ b/fontc/src/main.rs @@ -1,16 +1,9 @@ -use std::{ - fs::OpenOptions, - io::{BufWriter, Write}, - time::Instant, -}; +use std::{io::Write, time::Instant}; use clap::Parser; -use fontbe::orchestration::{AnyWorkId, Context as BeContext}; -use fontc::{ - create_timer, init_paths, write_font_file, Args, ChangeDetector, Config, Error, JobTimer, -}; -use fontir::orchestration::{Context as FeContext, Flags}; +use fontbe::orchestration::AnyWorkId; +use fontc::{create_timer, Args, Error, JobTimer}; fn main() { // catch and print errors manually, to avoid just seeing the Debug impls @@ -48,52 +41,7 @@ fn run() -> Result<(), Error> { .init(); timer.add(time.complete()); - let time = create_timer(AnyWorkId::InternalTiming("Init config"), 0) - .queued() - .run(); - let (ir_paths, be_paths) = init_paths(&args)?; - let config = Config::new(args)?; - let prev_inputs = config.init()?; - timer.add(time.complete()); - - let mut change_detector = ChangeDetector::new( - config.clone(), - ir_paths.clone(), - be_paths.clone(), - prev_inputs, - &mut timer, - )?; - - let workload = fontc::create_workload(&mut change_detector, timer)?; - - let fe_root = FeContext::new_root( - config.args.flags(), - ir_paths, - workload.current_inputs().clone(), - ); - let be_root = BeContext::new_root(config.args.flags(), be_paths, &fe_root); - let mut timing = workload.exec(&fe_root, &be_root)?; - - if config.args.flags().contains(Flags::EMIT_TIMING) { - let path = config.args.build_dir.join("threads.svg"); - let out_file = OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(&path) - .map_err(|source| Error::FileIo { - path: path.clone(), - source, - })?; - let mut buf = BufWriter::new(out_file); - timing - .write_svg(&mut buf) - .map_err(|source| Error::FileIo { path, source })?; - } - - change_detector.finish_successfully()?; - - write_font_file(&config.args, &be_root) + fontc::run(args, timer) } fn print_verbose_version() -> Result<(), std::io::Error> { diff --git a/fontc_crater/Cargo.toml b/fontc_crater/Cargo.toml new file mode 100644 index 000000000..728905a5d --- /dev/null +++ b/fontc_crater/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "fontc_crater" +version = "0.1.0" +edition = "2021" +license = "MIT/Apache-2.0" +description = "A compiler for fonts." +repository = "https://github.com/googlefonts/fontc" +readme = "README.md" + +[dependencies] +fontc = { version = "0.0.1", path = "../fontc" } + +google-fonts-sources = "0.1.0" +write-fonts.workspace = true +serde.workspace = true +thiserror.workspace = true +clap.workspace = true +serde_yaml.workspace = true +serde_json.workspace = true +tempfile.workspace = true +env_logger.workspace = true + diff --git a/fontc_crater/README.md b/fontc_crater/README.md new file mode 100644 index 000000000..928471fea --- /dev/null +++ b/fontc_crater/README.md @@ -0,0 +1,32 @@ +# `fontc_crater` + +The `fontc_crater` crate (named after [rust-lang/crater]) is a tool for +performing compilation and related actions across a large number of source +fonts. + + + +```sh +$ cargo run --release --p=fontc_crater -- compile FONT_CACHE --fonts-repo GOOGLE/FONTS -o results.json +``` + +This is a binary for executing font compilation (and possibly other tasks) in +bulk. + +Discovery of font sources is managed by a separate tool, +[google-fonts-sources][]; this tool checks out the +[github.com/google/fonts][google/fonts] repository and looks for fonts with +known source repos. You can use the `--fonts-repo` argument to pass the path to +an existing checkout of this repository, which saves time. + +Once sources are identified, they are checked out into the `FONT_CACHE` +directory, where they can be reused between runs. + +## output + +For detailed output, use the `-o/--out` flag to specify a path where we should +dump a json dictionary containing the outcome for each source. + +[google-fonts-sources]: https://github.com/googlefonts/google-fonts-sources +[google/fonts]: https://github.com/google/fonts +[rust-lang/crater]: https://github.com/rust-lang/crater diff --git a/fontc_crater/src/args.rs b/fontc_crater/src/args.rs new file mode 100644 index 000000000..ae55f5bd7 --- /dev/null +++ b/fontc_crater/src/args.rs @@ -0,0 +1,30 @@ +//! CLI args + +use std::path::PathBuf; + +use clap::{Parser, ValueEnum}; + +#[derive(Debug, Clone, PartialEq, Parser)] +#[command(about = "compile multiple fonts and report the results")] +pub(super) struct Args { + /// The task to perform with each font + pub(super) command: Tasks, + /// Directory to store font sources. + /// + /// Reusing this directory saves us having to clone all the repos on each run. + /// + /// This directory is also used to write cached results during repo discovery. + pub(super) font_cache: PathBuf, + /// Path to local checkout of google/fonts repository + #[arg(short, long)] + pub(super) fonts_repo: Option, + /// Optional path to write out results (as json) + #[arg(short = 'o', long = "out")] + pub(super) out_path: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, ValueEnum)] +pub(super) enum Tasks { + Compile, + // this will expand to include at least 'ttx_diff' +} diff --git a/fontc_crater/src/error.rs b/fontc_crater/src/error.rs new file mode 100644 index 000000000..9cc372718 --- /dev/null +++ b/fontc_crater/src/error.rs @@ -0,0 +1,21 @@ +use std::path::PathBuf; + +use thiserror::Error; + +#[derive(Debug, Error)] +pub(super) enum Error { + #[error("Failed to load input file: '{0}'")] + InputFile(std::io::Error), + #[error("Failed to write file '{path}': '{error}'")] + WriteFile { + path: PathBuf, + #[source] + error: std::io::Error, + }, + #[error("Failed to parse input json: '{0}'")] + InputJson(#[source] serde_json::Error), + #[error("Failed to encode json: '{0}'")] + OutputJson(#[source] serde_json::Error), + #[error("Failed to create cache directory: '{0}'")] + CacheDir(std::io::Error), +} diff --git a/fontc_crater/src/main.rs b/fontc_crater/src/main.rs new file mode 100644 index 000000000..60006ff59 --- /dev/null +++ b/fontc_crater/src/main.rs @@ -0,0 +1,274 @@ +//! run bulk operations on fonts + +use std::{ + collections::{BTreeMap, BTreeSet}, + path::{Path, PathBuf}, + time::Instant, +}; + +use clap::Parser; +use fontc::JobTimer; +use google_fonts_sources::RepoInfo; +use write_fonts::types::Tag; + +mod args; +mod error; +mod sources; + +use sources::RepoList; + +use args::{Args, Tasks}; +use error::Error; + +fn main() { + env_logger::init(); + let args = Args::parse(); + if let Err(e) = run(&args) { + eprintln!("{e}"); + } +} + +fn run(args: &Args) -> Result<(), Error> { + if !args.font_cache.exists() { + std::fs::create_dir_all(&args.font_cache).map_err(Error::CacheDir)?; + } + let sources = RepoList::get_or_create(&args.font_cache, args.fonts_repo.as_deref())?; + + match args.command { + Tasks::Compile => { + compile_all(&sources.sources, &args.font_cache, args.out_path.as_deref())? + } + }; + sources.save(&args.font_cache)?; + Ok(()) +} + +/// Results of all runs +#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +struct Results { + success: BTreeSet, + failure: BTreeMap, + panic: BTreeSet, + skipped: BTreeMap, +} + +/// The output of trying to run on one font. +/// +/// We don't use a normal Result because failure is okay, we will report it all at the end. +enum RunResult { + Skipped(SkipReason), + Success, + Fail(String), + Panic, +} + +/// Reason why we did not run a font +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +enum SkipReason { + /// Checkout failed + GitFail, + /// There was no config.yaml file + NoConfig, +} + +fn compile_all( + sources: &[RepoInfo], + cache_dir: &Path, + out_path: Option<&Path>, +) -> Result<(), Error> { + let results = sources + .iter() + .flat_map(|info| { + let font_dir = cache_dir.join(&info.repo_name); + fetch_and_run_repo(&font_dir, info) + }) + .collect::(); + + if let Some(path) = out_path { + let as_json = serde_json::to_string_pretty(&results).map_err(Error::OutputJson)?; + std::fs::write(path, as_json).map_err(|error| Error::WriteFile { + path: path.to_owned(), + error, + })?; + } else { + results.print_summary(); + } + Ok(()) +} + +// one repo can contain multiple sources, so we return a vec. +fn fetch_and_run_repo(font_dir: &Path, repo: &RepoInfo) -> Vec<(PathBuf, RunResult)> { + if !font_dir.exists() && clone_repo(font_dir, &repo.repo_url).is_err() { + return vec![(font_dir.to_owned(), RunResult::Skipped(SkipReason::GitFail))]; + } + + let source_dir = font_dir.join("sources"); + let configs = repo + .config_files + .iter() + .flat_map(|filename| { + let config_path = source_dir.join(filename); + load_config(&config_path) + }) + .collect::>(); + + if configs.is_empty() { + return vec![( + font_dir.to_owned(), + RunResult::Skipped(SkipReason::NoConfig), + )]; + }; + + // collect to set in case configs duplicate sources + let sources = configs + .iter() + .flat_map(|c| c.sources.iter()) + .map(|source| source_dir.join(source)) + .collect::>(); + + sources + .into_iter() + .map(|source| { + let result = compile_one(&source); + (source, result) + }) + .collect() +} + +fn compile_one(source_path: &Path) -> RunResult { + let tempdir = tempfile::tempdir().unwrap(); + let args = fontc::Args::new(tempdir.path(), source_path.to_owned()); + let timer = JobTimer::new(Instant::now()); + eprintln!("compiling {}", source_path.display()); + match std::panic::catch_unwind(|| fontc::run(args, timer)) { + Ok(Ok(_)) => RunResult::Success, + Ok(Err(e)) => RunResult::Fail(e.to_string()), + Err(_) => RunResult::Panic, + } +} + +// on fail returns contents of stderr +fn clone_repo(to_dir: &Path, repo: &str) -> Result<(), String> { + assert!(!to_dir.exists()); + eprintln!("cloning '{repo}' to {}", to_dir.display()); + let output = std::process::Command::new("git") + // if a repo requires credentials fail instead of waiting + .env("GIT_TERMINAL_PROMPT", "0") + .arg("clone") + .args(["--depth", "1"]) + .arg(repo) + .arg(to_dir) + .output() + .expect("failed to execute git command"); + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!("clone failed: '{stderr}'"); + return Err(stderr.into_owned()); + } + Ok(()) +} + +/// Parse and return a config.yaml file for the provided font source +fn load_config(config_path: &Path) -> Option { + let contents = match std::fs::read_to_string(config_path) { + Ok(contents) => contents, + Err(e) => { + eprintln!("failed to load config at '{}': {e}", config_path.display()); + return None; + } + }; + match serde_yaml::from_str(&contents) { + Ok(config) => Some(config), + Err(e) => { + eprintln!("BAD YAML: {contents}: '{e}'"); + None + } + } +} + +impl FromIterator<(PathBuf, RunResult)> for Results { + fn from_iter>(iter: T) -> Self { + let mut out = Results::default(); + for (path, reason) in iter.into_iter() { + match reason { + RunResult::Skipped(reason) => { + out.skipped.insert(path, reason); + } + RunResult::Success => { + out.success.insert(path); + } + RunResult::Fail(reason) => { + out.failure.insert(path, reason); + } + RunResult::Panic => { + out.panic.insert(path); + } + } + } + out + } +} + +impl Results { + fn print_summary(&self) { + let total = self.success.len() + self.failure.len() + self.panic.len() + self.skipped.len(); + + println!( + "\ncompiled {total} fonts: {} skipped, {} panics, {} failures {} success", + self.skipped.len(), + self.panic.len(), + self.failure.len(), + self.success.len(), + ); + + if self.skipped.is_empty() { + println!("\n#### {} fonts were skipped ####", self.skipped.len()); + for (path, reason) in &self.skipped { + println!("{}: {}", path.display(), reason); + } + } + + if !self.panic.is_empty() { + println!( + "\n#### {} fonts panicked the compiler: ####", + self.panic.len() + ); + + for path in &self.panic { + println!("{}", path.display()) + } + } + + if !self.failure.is_empty() { + println!("\n#### {} fonts failed to compile ####", self.failure.len()); + for path in self.failure.keys() { + println!("{}", path.display()); + } + } + } +} + +/// Google fonts config file ('config.yaml') +#[derive(Clone, Debug, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +// there are a few fields of this that we dont' care about but parse anyway? +// maybe we will want them later? +#[allow(dead_code)] +struct Config { + sources: Vec, + family_name: Option, + #[serde(default)] + build_variable: bool, + #[serde(default)] + axis_order: Vec, +} + +impl std::fmt::Display for SkipReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SkipReason::GitFail => f.write_str("Git checkout failed"), + SkipReason::NoConfig => f.write_str("No config.yaml file found"), + } + } +} diff --git a/fontc_crater/src/sources.rs b/fontc_crater/src/sources.rs new file mode 100644 index 000000000..5f26dd88c --- /dev/null +++ b/fontc_crater/src/sources.rs @@ -0,0 +1,63 @@ +//! finding font sources +use std::{ + path::Path, + time::{Duration, SystemTime}, +}; + +use google_fonts_sources::RepoInfo; +use serde::{Deserialize, Serialize}; + +use crate::error::Error; + +static CACHED_REPO_INFO_FILE: &str = "google_fonts_repos.json"; +const ONE_WEEK: Duration = Duration::from_secs(60 * 60 * 24 * 7); + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub(crate) struct RepoList { + pub(crate) created: SystemTime, + pub(crate) sources: Vec, +} + +impl RepoList { + pub(crate) fn get_or_create( + cache_dir: &Path, + fonts_repo: Option<&Path>, + ) -> Result { + let cache_file_path = cache_dir.join(CACHED_REPO_INFO_FILE); + if let Some(cached_list) = Self::load(&cache_file_path)? { + let stale = cached_list + .created + .elapsed() + .map(|d| d > ONE_WEEK) + .unwrap_or(true); + if !stale { + return Ok(cached_list); + } + } + + let mut sources = + google_fonts_sources::discover_sources(fonts_repo, Some(cache_dir), false); + + // only keep sources for which we have a repo + config + sources.retain(|s| !s.config_files.is_empty()); + + Ok(RepoList { + created: SystemTime::now(), + sources, + }) + } + + fn load(path: &Path) -> Result, Error> { + if !path.exists() { + return Ok(None); + } + let string = std::fs::read_to_string(path).map_err(Error::InputFile)?; + Some(serde_json::from_str(&string).map_err(Error::InputJson)).transpose() + } + + pub(crate) fn save(&self, cache_dir: &Path) -> Result<(), Error> { + let path = cache_dir.join(CACHED_REPO_INFO_FILE); + let string = serde_json::to_string_pretty(&self).map_err(Error::OutputJson)?; + std::fs::write(&path, string).map_err(|error| Error::WriteFile { path, error }) + } +} diff --git a/resources/scripts/check_no_println.sh b/resources/scripts/check_no_println.sh index 28bcf82eb..e1867cb7f 100755 --- a/resources/scripts/check_no_println.sh +++ b/resources/scripts/check_no_println.sh @@ -93,6 +93,7 @@ allowlist+=("fea-rs/src/tests/parse.rs") allowlist+=("otl-normalizer") allowlist+=("glyphs-reader/build.rs") allowlist+=("fontdrasil/build.rs") +allowlist+=("fontc_crater/src/main.rs") allowlist=$(join "|" "${allowlist[@]}") echo grep -v "($allowlist)"