Merge pull request #1 from bradfier/nogpl-feature

Add with-gpl-data feature for embedding MIME DB

Author: mbrubeck

Cargo.toml

@@ -10,7 +10,7 @@ repository = "https://github.com/mbrubeck/tree_magic/"
 documentation = "https://docs.rs/tree_magic_mini/"
 readme = "README.md"
 keywords = ["mime", "filesystem", "media-types"]
-license = "GPL-2.0-or-later"
+license = "MIT"
 exclude = ["tests/*", "benches/*/"]
 edition = "2018"
 
@@ -19,6 +19,11 @@ petgraph = "^0.5"
 nom = "6.0"
 lazy_static = "^1.4"
 fnv = "^1"
+once_cell = "^1"
+tree_magic_db = { path = "./magic_db" , optional = true }
+
+[features]
+with-gpl-data = ["tree_magic_db"]
 
 [dev-dependencies]
 bencher = "^0.1"
@@ -27,6 +32,9 @@ bencher = "^0.1"
 crate-type = ["lib"]
 path = "src/lib.rs"
 
+[workspace]
+members = ["magic_db"]
+
 [[bench]]
 name = "from_u8"
 harness = false

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Aaron Hancock
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -19,6 +19,20 @@ simplify the effort to maintain and optimize of this fork. I would like to
 eventually merge these changes back to the original `tree_magic` crate, and/or
 restore some of the removed features if there is demand for that.
 
+## Licensing and the MIME database
+
+By default, `tree_magic_mini` will attempt to load the shared MIME info
+database from the standard locations at runtime.
+
+If you won't have the database files available, or would like to include them
+in your binary for simplicity, you can optionally embed the database
+information if you enable the `tree_magic_db` feature.
+
+**As the magic database files themselves are licensed under the GPL, you must
+make sure your project uses a compatible license if you enable this behaviour.**
+
+---
+
 Continue reading for the original `tree_magic` documentation.
 
 ## About tree_magic

COPYING magic_db/COPYING

@@ -0,0 +1,7 @@
+[package]
+name = "tree_magic_db"
+version = "1.0.0"
+authors = ["Richard Bradfield <bradfier@fstab.me>"]
+description = "Packages the FreeDesktop.org shared MIME database for optional use with tree_magic_mini"
+license = "GPL-2.0-or-later"
+edition = "2018"

magic_db/Cargo.toml

@@ -0,0 +1,19 @@
+# Magic DB
+
+This subcrate contains the magic definitions copied from the FreeDesktop.org
+shared MIME info database.
+
+These files are distributed under the GPL-2.0 or later, as such if you include
+them in your project, it must be licensed in a compatible manner.
+
+# References
+
+- [xdg-shared-mime-info Repository](https://github.com/freedesktop/xdg-shared-mime-info)
+- [FreeDesktop shared-mime-info Spec](https://www.freedesktop.org/wiki/Specifications/shared-mime-info-spec/)
+
+# Licence
+
+The contents of this crate are licenced under the GNU General Public License,
+version 2.0 or later, at your option:
+
+ * GNU GPL, Version 2.0 ([COPYING](COPYING) or https://www.gnu.org/licenses/old-licenses/gpl-2.0.txt)

magic_db/README.md

@@ -0,0 +1,26 @@
+//! Packaging crate for the FreeDesktop.org shared MIME info database
+
+// This program is free software; you can redistribute it and/or
+// modify it under the terms of the GNU General Public License
+// as published by the Free Software Foundation; either version 2
+// of the License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+/// Returns a static reference to the MIME database 'aliases'
+pub fn aliases() -> &'static str {
+    include_str!("aliases")
+}
+
+/// Returns a static reference to the MIME database 'subclasses'
+pub fn subclasses() -> &'static str {
+    include_str!("subclasses")
+}
+
+/// Returns a static reference to the MIME database 'magic' data
+pub fn magic() -> &'static [u8] {
+    include_bytes!("magic")
+}

src/fdo_magic/builtin/aliases magic_db/src/aliases

@@ -1,8 +1,25 @@
 use crate::MIME;
 use fnv::FnvHashMap;
 
+#[cfg(not(feature = "with-gpl-data"))]
+use super::runtime;
+
+fn aliases() -> &'static str {
+    #[cfg(feature = "with-gpl-data")]
+    return tree_magic_db::aliases();
+    #[cfg(not(feature = "with-gpl-data"))]
+    return runtime::aliases();
+}
+
+fn subclasses() -> &'static str {
+    #[cfg(feature = "with-gpl-data")]
+    return tree_magic_db::subclasses();
+    #[cfg(not(feature = "with-gpl-data"))]
+    return runtime::subclasses();
+}
+
 pub fn get_aliaslist() -> FnvHashMap<MIME, MIME> {
-    include_str!("aliases")
+    aliases()
         .lines()
         .map(|line| {
             let mut parts = line.split_whitespace();
@@ -20,7 +37,7 @@ pub fn get_supported() -> Vec<MIME> {
 
 /// Get list of parent -> child subclass links
 pub fn get_subclasses() -> Vec<(MIME, MIME)> {
-    include_str!("subclasses")
+    subclasses()
         .lines()
         .map(|line| {
             let mut parts = line.split_whitespace();

magic_db/src/lib.rs

@@ -13,9 +13,28 @@ lazy_static! {
 
 /// Load magic file before anything else.
 lazy_static! {
-    static ref ALLRULES: FnvHashMap<MIME, DiGraph<MagicRule<'static>, u32>> =
-        super::ruleset::from_u8(include_bytes!("magic")).unwrap_or(FnvHashMap::default());
+    static ref ALLRULES: FnvHashMap<MIME, DiGraph<MagicRule<'static>, u32>> = rules();
 }
 
 pub mod check;
 pub mod init;
+
+#[cfg(not(feature = "with-gpl-data"))]
+mod runtime;
+
+fn rules() -> FnvHashMap<MIME, DiGraph<MagicRule<'static>, u32>> {
+    #[cfg(feature = "with-gpl-data")]
+    return static_rules();
+    #[cfg(not(feature = "with-gpl-data"))]
+    return runtime_rules();
+}
+
+#[cfg(feature = "with-gpl-data")]
+fn static_rules() -> FnvHashMap<MIME, DiGraph<MagicRule<'static>, u32>> {
+    super::ruleset::from_u8(tree_magic_db::magic()).unwrap_or(FnvHashMap::default())
+}
+
+#[cfg(not(feature = "with-gpl-data"))]
+fn runtime_rules() -> FnvHashMap<MIME, DiGraph<MagicRule<'static>, u32>> {
+    runtime::rules().unwrap_or(FnvHashMap::default())
+}

src/fdo_magic/builtin/magic magic_db/src/magic

@@ -0,0 +1,94 @@
+///! Enable loading the magic database files at runtime rather than embedding the GPLed database
+use std::fs::File;
+use std::io::Read;
+
+use fnv::FnvHashMap;
+use once_cell::sync::OnceCell;
+use petgraph::prelude::DiGraph;
+
+use super::MagicRule;
+use crate::fdo_magic::ruleset;
+use crate::MIME;
+
+static RUNTIME_RULES: OnceCell<Vec<Vec<u8>>> = OnceCell::new();
+static ALIAS_STRING: OnceCell<String> = OnceCell::new();
+static SUBCLASS_STRING: OnceCell<String> = OnceCell::new();
+
+/// Load the magic database from the predefined locations in the XDG standard
+fn load_xdg_shared_magic() -> Vec<Vec<u8>> {
+    const SEARCH_PATHS: &[&str; 3] = &[
+        "/usr/share/mime/magic",
+        "/usr/local/share/mime/magic",
+        "$HOME/.local/share/mime/magic",
+    ];
+
+    let files: Vec<Vec<u8>> = SEARCH_PATHS
+        .iter()
+        .map(|p| File::open(p).ok())
+        .filter_map(|f| f)
+        .map(|mut f| {
+            let mut buf = vec![];
+            f.read_to_end(&mut buf)
+                .expect("Failed to read magic file bytes");
+            buf
+        })
+        .collect();
+
+    if files.is_empty() {
+        panic!("No MIME magic files found in the XDG default paths");
+    }
+
+    files
+}
+
+/// Load a number of files at `paths` and concatenate them together with a newline
+fn load_concat_strings(paths: &[&str]) -> String {
+    let strings: Vec<String> = paths
+        .iter()
+        .map(|p| File::open(p).ok())
+        .filter_map(|f| f)
+        .map(|mut f| {
+            let mut s = String::new();
+            f.read_to_string(&mut s)
+                .expect("Failed to read aliases from file");
+            s
+        })
+        .collect();
+
+    strings.join("\n")
+}
+
+/// Load the magic aliases from the XDG standard locations and concatenate them together
+fn load_aliases() -> String {
+    const SEARCH_PATHS: &[&str; 3] = &[
+        "/usr/share/mime/aliases",
+        "/usr/local/share/mime/aliases",
+        "$HOME/.local/share/mime/aliases",
+    ];
+
+    load_concat_strings(SEARCH_PATHS)
+}
+
+/// Load the subclass definitions from the XDG standard locations and concatenate them together
+fn load_subclasses() -> String {
+    const SEARCH_PATHS: &[&str; 3] = &[
+        "/usr/share/mime/subclasses",
+        "/usr/local/share/mime/subclasses",
+        "$HOME/.local/share/mime/subclasses",
+    ];
+
+    load_concat_strings(SEARCH_PATHS)
+}
+
+pub(crate) fn aliases() -> &'static str {
+    ALIAS_STRING.get_or_init(load_aliases)
+}
+
+pub(crate) fn subclasses() -> &'static str {
+    SUBCLASS_STRING.get_or_init(load_subclasses)
+}
+
+pub(crate) fn rules() -> Result<FnvHashMap<MIME, DiGraph<MagicRule<'static>, u32>>, String> {
+    let files = RUNTIME_RULES.get_or_init(load_xdg_shared_magic);
+    Ok(ruleset::from_multiple(files).unwrap())
+}

src/fdo_magic/builtin/subclasses magic_db/src/subclasses

@@ -99,6 +99,7 @@ fn gen_graph(magic_rules: Vec<MagicRule<'_>>) -> DiGraph<MagicRule<'_>, u32> {
     graph
 }
 
+#[cfg(feature = "with-gpl-data")]
 pub fn from_u8(b: &[u8]) -> Result<FnvHashMap<&str, DiGraph<MagicRule<'_>, u32>>, String> {
     let tuplevec = ruleset(b).map_err(|e| e.to_string())?.1;
     let res = tuplevec
@@ -107,3 +108,19 @@ pub fn from_u8(b: &[u8]) -> Result<FnvHashMap<&str, DiGraph<MagicRule<'_>, u32>>
         .collect();
     Ok(res)
 }
+
+#[cfg(not(feature = "with-gpl-data"))]
+/// Parse multiple ruleset magic files and aggregate the tuples into a single graph
+pub fn from_multiple<'a>(
+    files: &'a [Vec<u8>],
+) -> Result<FnvHashMap<&'a str, DiGraph<MagicRule<'a>, u32>>, String> {
+    let mut tuplevec = vec![];
+    for slice in files {
+        tuplevec.append(&mut ruleset(slice.as_ref()).map_err(|e| e.to_string())?.1);
+    }
+    let res = tuplevec
+        .into_iter()
+        .map(|x| (x.0, gen_graph(x.1)))
+        .collect();
+    Ok(res)
+}

src/fdo_magic/builtin/init.rs

@@ -2,7 +2,7 @@
 //!
 //! This is a fork of the [tree_magic](https://crates.io/crates/tree_magic)
 //! crate by Allison Hancock. It includes the following changes:
-//! 
+//!
 //! * Updated dependencies.
 //! * Reduced copying and memory allocation, for a slight increase in speed and
 //!   decrease in memory use.
@@ -27,6 +27,18 @@
 //! - Can delegate different file types to different "checkers", reducing false positives
 //!   by choosing a different method of attack.
 //!
+//! ## Licensing and the MIME database
+//!
+//! By default, `tree_magic_mini` will attempt to load the shared MIME info
+//! database from the standard locations at runtime.
+//!
+//! If you won't have the database files available, or would like to include them
+//! in your binary for simplicity, you can optionally embed the database
+//! information if you enable the `tree_magic_db` feature.
+//!
+//! **As the magic database files themselves are licensed under the GPL, you must
+//! make sure your project uses a compatible license if you enable this behaviour.**
+//!
 //! # Example
 //! ```rust
 //! // Load a GIF file
@@ -41,16 +53,6 @@
 //! assert_eq!(result, true);
 //! ```
 
-// This program is free software; you can redistribute it and/or
-// modify it under the terms of the GNU General Public License
-// as published by the Free Software Foundation; either version 2
-// of the License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
 #![allow(unused_doc_comments)]
 
 use fnv::FnvHashMap;

src/fdo_magic/builtin/mod.rs

@@ -1,6 +1,6 @@
 mod from_filepath {
-    use tree_magic_mini as tree_magic;
     use std::path::Path;
+    use tree_magic_mini as tree_magic;
 
     #[test]
     fn nonexistent_file_returns_none() {