release v0.5.1

.github/workflows/release.yml

@@ -9,7 +9,18 @@ on:
       - v[0-9]+.*
 
 jobs:
+  # make sure release content has correct format and README is up-to-date
+  format-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run format check
+        run: cargo fmt --check
+      - name: Run cargo-readme check
+        run: cargo install cargo-readme && cargo readme > TMP_README.md && diff -b TMP_README.md README.md
+
   create-release:
+    needs: format-check
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -20,17 +31,6 @@ jobs:
           # (required) GitHub token for creating GitHub Releases.
           token: ${{ secrets.GITHUB_TOKEN }}
 
-  format-check:
-    needs: create-release
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Run format check
-        run: cargo fmt --check
-      - name: Run cargo-readme check
-        run: cargo install cargo-readme && cargo readme > TMP_README.md && diff -b TMP_README.md README.md
-
-
   cargo-publish:
     needs: create-release
     runs-on: ubuntu-latest
@@ -41,4 +41,30 @@ jobs:
           cargo publish
           --all-features
           --verbose
-          --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+          --token ${{ secrets.CARGO_REGISTRY_TOKEN }}
+
+  upload-assets:
+    needs: create-release
+    strategy:
+      matrix:
+        include:
+          - target: aarch64-unknown-linux-gnu
+            os: ubuntu-latest
+          - target: x86_64-unknown-linux-gnu
+            os: ubuntu-latest
+          - target: universal-apple-darwin
+            os: macos-latest
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: taiki-e/upload-rust-binary-action@v1
+        with:
+          # (required) Comma-separated list of binary names (non-extension portion of filename) to build and upload.
+          # Note that glob pattern is not supported yet.
+          bin: bgpkit-commons
+          features: cli
+          checksum: sha256
+          # (optional) Target triple, default is host triple.
+          target: ${{ matrix.target }}
+          # (required) GitHub token for uploading assets to GitHub Releases.
+          token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,3 +1,4 @@
 /target
 /Cargo.lock
-.idea
+.idea
+*.json

CHANGELOG.md

@@ -2,9 +2,17 @@
 
 All notable changes to this project will be documented in this file.
 
+## v0.5.1 - 2024-03-20
+
+### Highlights
+
+* add new `bgpkit-commons` binary with `export` subcommand to export all data to JSON files
+* replace `reqwest` with `oneio` as the default HTTP client
+
 ## v0.5.0 - 2024-01-30
 
 ### Breaking changes
 
 - switch to `rustls` as the default TLS backend
-  - users can still opt-in to use `native-tls` by specifying `default-features = false` and use `native-tls` feature flag
+    - users can still opt-in to use `native-tls` by specifying `default-features = false` and use `native-tls` feature
+      flag

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bgpkit-commons"
-version = "0.5.0"
+version = "0.5.1"
 edition = "2021"
 authors = ["Mingwei Zhang <mingwei@bgpkit.com>"]
 readme = "README.md"
@@ -14,23 +14,35 @@ keywords = ["bgp", "bgpkit"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[[bin]]
+name = "bgpkit-commons"
+path = "src/bin/main.rs"
+required-features = ["cli"]
+
 [dependencies]
-reqwest = {version = "0.11", default-features = false, features = ["blocking", "json"]}
-serde = {version="1.0", features = ["derive"]}
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
-oneio = {version="0.16.0", default-features = false, features=["lib-core"]}
+oneio = { version = "0.16.4", default-features = false, features = ["lib-core"] }
 regex = "1"
 anyhow = "1.0"
-chrono = "0.4"
+chrono = { version = "0.4", features = ["serde"] }
 ipnet-trie = "0.1.0"
 ipnet = "2.8"
 tar = "0.4"
 
 tracing = "0.1"
 tracing-subscriber = "0.3"
 
+clap = { version = "4.4", features = ["derive"], optional = true }
+
 
 [features]
 default = ["rustls"]
-native-tls = ["reqwest/default-tls", "oneio/native-tls"]
-rustls = ["reqwest/rustls-tls", "oneio/rustls"]
+native-tls = ["oneio/native-tls"]
+rustls = ["oneio/rustls"]
+
+cli = ["clap", "default"]
+
+[package.metadata.binstall]
+pkg-url = "{ repo }/releases/download/v{ version }/{ name }-{ target }.tar.gz"
+pkg-fmt = "tgz"

README.md

@@ -188,6 +188,36 @@ assert_eq!(rpki.validate(&prefix, 13335), RpkiValidation::Valid);
 
 - `rustls`: use rustls instead of native-tls for the underlying HTTPS requests
 
+## Commandline tool
+
+This crate also provides a commandline tool `bgpkit-commons` for easy access to the data and utilities.
+
+### Installation
+
+On macOS:
+```sh
+brew install bgpkit/tap/bgpkit-commons
+```
+
+On other platforms:
+```sh
+cargo binstall bgpkit-commons
+```
+
+#### Export all data to JSON
+
+```sh
+bgpkit-commons export --help
+Export data to local files
+
+ Usage: bgpkit-commons export [OPTIONS]
+
+ Options:
+   -o, --output-dir <OUTPUT_DIR>  output directory [default: .]
+   -h, --help                     Print help
+   -V, --version                  Print version
+```
+
 ## Built with ❤️ by BGPKIT Team
 
 <a href="https://bgpkit.com"><img src="https://bgpkit.com/Original%20Logo%20Cropped.png" alt="https://bgpkit.com/favicon.ico" width="200"/></a>

src/asnames/mod.rs

@@ -28,9 +28,10 @@
 //! ```
 
 use anyhow::Result;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AsName {
     pub asn: u32,
     pub name: String,
@@ -40,7 +41,7 @@ pub struct AsName {
 const DATA_URL: &str = "https://ftp.ripe.net/ripe/asnames/asn.txt";
 
 pub fn get_asnames() -> Result<HashMap<u32, AsName>> {
-    let text = reqwest::blocking::get(DATA_URL)?.text()?;
+    let text = oneio::read_to_string(DATA_URL)?;
     let asnames = text
         .lines()
         .filter_map(|line| {

src/bin/main.rs

@@ -0,0 +1,69 @@
+use bgpkit_commons::asnames::{get_asnames, AsName};
+use bgpkit_commons::collectors::get_all_collectors;
+use bgpkit_commons::countries::Countries;
+use clap::{Parser, Subcommand};
+use tracing::info;
+
+#[derive(Parser)]
+#[clap(author, version)]
+#[clap(propagate_version = true)]
+#[command(arg_required_else_help(true))]
+/// oneio reads files from local or remote locations with any compression.
+struct Cli {
+    #[clap(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+    /// Export data to local files
+    Export {
+        /// output directory
+        #[clap(short, long, default_value = ".")]
+        output_dir: String,
+    },
+}
+
+fn main() {
+    tracing_subscriber::fmt::init();
+
+    let cli = Cli::parse();
+    match cli.command {
+        Commands::Export { output_dir } => {
+            // strip output dir's trailing slash
+            let output_dir = output_dir.trim_end_matches('/');
+            std::fs::create_dir_all(output_dir).unwrap();
+
+            info!("exporting MRT collectors data...");
+            let collectors_vec = get_all_collectors().unwrap();
+            let file_path = format!("{}/collectors.json", output_dir);
+            let collectors_json = serde_json::to_string_pretty(&collectors_vec).unwrap();
+            oneio::get_writer(&file_path)
+                .unwrap()
+                .write_all(collectors_json.as_bytes())
+                .unwrap();
+
+            info!("exporting countries data...");
+            let countries_vec = Countries::new().unwrap().all_countries();
+            let countries_file = format!("{}/countries.json", output_dir);
+            let countries_json = serde_json::to_string_pretty(&countries_vec).unwrap();
+            oneio::get_writer(&countries_file)
+                .unwrap()
+                .write_all(countries_json.as_bytes())
+                .unwrap();
+
+            info!("exporting ASNs data...");
+            let asns_vec = get_asnames()
+                .unwrap()
+                .values()
+                .cloned()
+                .collect::<Vec<AsName>>();
+            let asns_json = serde_json::to_string_pretty(&asns_vec).unwrap();
+            let asns_file = format!("{}/asns.json", output_dir);
+            oneio::get_writer(&asns_file)
+                .unwrap()
+                .write_all(asns_json.as_bytes())
+                .unwrap();
+        }
+    }
+}

src/collectors/mod.rs

@@ -9,6 +9,7 @@ Currently supported MRT collector projects:
 
 use anyhow::Result;
 use chrono::NaiveDateTime;
+use serde::{Deserialize, Serialize, Serializer};
 use std::cmp::Ordering;
 use std::fmt::{Display, Formatter};
 
@@ -19,12 +20,23 @@ pub use riperis::get_riperis_collectors;
 pub use routeviews::get_routeviews_collectors;
 
 /// MRT collector project enum
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)]
 pub enum MrtCollectorProject {
     RouteViews,
     RipeRis,
 }
 
+// Custom serialization function for the `age` field
+fn serialize_project<S>(project: &MrtCollectorProject, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    serializer.serialize_str(match project {
+        MrtCollectorProject::RouteViews => "routeview",
+        MrtCollectorProject::RipeRis => "riperis",
+    })
+}
+
 impl Display for MrtCollectorProject {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -39,11 +51,12 @@ impl Display for MrtCollectorProject {
 }
 
 /// MRT collector meta information
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
 pub struct MrtCollector {
     /// name of the collector
     pub name: String,
     /// collector project
+    #[serde(serialize_with = "serialize_project")]
     pub project: MrtCollectorProject,
     /// MRT data files root URL
     pub data_url: String,

src/collectors/riperis.rs

@@ -71,8 +71,7 @@ impl ToMrtCollector for RisCollector {
 
 /// Get RIPE RIS collectors meta information
 pub fn get_riperis_collectors() -> Result<Vec<MrtCollector>> {
-    let data = reqwest::blocking::get("https://stat.ripe.net/data/rrc-info/data.json")?
-        .json::<RisData>()?;
+    let data = oneio::read_json_struct::<RisData>("https://stat.ripe.net/data/rrc-info/data.json")?;
     Ok(data
         .data
         .rrcs

src/collectors/routeviews.rs

@@ -35,7 +35,7 @@ struct RVCollector {
 struct RvData {
     pub count: i64,
     #[serde(rename(deserialize = "results"))]
-    pub collectors: Vec<RVCollector>,
+    pub data: Vec<RVCollector>,
 }
 
 impl ToMrtCollector for RVCollector {
@@ -78,13 +78,23 @@ impl ToMrtCollector for RVCollector {
 
 /// Get RouteViews collectors meta information
 pub fn get_routeviews_collectors() -> Result<Vec<MrtCollector>> {
-    let data = reqwest::blocking::get("https://api.routeviews.org/collector/?format=json")?
-        .json::<RvData>()?;
+    let data =
+        oneio::read_json_struct::<RvData>("https://api.routeviews.org/collector/?format=json")?;
     Ok(data
-        .collectors
+        .data
         .into_iter()
         // exclude the Cisco collector that does not provide MRT archive
         .filter(|c| c.name.as_str() != "route-views")
         .map(|c| c.to_mrt_collector().unwrap())
         .collect())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_routeviews_collectors() {
+        dbg!(get_routeviews_collectors().unwrap());
+    }
+}

src/countries/mod.rs

@@ -38,12 +38,13 @@
 //!
 //! This method returns all countries in the collection. It returns a `Vec<Country>`.
 use anyhow::{anyhow, Result};
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
 /// Country data structure
 ///
 /// Information coming from <https://download.geonames.org/export/dump/countryInfo.txt>
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Country {
     /// 2-letter country code
     pub code: String,