diff --git a/a.txt b/a.txt new file mode 100644 index 0000000..28fd67f --- /dev/null +++ b/a.txt @@ -0,0 +1,4263 @@ +./pubky-common/src/auth.rs +``` +//! Client-server Authentication using signed timesteps + +use std::sync::{Arc, Mutex}; + +use serde::{Deserialize, Serialize}; + +use crate::{ + capabilities::{Capabilities, Capability}, + crypto::{Keypair, PublicKey, Signature}, + namespaces::PUBKY_AUTH, + timestamp::Timestamp, +}; + +// 30 seconds +const TIME_INTERVAL: u64 = 30 * 1_000_000; + +const CURRENT_VERSION: u8 = 0; +// 45 seconds in the past or the future +const TIMESTAMP_WINDOW: i64 = 45 * 1_000_000; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +/// Implementation of the [Pubky Auth spec](https://pubky.github.io/pubky-core/spec/auth.html). +pub struct AuthToken { + /// Signature over the token. + signature: Signature, + /// A namespace to ensure this signature can't be used for any + /// other purposes that share the same message structurea by accident. + namespace: [u8; 10], + /// Version of the [AuthToken], in case we need to upgrade it to support unforeseen usecases. + /// + /// Version 0: + /// - Signer is implicitly the same as the root keypair for + /// the [AuthToken::pubky], without any delegation. + /// - Capabilities are only meant for resoucres on the homeserver. + version: u8, + /// Timestamp + timestamp: Timestamp, + /// The [PublicKey] of the owner of the resources being accessed by this token. + pubky: PublicKey, + // Variable length capabilities + capabilities: Capabilities, +} + +impl AuthToken { + /// Sign a new AuthToken with given capabilities. + pub fn sign(keypair: &Keypair, capabilities: impl Into) -> Self { + let timestamp = Timestamp::now(); + + let mut token = Self { + signature: Signature::from_bytes(&[0; 64]), + namespace: *PUBKY_AUTH, + version: 0, + timestamp, + pubky: keypair.public_key(), + capabilities: capabilities.into(), + }; + + let serialized = token.serialize(); + + token.signature = keypair.sign(&serialized[65..]); + + token + } + + // === Getters === + + /// Returns the pubky that is providing this AuthToken + pub fn pubky(&self) -> &PublicKey { + &self.pubky + } + + /// Returns the capabilities in this AuthToken. + pub fn capabilities(&self) -> &[Capability] { + &self.capabilities.0 + } + + // === Public Methods === + + /// Parse and verify an AuthToken. + pub fn verify(bytes: &[u8]) -> Result { + if bytes[75] > CURRENT_VERSION { + return Err(Error::UnknownVersion); + } + + let token = AuthToken::deserialize(bytes)?; + + match token.version { + 0 => { + let now = Timestamp::now(); + + // Chcek timestamp; + let diff = token.timestamp.as_u64() as i64 - now.as_u64() as i64; + if diff > TIMESTAMP_WINDOW { + return Err(Error::TooFarInTheFuture); + } + if diff < -TIMESTAMP_WINDOW { + return Err(Error::Expired); + } + + token + .pubky + .verify(AuthToken::signable(token.version, bytes), &token.signature) + .map_err(|_| Error::InvalidSignature)?; + + Ok(token) + } + _ => unreachable!(), + } + } + + /// Serialize this AuthToken to its canonical binary representation. + pub fn serialize(&self) -> Vec { + postcard::to_allocvec(self).unwrap() + } + + /// Deserialize an AuthToken from its canonical binary representation. + pub fn deserialize(bytes: &[u8]) -> Result { + Ok(postcard::from_bytes(bytes)?) + } + + /// Returns the unique ID for this [AuthToken], which is a concatenation of + /// [AuthToken::pubky] and [AuthToken::timestamp]. + /// + /// Assuming that [AuthToken::timestamp] is unique for every [AuthToken::pubky]. + fn id(version: u8, bytes: &[u8]) -> Box<[u8]> { + match version { + 0 => bytes[75..115].into(), + _ => unreachable!(), + } + } + + fn signable(version: u8, bytes: &[u8]) -> &[u8] { + match version { + 0 => bytes[65..].into(), + _ => unreachable!(), + } + } +} + +#[derive(Debug, Clone, Default)] +/// Keeps track of used AuthToken until they expire. +pub struct AuthVerifier { + seen: Arc>>>, +} + +impl AuthVerifier { + /// Verify an [AuthToken] by parsing it from its canonical binary representation, + /// verifying its signature, and confirm it wasn't already used. + pub fn verify(&self, bytes: &[u8]) -> Result { + self.gc(); + + let token = AuthToken::verify(bytes)?; + + let mut seen = self.seen.lock().unwrap(); + + let id = AuthToken::id(token.version, bytes); + + match seen.binary_search_by(|element| element.cmp(&id)) { + Ok(_) => Err(Error::AlreadyUsed), + Err(index) => { + seen.insert(index, id); + Ok(token) + } + } + } + + // === Private Methods === + + /// Remove all tokens older than two time intervals in the past. + fn gc(&self) { + let threshold = ((Timestamp::now().as_u64() / TIME_INTERVAL) - 2).to_be_bytes(); + + let mut inner = self.seen.lock().unwrap(); + + match inner.binary_search_by(|element| element[0..8].cmp(&threshold)) { + Ok(index) | Err(index) => { + inner.drain(0..index); + } + } + } +} + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +/// Error verifying an [AuthToken] +pub enum Error { + #[error("Unknown version")] + /// Unknown version + UnknownVersion, + #[error("AuthToken has a timestamp that is more than 45 seconds in the future")] + /// AuthToken has a timestamp that is more than 45 seconds in the future + TooFarInTheFuture, + #[error("AuthToken has a timestamp that is more than 45 seconds in the past")] + /// AuthToken has a timestamp that is more than 45 seconds in the past + Expired, + #[error("Invalid Signature")] + /// Invalid Signature + InvalidSignature, + #[error(transparent)] + /// Error parsing [AuthToken] using Postcard + Parsing(#[from] postcard::Error), + #[error("AuthToken already used")] + /// AuthToken already used + AlreadyUsed, +} + +#[cfg(test)] +mod tests { + use crate::{ + auth::TIMESTAMP_WINDOW, capabilities::Capability, crypto::Keypair, timestamp::Timestamp, + }; + + use super::*; + + #[test] + fn v0_id_signable() { + let signer = Keypair::random(); + let capabilities = vec![Capability::root()]; + + let token = AuthToken::sign(&signer, capabilities.clone()); + + let serialized = &token.serialize(); + + let mut id = vec![]; + id.extend_from_slice(&token.timestamp.to_bytes()); + id.extend_from_slice(signer.public_key().as_bytes()); + + assert_eq!(AuthToken::id(token.version, serialized), id.into()); + + assert_eq!( + AuthToken::signable(token.version, serialized), + &serialized[65..] + ) + } + + #[test] + fn sign_verify() { + let signer = Keypair::random(); + let capabilities = vec![Capability::root()]; + + let verifier = AuthVerifier::default(); + + let token = AuthToken::sign(&signer, capabilities.clone()); + + let serialized = &token.serialize(); + + verifier.verify(serialized).unwrap(); + + assert_eq!(token.capabilities, capabilities.into()); + } + + #[test] + fn expired() { + let signer = Keypair::random(); + let capabilities = Capabilities(vec![Capability::root()]); + + let verifier = AuthVerifier::default(); + + let timestamp = (Timestamp::now()) - (TIMESTAMP_WINDOW as u64); + + let mut signable = vec![]; + signable.extend_from_slice(signer.public_key().as_bytes()); + signable.extend_from_slice(&postcard::to_allocvec(&capabilities).unwrap()); + + let signature = signer.sign(&signable); + + let token = AuthToken { + signature, + namespace: *PUBKY_AUTH, + version: 0, + timestamp, + pubky: signer.public_key(), + capabilities, + }; + + let serialized = token.serialize(); + + let result = verifier.verify(&serialized); + + assert_eq!(result, Err(Error::Expired)); + } + + #[test] + fn already_used() { + let signer = Keypair::random(); + let capabilities = vec![Capability::root()]; + + let verifier = AuthVerifier::default(); + + let token = AuthToken::sign(&signer, capabilities.clone()); + + let serialized = &token.serialize(); + + verifier.verify(serialized).unwrap(); + + assert_eq!(token.capabilities, capabilities.into()); + + assert_eq!(verifier.verify(serialized), Err(Error::AlreadyUsed)); + } +} +``` +./pubky-common/src/session.rs +``` +//! Pubky homeserver session struct. + +use pkarr::PublicKey; +use postcard::{from_bytes, to_allocvec}; +use serde::{Deserialize, Serialize}; + +extern crate alloc; +use alloc::vec::Vec; + +use crate::{capabilities::Capability, timestamp::Timestamp}; + +// TODO: add IP address? +// TODO: use https://crates.io/crates/user-agent-parser to parse the session +// and get more informations from the user-agent. +#[derive(Clone, Serialize, Deserialize, Debug, Eq, PartialEq)] +/// Pubky homeserver session struct. +pub struct Session { + version: usize, + pubky: PublicKey, + created_at: u64, + /// User specified name, defaults to the user-agent. + name: String, + user_agent: String, + capabilities: Vec, +} + +impl Session { + /// Create a new session. + pub fn new(pubky: &PublicKey, capabilities: &[Capability], user_agent: Option) -> Self { + Self { + version: 0, + pubky: pubky.clone(), + created_at: Timestamp::now().as_u64(), + capabilities: capabilities.to_vec(), + user_agent: user_agent.as_deref().unwrap_or("").to_string(), + name: user_agent.as_deref().unwrap_or("").to_string(), + } + } + + // === Getters === + + /// Returns the pubky of this session authorizes for. + pub fn pubky(&self) -> &PublicKey { + &self.pubky + } + + /// Returns the capabilities this session provide on this session's pubky's resources. + pub fn capabilities(&self) -> &Vec { + &self.capabilities + } + + // === Setters === + + /// Set this session user agent. + pub fn set_user_agent(&mut self, user_agent: String) -> &mut Self { + self.user_agent = user_agent; + + if self.name.is_empty() { + self.name.clone_from(&self.user_agent) + } + + self + } + + /// Set this session's capabilities. + pub fn set_capabilities(&mut self, capabilities: Vec) -> &mut Self { + self.capabilities = capabilities; + + self + } + + // === Public Methods === + + /// Serialize this session to its canonical binary representation. + pub fn serialize(&self) -> Vec { + to_allocvec(self).expect("Session::serialize") + } + + /// Deserialize this session from its canonical binary representation. + pub fn deserialize(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(Error::EmptyPayload); + } + + if bytes[0] > 0 { + return Err(Error::UnknownVersion); + } + + Ok(from_bytes(bytes)?) + } + + // TODO: add `can_read()`, `can_write()` and `is_root()` methods +} + +#[derive(thiserror::Error, Debug, PartialEq)] +/// Error deserializing a [Session]. +pub enum Error { + #[error("Empty payload")] + /// Empty payload + EmptyPayload, + #[error("Unknown version")] + /// Unknown version + UnknownVersion, + #[error(transparent)] + /// Error parsing the binary representation. + Parsing(#[from] postcard::Error), +} + +#[cfg(test)] +mod tests { + use crate::crypto::Keypair; + + use super::*; + + #[test] + fn serialize() { + let keypair = Keypair::from_secret_key(&[0; 32]); + let pubky = keypair.public_key(); + + let session = Session { + user_agent: "foo".to_string(), + capabilities: vec![Capability::root()], + created_at: 0, + pubky, + version: 0, + name: "".to_string(), + }; + + let serialized = session.serialize(); + + assert_eq!( + serialized, + [ + 0, 59, 106, 39, 188, 206, 182, 164, 45, 98, 163, 168, 208, 42, 111, 13, 115, 101, + 50, 21, 119, 29, 226, 67, 166, 58, 192, 72, 161, 139, 89, 218, 41, 0, 0, 3, 102, + 111, 111, 1, 4, 47, 58, 114, 119 + ] + ); + + let deseiralized = Session::deserialize(&serialized).unwrap(); + + assert_eq!(deseiralized, session) + } + + #[test] + fn deserialize() { + let result = Session::deserialize(&[]); + + assert_eq!(result, Err(Error::EmptyPayload)); + } +} +``` +./pubky-common/src/recovery_file.rs +``` +//! Tools for encrypting and decrypting a recovery file storing user's root key's secret. + +use argon2::Argon2; +use pkarr::Keypair; + +use crate::crypto::{decrypt, encrypt}; + +static SPEC_NAME: &str = "recovery"; +static SPEC_LINE: &str = "pubky.org/recovery"; + +/// Decrypt a recovery file. +pub fn decrypt_recovery_file(recovery_file: &[u8], passphrase: &str) -> Result { + let encryption_key = recovery_file_encryption_key_from_passphrase(passphrase); + + let newline_index = recovery_file + .iter() + .position(|&r| r == 10) + .ok_or(()) + .map_err(|_| Error::RecoveryFileMissingSpecLine)?; + + let spec_line = &recovery_file[..newline_index]; + + if !(spec_line.starts_with(SPEC_LINE.as_bytes()) + || spec_line.starts_with(b"pkarr.org/recovery")) + { + return Err(Error::RecoveryFileVersionNotSupported); + } + + let encrypted = &recovery_file[newline_index + 1..]; + + if encrypted.is_empty() { + return Err(Error::RecoverFileMissingEncryptedSecretKey); + }; + + let decrypted = decrypt(encrypted, &encryption_key)?; + let length = decrypted.len(); + let secret_key: [u8; 32] = decrypted + .try_into() + .map_err(|_| Error::RecoverFileInvalidSecretKeyLength(length))?; + + Ok(Keypair::from_secret_key(&secret_key)) +} + +/// Encrypt a recovery file. +pub fn create_recovery_file(keypair: &Keypair, passphrase: &str) -> Vec { + let encryption_key = recovery_file_encryption_key_from_passphrase(passphrase); + let secret_key = keypair.secret_key(); + + let encrypted_secret_key = encrypt(&secret_key, &encryption_key); + + let mut out = Vec::with_capacity(SPEC_LINE.len() + 1 + encrypted_secret_key.len()); + + out.extend_from_slice(SPEC_LINE.as_bytes()); + out.extend_from_slice(b"\n"); + out.extend_from_slice(&encrypted_secret_key); + + out +} + +fn recovery_file_encryption_key_from_passphrase(passphrase: &str) -> [u8; 32] { + let argon2id = Argon2::default(); + + let mut out = [0; 32]; + + argon2id + .hash_password_into(passphrase.as_bytes(), SPEC_NAME.as_bytes(), &mut out) + .expect("Output is the correct length, so this should be infallible"); + + out +} + +#[derive(thiserror::Error, Debug)] +/// Error decrypting a recovery file +pub enum Error { + // === Recovery file == + #[error("Recovery file should start with a spec line, followed by a new line character")] + /// Recovery file should start with a spec line, followed by a new line character + RecoveryFileMissingSpecLine, + + #[error("Recovery file should start with a spec line, followed by a new line character")] + /// Recovery file should start with a spec line, followed by a new line character + RecoveryFileVersionNotSupported, + + #[error("Recovery file should contain an encrypted secret key after the new line character")] + /// Recovery file should contain an encrypted secret key after the new line character + RecoverFileMissingEncryptedSecretKey, + + #[error("Recovery file encrypted secret key should be 32 bytes, got {0}")] + /// Recovery file encrypted secret key should be 32 bytes, got {0} + RecoverFileInvalidSecretKeyLength(usize), + + #[error(transparent)] + /// Error while decrypting a message + DecryptError(#[from] crate::crypto::DecryptError), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encrypt_decrypt_recovery_file() { + let passphrase = "very secure password"; + let keypair = Keypair::random(); + + let recovery_file = create_recovery_file(&keypair, passphrase); + let recovered = decrypt_recovery_file(&recovery_file, passphrase).unwrap(); + + assert_eq!(recovered.public_key(), keypair.public_key()); + } +} +``` +./pubky-common/src/crypto.rs +``` +//! Cryptographic functions (hashing, encryption, and signatures). + +use crypto_secretbox::{ + aead::{Aead, AeadCore, KeyInit, OsRng}, + XSalsa20Poly1305, +}; +use rand::random; + +pub use pkarr::{Keypair, PublicKey}; + +pub use ed25519_dalek::Signature; + +/// Blake3 Hash. +pub type Hash = blake3::Hash; + +pub use blake3::hash; + +pub use blake3::Hasher; + +/// Create a random hash. +pub fn random_hash() -> Hash { + Hash::from_bytes(random()) +} + +/// Create an array of random bytes with a size `N`. +pub fn random_bytes() -> [u8; N] { + let arr: [u8; N] = random(); + + arr +} + +/// Encrypt a message using `XSalsa20Poly1305`. +pub fn encrypt(plain_text: &[u8], encryption_key: &[u8; 32]) -> Vec { + if plain_text.is_empty() { + return plain_text.to_vec(); + } + + let cipher = XSalsa20Poly1305::new(encryption_key.into()); + let nonce = XSalsa20Poly1305::generate_nonce(&mut OsRng); // unique per message + let ciphertext = cipher + .encrypt(&nonce, plain_text) + .expect("XSalsa20Poly1305 encrypt should be infallible"); + + let mut out: Vec = Vec::with_capacity(nonce.len() + ciphertext.len()); + out.extend_from_slice(nonce.as_slice()); + out.extend_from_slice(&ciphertext); + + out +} + +/// Encrypt an encrypted message using `XSalsa20Poly1305`. +pub fn decrypt(bytes: &[u8], encryption_key: &[u8; 32]) -> Result, DecryptError> { + if bytes.is_empty() { + return Ok(bytes.to_vec()); + } + + let cipher = XSalsa20Poly1305::new(encryption_key.into()); + + if bytes.len() < 24 { + return Err(DecryptError::TooSmall(bytes.len())); + } + + Ok(cipher.decrypt(bytes[..24].into(), &bytes[24..])?) +} + +#[derive(thiserror::Error, Debug)] +/// Error while decrypting a message +pub enum DecryptError { + #[error(transparent)] + /// Failed to decrypt message. + Fail(#[from] crypto_secretbox::Error), + + #[error("Encrypted message too small, expected at least 24 bytes nonce, received {0} bytes")] + /// Encrypted message too small, expected at least 24 bytes nonce, received {0} bytes + TooSmall(usize), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encrypt_decrypt() { + let plain_text = "Plain text!"; + let encryption_key = [0; 32]; + + let encrypted = encrypt(plain_text.as_bytes(), &encryption_key); + let decrypted = decrypt(&encrypted, &encryption_key).unwrap(); + + assert_eq!(decrypted, plain_text.as_bytes()) + } +} +``` +./pubky-common/src/constants.rs +``` +//! Constants used across Pubky. + +/// [Reserved param keys](https://www.rfc-editor.org/rfc/rfc9460#name-initial-contents) for HTTPS Resource Records +pub mod reserved_param_keys { + /// HTTPS (RFC 9460) record's private param key, used to inform browsers + /// about the HTTP port to use when the domain is localhost. + pub const HTTP_PORT: u16 = 65280; +} + +/// Local test network's hardcoded port numbers for local development. +pub mod testnet_ports { + /// The local test network's hardcorded DHT bootstrapping node's port number. + pub const BOOTSTRAP: u16 = 6881; + /// The local test network's hardcorded Pkarr Relay port number. + pub const PKARR_RELAY: u16 = 15411; + /// The local test network's hardcorded HTTP Relay port number. + pub const HTTP_RELAY: u16 = 15412; +} +``` +./pubky-common/src/lib.rs +``` +#![doc = include_str!("../README.md")] +//! + +#![deny(missing_docs)] +#![deny(rustdoc::broken_intra_doc_links)] +#![cfg_attr(any(), deny(clippy::unwrap_used))] + +pub mod auth; +pub mod capabilities; +pub mod constants; +pub mod crypto; +pub mod namespaces; +pub mod recovery_file; +pub mod session; + +pub mod timestamp { + //! Timestamp used across Pubky crates. + pub use pubky_timestamp::*; +} +``` +./pubky-common/src/namespaces.rs +``` +//! Namespaces using to prepend signed messages to avoid collisions. + +/// Pubky Auth namespace as defined at the [spec](https://pubky.github.io/pubky-core/spec/auth.html) +pub const PUBKY_AUTH: &[u8; 10] = b"PUBKY:AUTH"; +``` +./pubky-common/src/capabilities.rs +``` +//! Capabilities defining what scopes of resources can be accessed with what actions. + +use std::fmt::Display; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq)] +/// A Capability defines the scope of resources and the actions that the holder +/// of this capability can access. +pub struct Capability { + /// Scope of resources (for example directories). + pub scope: String, + /// Actions allowed on the [Capability::scope]. + pub actions: Vec, +} + +impl Capability { + /// Create a root [Capability] at the `/` path with all the available [Action]s + pub fn root() -> Self { + Capability { + scope: "/".to_string(), + actions: vec![Action::Read, Action::Write], + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Actions allowed on a given resource or scope of resources. +pub enum Action { + /// Can read the scope at the specified path (GET requests). + Read, + /// Can write to the scope at the specified path (PUT/POST/DELETE requests). + Write, + /// Unknown ability + Unknown(char), +} + +impl From<&Action> for char { + fn from(value: &Action) -> Self { + match value { + Action::Read => 'r', + Action::Write => 'w', + Action::Unknown(char) => char.to_owned(), + } + } +} + +impl TryFrom for Action { + type Error = Error; + + fn try_from(value: char) -> Result { + match value { + 'r' => Ok(Self::Read), + 'w' => Ok(Self::Write), + _ => Err(Error::InvalidAction), + } + } +} + +impl Display for Capability { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}:{}", + self.scope, + self.actions.iter().map(char::from).collect::() + ) + } +} + +impl TryFrom for Capability { + type Error = Error; + + fn try_from(value: String) -> Result { + value.as_str().try_into() + } +} + +impl TryFrom<&str> for Capability { + type Error = Error; + + fn try_from(value: &str) -> Result { + if value.matches(':').count() != 1 { + return Err(Error::InvalidFormat); + } + + if !value.starts_with('/') { + return Err(Error::InvalidScope); + } + + let actions_str = value.rsplit(':').next().unwrap_or(""); + + let mut actions = Vec::new(); + + for char in actions_str.chars() { + let ability = Action::try_from(char)?; + + match actions.binary_search_by(|element| char::from(element).cmp(&char)) { + Ok(_) => {} + Err(index) => { + actions.insert(index, ability); + } + } + } + + let scope = value[0..value.len() - actions_str.len() - 1].to_string(); + + Ok(Capability { scope, actions }) + } +} + +impl Serialize for Capability { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let string = self.to_string(); + + string.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Capability { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let string: String = Deserialize::deserialize(deserializer)?; + + string.try_into().map_err(serde::de::Error::custom) + } +} + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +/// Error parsing a [Capability]. +pub enum Error { + #[error("Capability: Invalid scope: does not start with `/`")] + /// Capability: Invalid scope: does not start with `/` + InvalidScope, + #[error("Capability: Invalid format should be :")] + /// Capability: Invalid format should be `:` + InvalidFormat, + #[error("Capability: Invalid Action")] + /// Capability: Invalid Action + InvalidAction, + #[error("Capabilities: Invalid capabilities format")] + /// Capabilities: Invalid capabilities format + InvalidCapabilities, +} + +#[derive(Clone, Default, Debug, PartialEq, Eq)] +/// A wrapper around `Vec` to enable serialization without +/// a varint. Useful when [Capabilities] are at the end of a struct. +pub struct Capabilities(pub Vec); + +impl Capabilities { + /// Returns true if the list of capabilities contains a given capability. + pub fn contains(&self, capability: &Capability) -> bool { + self.0.contains(capability) + } +} + +impl From> for Capabilities { + fn from(value: Vec) -> Self { + Self(value) + } +} + +impl From for Vec { + fn from(value: Capabilities) -> Self { + value.0 + } +} + +impl TryFrom<&str> for Capabilities { + type Error = Error; + + fn try_from(value: &str) -> Result { + let mut caps = vec![]; + + for s in value.split(',') { + if let Ok(cap) = Capability::try_from(s) { + caps.push(cap); + }; + } + + Ok(Capabilities(caps)) + } +} + +impl Display for Capabilities { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let string = self + .0 + .iter() + .map(|c| c.to_string()) + .collect::>() + .join(","); + + write!(f, "{}", string) + } +} + +impl Serialize for Capabilities { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for Capabilities { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let string: String = Deserialize::deserialize(deserializer)?; + + let mut caps = vec![]; + + for s in string.split(',') { + if let Ok(cap) = Capability::try_from(s) { + caps.push(cap); + }; + } + + Ok(Capabilities(caps)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pubky_caps() { + let cap = Capability { + scope: "/pub/pubky.app/".to_string(), + actions: vec![Action::Read, Action::Write], + }; + + // Read and write within directory `/pub/pubky.app/`. + let expected_string = "/pub/pubky.app/:rw"; + + assert_eq!(cap.to_string(), expected_string); + + assert_eq!(Capability::try_from(expected_string), Ok(cap)) + } +} +``` +./pubky-common/Cargo.toml +``` +[package] +name = "pubky-common" +version = "0.3.1" +edition = "2021" +authors = ["Nuh "] +description = "Types and struct in common between Pubky client and homeserver" +license = "MIT" +homepage = "https://github.com/pubky/pubky-core" +repository = "https://github.com/pubky/pubky-core" +keywords = ["pkarr", "pubky", "auth", "pubkey"] +categories = ["web-programming", "authentication", "cryptography"] + +[dependencies] +base32 = "0.5.1" +blake3 = "1.5.5" +ed25519-dalek = { version = "2.1.1", features = ["serde"] } +once_cell = "1.20.3" +rand = "0.9.0" +thiserror = "2.0.11" +postcard = { version = "1.1.1", features = ["alloc"] } +crypto_secretbox = { version = "0.1.1", features = ["std"] } +argon2 = { version = "0.5.3", features = ["std"] } +pubky-timestamp = { version = "0.4.0", features = ["full"] } +serde = { version = "1.0.217", features = ["derive"] } +pkarr = { version = "3.3.3", default-features = false, features = ["keys"] } + +[target.'cfg(target_arch = "wasm32")'.dependencies] +js-sys = "0.3.77" + +[dev-dependencies] +postcard = "1.1.1" +``` +./Cargo.toml +``` +[workspace] +members = [ + "pubky", + "pubky-*", + + "http-relay", + + "examples" +] + +# See: https://github.com/rust-lang/rust/issues/90148#issuecomment-949194352 +resolver = "2" + +[profile.release] +lto = true +opt-level = 'z' +``` +./pubky-homeserver/src/main.rs +``` +use std::path::PathBuf; + +use anyhow::Result; +use pubky_homeserver::Homeserver; + +use clap::Parser; + +#[derive(Parser, Debug)] +struct Cli { + /// [tracing_subscriber::EnvFilter] + #[clap(short, long)] + tracing_env_filter: Option, + + /// Optional Path to config file. + #[clap(short, long)] + config: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Cli::parse(); + + tracing_subscriber::fmt() + .with_env_filter( + args.tracing_env_filter + .unwrap_or("pubky_homeserver=debug,tower_http=debug".to_string()), + ) + .init(); + + let server = unsafe { + if let Some(config_path) = args.config { + Homeserver::run_with_config_file(config_path).await? + } else { + Homeserver::builder().run().await? + } + }; + + tokio::signal::ctrl_c().await?; + + tracing::info!("Shutting down Homeserver"); + + server.shutdown(); + + Ok(()) +} +``` +./pubky-homeserver/src/core/routes/feed.rs +``` +use axum::{ + body::Body, + extract::State, + http::{header, Response, StatusCode}, + response::IntoResponse, +}; +use pubky_common::timestamp::Timestamp; + +use crate::core::{ + error::{Error, Result}, + extractors::ListQueryParams, + AppState, +}; + +pub async fn feed( + State(state): State, + params: ListQueryParams, +) -> Result { + if let Some(ref cursor) = params.cursor { + if Timestamp::try_from(cursor.to_string()).is_err() { + Err(Error::new( + StatusCode::BAD_REQUEST, + "Cursor should be valid base32 Crockford encoding of a timestamp".into(), + ))? + } + } + + let result = state.db.list_events(params.limit, params.cursor)?; + + Ok(Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/plain") + .body(Body::from(result.join("\n"))) + .unwrap()) +} +``` +./pubky-homeserver/src/core/routes/tenants/write.rs +``` +use std::io::Write; + +use futures_util::stream::StreamExt; + +use axum::{ + body::Body, + extract::{OriginalUri, State}, + http::StatusCode, + response::IntoResponse, +}; + +use crate::core::{ + error::{Error, Result}, + extractors::PubkyHost, + AppState, +}; + +pub async fn delete( + State(mut state): State, + pubky: PubkyHost, + path: OriginalUri, +) -> Result { + let public_key = pubky.public_key().clone(); + + // TODO: should we wrap this with `tokio::task::spawn_blocking` in case it takes too long? + let deleted = state.db.delete_entry(&public_key, path.0.path())?; + + if !deleted { + // TODO: if the path ends with `/` return a `CONFLICT` error? + return Err(Error::with_status(StatusCode::NOT_FOUND)); + }; + + Ok(()) +} + +pub async fn put( + State(mut state): State, + pubky: PubkyHost, + path: OriginalUri, + body: Body, +) -> Result { + let public_key = pubky.public_key().clone(); + + let mut entry_writer = state.db.write_entry(&public_key, path.0.path())?; + + let mut stream = body.into_data_stream(); + while let Some(next) = stream.next().await { + let chunk = next?; + entry_writer.write_all(&chunk)?; + } + + let _entry = entry_writer.commit()?; + + // TODO: return relevant headers, like Etag? + + Ok(()) +} +``` +./pubky-homeserver/src/core/routes/tenants/read.rs +``` +use axum::{ + body::Body, + extract::{OriginalUri, State}, + http::{header, HeaderMap, HeaderValue, Response, StatusCode}, + response::IntoResponse, +}; +use httpdate::HttpDate; +use pkarr::PublicKey; +use std::str::FromStr; + +use crate::core::{ + database::tables::entries::Entry, + error::{Error, Result}, + extractors::{ListQueryParams, PubkyHost}, + AppState, +}; + +pub async fn head( + State(state): State, + pubky: PubkyHost, + headers: HeaderMap, + path: OriginalUri, +) -> Result { + let rtxn = state.db.env.read_txn()?; + + get_entry( + headers, + state + .db + .get_entry(&rtxn, pubky.public_key(), path.0.path())?, + None, + ) +} + +pub async fn get( + State(state): State, + headers: HeaderMap, + pubky: PubkyHost, + path: OriginalUri, + params: ListQueryParams, +) -> Result { + let public_key = pubky.public_key().clone(); + let path = path.0.path().to_string(); + + if path.ends_with('/') { + return list(state, &public_key, &path, params); + } + + let (entry_tx, entry_rx) = flume::bounded::>(1); + let (chunks_tx, chunks_rx) = flume::unbounded::, heed::Error>>(); + + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let rtxn = state.db.env.read_txn()?; + + let option = state.db.get_entry(&rtxn, &public_key, &path)?; + + if let Some(entry) = option { + let iter = entry.read_content(&state.db, &rtxn)?; + + entry_tx.send(Some(entry))?; + + for next in iter { + chunks_tx.send(next.map(|b| b.to_vec()))?; + } + }; + + entry_tx.send(None)?; + + Ok(()) + }); + + get_entry( + headers, + entry_rx.recv_async().await?, + Some(Body::from_stream(chunks_rx.into_stream())), + ) +} + +pub fn list( + state: AppState, + public_key: &PublicKey, + path: &str, + params: ListQueryParams, +) -> Result> { + let txn = state.db.env.read_txn()?; + + let path = format!("{public_key}{path}"); + + if !state.db.contains_directory(&txn, &path)? { + return Err(Error::new( + StatusCode::NOT_FOUND, + "Directory Not Found".into(), + )); + } + + // Handle listing + let vec = state.db.list( + &txn, + &path, + params.reverse, + params.limit, + params.cursor, + params.shallow, + )?; + + Ok(Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/plain") + .body(Body::from(vec.join("\n")))?) +} + +pub fn get_entry( + headers: HeaderMap, + entry: Option, + body: Option, +) -> Result> { + if let Some(entry) = entry { + // TODO: Enable seek API (range requests) + // TODO: Gzip? or brotli? + + let mut response = HeaderMap::from(&entry).into_response(); + + // Handle IF_MODIFIED_SINCE + if let Some(condition_http_date) = headers + .get(header::IF_MODIFIED_SINCE) + .and_then(|h| h.to_str().ok()) + .and_then(|s| HttpDate::from_str(s).ok()) + { + let entry_http_date: HttpDate = entry.timestamp().to_owned().into(); + + if condition_http_date >= entry_http_date { + *response.status_mut() = StatusCode::NOT_MODIFIED; + } + }; + + // Handle IF_NONE_MATCH + if let Some(str) = headers + .get(header::IF_NONE_MATCH) + .and_then(|h| h.to_str().ok()) + { + let etag = format!("\"{}\"", entry.content_hash()); + if str + .trim() + .split(',') + .collect::>() + .contains(&etag.as_str()) + { + *response.status_mut() = StatusCode::NOT_MODIFIED; + }; + } + + if let Some(body) = body { + *response.body_mut() = body; + }; + + Ok(response) + } else { + Err(Error::with_status(StatusCode::NOT_FOUND))? + } +} + +impl From<&Entry> for HeaderMap { + fn from(entry: &Entry) -> Self { + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_LENGTH, entry.content_length().into()); + headers.insert( + header::LAST_MODIFIED, + HeaderValue::from_str(&entry.timestamp().format_http_date()) + .expect("http date is valid header value"), + ); + headers.insert( + header::CONTENT_TYPE, + // TODO: when setting content type from user input, we should validate it as a HeaderValue + entry + .content_type() + .try_into() + .or(HeaderValue::from_str("")) + .expect("valid header value"), + ); + headers.insert( + header::ETAG, + format!("\"{}\"", entry.content_hash()) + .try_into() + .expect("hex string is valid"), + ); + + headers + } +} + +#[cfg(test)] +mod tests { + use axum::{ + body::Body, + http::{header, Method, Request, StatusCode}, + }; + use pkarr::Keypair; + + use crate::core::HomeserverCore; + + #[tokio::test] + async fn if_last_modified() { + let mut server = HomeserverCore::test().unwrap(); + + let keypair = Keypair::random(); + let public_key = keypair.public_key(); + let cookie = server.create_root_user(&keypair).await.unwrap().to_string(); + + let data = vec![1_u8, 2, 3, 4, 5]; + + let response = server + .call( + Request::builder() + .header("host", public_key.to_string()) + .uri("/pub/foo") + .method(Method::PUT) + .header(header::COOKIE, cookie) + .body(Body::from(data)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + let response = server + .call( + Request::builder() + .header("host", public_key.to_string()) + .uri("/pub/foo") + .method(Method::GET) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + let response = server + .call( + Request::builder() + .header("host", public_key.to_string()) + .uri("/pub/foo") + .method(Method::GET) + .header( + header::IF_MODIFIED_SINCE, + response.headers().get(header::LAST_MODIFIED).unwrap(), + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + } + + #[tokio::test] + async fn if_none_match() { + let mut server = HomeserverCore::test().unwrap(); + + let keypair = Keypair::random(); + let public_key = keypair.public_key(); + + let cookie = server.create_root_user(&keypair).await.unwrap().to_string(); + + let data = vec![1_u8, 2, 3, 4, 5]; + + let response = server + .call( + Request::builder() + .uri("/pub/foo") + .header("host", public_key.to_string()) + .method(Method::PUT) + .header(header::COOKIE, cookie) + .body(Body::from(data)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + let response = server + .call( + Request::builder() + .uri("/pub/foo") + .header("host", public_key.to_string()) + .method(Method::GET) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + let response = server + .call( + Request::builder() + .uri("/pub/foo") + .header("host", public_key.to_string()) + .method(Method::GET) + .header( + header::IF_NONE_MATCH, + response.headers().get(header::ETAG).unwrap(), + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + } +} +``` +./pubky-homeserver/src/core/routes/tenants/session.rs +``` +use axum::{extract::State, http::StatusCode, response::IntoResponse}; +use tower_cookies::Cookies; + +use crate::core::{ + error::{Error, Result}, + extractors::PubkyHost, + layers::authz::session_secret_from_cookies, + AppState, +}; + +pub async fn session( + State(state): State, + cookies: Cookies, + pubky: PubkyHost, +) -> Result { + if let Some(secret) = session_secret_from_cookies(&cookies, pubky.public_key()) { + if let Some(session) = state.db.get_session(&secret)? { + // TODO: add content-type + return Ok(session.serialize()); + }; + } + + Err(Error::with_status(StatusCode::NOT_FOUND)) +} +pub async fn signout( + State(mut state): State, + cookies: Cookies, + pubky: PubkyHost, +) -> Result { + // TODO: Set expired cookie to delete the cookie on client side. + + if let Some(secret) = session_secret_from_cookies(&cookies, pubky.public_key()) { + state.db.delete_session(&secret)?; + } + + // Idempotent Success Response (200 OK) + Ok(()) +} +``` +./pubky-homeserver/src/core/routes/tenants/mod.rs +``` +//! Per Tenant (user / Pubky) routes. +//! +//! Every route here is relative to a tenant's Pubky host, +//! as opposed to routes relative to the Homeserver's owner. + +use axum::{ + extract::DefaultBodyLimit, + routing::{delete, get, head, put}, + Router, +}; + +use crate::core::{layers::authz::AuthorizationLayer, AppState}; + +pub mod read; +pub mod session; +pub mod write; + +pub fn router(state: AppState) -> Router { + Router::new() + // - Datastore routes + .route("/pub/", get(read::get)) + .route("/pub/{*path}", get(read::get)) + .route("/pub/{*path}", head(read::head)) + .route("/pub/{*path}", put(write::put)) + .route("/pub/{*path}", delete(write::delete)) + // - Session routes + .route("/session", get(session::session)) + .route("/session", delete(session::signout)) + // Layers + // TODO: different max size for sessions and other routes? + .layer(DefaultBodyLimit::max(100 * 1024 * 1024)) + .layer(AuthorizationLayer::new(state.clone())) +} +``` +./pubky-homeserver/src/core/routes/auth.rs +``` +use axum::{extract::State, response::IntoResponse}; +use axum_extra::{extract::Host, headers::UserAgent, TypedHeader}; +use bytes::Bytes; +use tower_cookies::{cookie::SameSite, Cookie, Cookies}; + +use pubky_common::{crypto::random_bytes, session::Session, timestamp::Timestamp}; + +use crate::core::{database::tables::users::User, error::Result, AppState}; + +pub async fn signup( + State(state): State, + user_agent: Option>, + cookies: Cookies, + host: Host, + body: Bytes, +) -> Result { + // TODO: Verify invitation link. + // TODO: add errors in case of already axisting user. + signin(State(state), user_agent, cookies, host, body).await +} + +pub async fn signin( + State(state): State, + user_agent: Option>, + cookies: Cookies, + Host(host): Host, + body: Bytes, +) -> Result { + let token = state.verifier.verify(&body)?; + + let public_key = token.pubky(); + + let mut wtxn = state.db.env.write_txn()?; + + let users = state.db.tables.users; + if let Some(existing) = users.get(&wtxn, public_key)? { + // TODO: why do we need this? + users.put(&mut wtxn, public_key, &existing)?; + } else { + users.put( + &mut wtxn, + public_key, + &User { + created_at: Timestamp::now().as_u64(), + }, + )?; + } + + let session_secret = base32::encode(base32::Alphabet::Crockford, &random_bytes::<16>()); + + let session = Session::new( + token.pubky(), + token.capabilities(), + user_agent.map(|ua| ua.to_string()), + ) + .serialize(); + + state + .db + .tables + .sessions + .put(&mut wtxn, &session_secret, &session)?; + + wtxn.commit()?; + + let mut cookie = Cookie::new(public_key.to_string(), session_secret); + + cookie.set_path("/"); + + // TODO: do we even have insecure anymore? + if is_secure(&host) { + cookie.set_secure(true); + cookie.set_same_site(SameSite::None); + } + cookie.set_http_only(true); + + cookies.add(cookie); + + Ok(session) +} + +/// Assuming that if the server is addressed by anything other than +/// localhost, or IP addresses, it is not addressed from a browser in an +/// secure (HTTPs) window, thus it no need to `secure` and `same_site=none` to cookies +fn is_secure(host: &str) -> bool { + url::Host::parse(host) + .map(|host| match host { + url::Host::Domain(domain) => domain != "localhost", + _ => false, + }) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use pkarr::Keypair; + + use super::*; + + #[test] + fn test_is_secure() { + assert!(!is_secure("")); + assert!(!is_secure("127.0.0.1")); + assert!(!is_secure("167.86.102.121")); + assert!(!is_secure("[2001:0db8:0000:0000:0000:ff00:0042:8329]")); + assert!(!is_secure("localhost")); + assert!(!is_secure("localhost:23423")); + assert!(is_secure(&Keypair::random().public_key().to_string())); + assert!(is_secure("example.com")); + } +} +``` +./pubky-homeserver/src/core/routes/mod.rs +``` +//! The controller part of the [super::HomeserverCore] + +use axum::{ + body::Body, + extract::Request, + http::{header, HeaderValue}, + middleware::{self, Next}, + response::Response, + routing::{get, post}, + Router, +}; +use tower::ServiceBuilder; +use tower_cookies::CookieManagerLayer; +use tower_http::cors::CorsLayer; + +use crate::core::AppState; + +use super::layers::{pubky_host::PubkyHostLayer, trace::with_trace_layer}; + +mod auth; +mod feed; +mod root; +mod tenants; + +static HOMESERVER_VERSION: &str = concat!("pubky.org", "@", env!("CARGO_PKG_VERSION"),); +const TRACING_EXCLUDED_PATHS: [&str; 1] = ["/events/"]; + +fn base() -> Router { + Router::new() + .route("/", get(root::handler)) + .route("/signup", post(auth::signup)) + .route("/session", post(auth::signin)) + // Events + .route("/events/", get(feed::feed)) + // TODO: add size limit + // TODO: revisit if we enable streaming big payloads + // TODO: maybe add to a separate router (drive router?). +} + +pub fn create_app(state: AppState) -> Router { + let app = base() + .merge(tenants::router(state.clone())) + .layer(CookieManagerLayer::new()) + .layer(CorsLayer::very_permissive()) + .layer(ServiceBuilder::new().layer(middleware::from_fn(add_server_header))) + .with_state(state); + + with_trace_layer(app, &TRACING_EXCLUDED_PATHS).layer(PubkyHostLayer) +} + +// Middleware to add a `Server` header to all responses +async fn add_server_header(request: Request, next: Next) -> Response { + let mut response = next.run(request).await; + + // Add a custom header to the response + response + .headers_mut() + .insert(header::SERVER, HeaderValue::from_static(HOMESERVER_VERSION)); + + response +} +``` +./pubky-homeserver/src/core/routes/root.rs +``` +use axum::response::IntoResponse; + +pub async fn handler() -> Result { + Ok("This a Pubky homeserver.".to_string()) +} +``` +./pubky-homeserver/src/core/extractors.rs +``` +use std::{collections::HashMap, fmt::Display}; + +use axum::{ + extract::{FromRequestParts, Query}, + http::{request::Parts, StatusCode}, + response::{IntoResponse, Response}, + RequestPartsExt, +}; + +use pkarr::PublicKey; + +use crate::core::error::Result; + +#[derive(Debug, Clone)] +pub struct PubkyHost(pub(crate) PublicKey); + +impl PubkyHost { + pub fn public_key(&self) -> &PublicKey { + &self.0 + } +} + +impl Display for PubkyHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl FromRequestParts for PubkyHost +where + S: Sync + Send, +{ + type Rejection = Response; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + let pubky_host = parts + .extensions + .get::() + .cloned() + .ok_or(( + StatusCode::INTERNAL_SERVER_ERROR, + "Can't extract PubkyHost. Is `PubkyHostLayer` enabled?", + )) + .map_err(|e| e.into_response())?; + + Ok(pubky_host) + } +} + +#[derive(Debug)] +pub struct ListQueryParams { + pub limit: Option, + pub cursor: Option, + pub reverse: bool, + pub shallow: bool, +} + +impl FromRequestParts for ListQueryParams +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + let params: Query> = + parts.extract().await.map_err(IntoResponse::into_response)?; + + let reverse = params.contains_key("reverse"); + let shallow = params.contains_key("shallow"); + let limit = params + .get("limit") + // Treat `limit=` as None + .and_then(|l| if l.is_empty() { None } else { Some(l) }) + .and_then(|l| l.parse::().ok()); + let cursor = params + .get("cursor") + .map(|c| c.as_str()) + // Treat `cursor=` as None + .and_then(|c| { + if c.is_empty() { + None + } else { + Some(c.to_string()) + } + }); + + Ok(ListQueryParams { + reverse, + shallow, + limit, + cursor, + }) + } +} +``` +./pubky-homeserver/src/core/mod.rs +``` +use std::path::PathBuf; + +use anyhow::Result; +use axum::Router; +use pubky_common::auth::AuthVerifier; + +pub mod database; +mod error; +mod extractors; +mod layers; +mod routes; + +use crate::config::{ + DEFAULT_LIST_LIMIT, DEFAULT_MAP_SIZE, DEFAULT_MAX_LIST_LIMIT, DEFAULT_STORAGE_DIR, +}; + +use database::DB; + +#[derive(Clone, Debug)] +pub(crate) struct AppState { + pub(crate) verifier: AuthVerifier, + pub(crate) db: DB, +} + +#[derive(Debug, Clone)] +/// A side-effect-free Core of the [crate::Homeserver]. +pub struct HomeserverCore { + pub(crate) router: Router, +} + +impl HomeserverCore { + /// Create a side-effect-free Homeserver core. + /// + /// # Safety + /// HomeserverCore uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub unsafe fn new(config: CoreConfig) -> Result { + let db = unsafe { DB::open(config.clone())? }; + + let state = AppState { + verifier: AuthVerifier::default(), + db, + }; + + let router = routes::create_app(state.clone()); + + Ok(Self { router }) + } +} + +#[cfg(test)] +mod tests { + + use anyhow::Result; + use axum::{ + body::Body, + extract::Request, + http::{header, Method}, + response::Response, + }; + use pkarr::Keypair; + use pubky_common::{auth::AuthToken, capabilities::Capability}; + use tower::ServiceExt; + + use super::*; + + impl HomeserverCore { + /// Test version of [HomeserverCore::new], using an ephemeral small storage. + pub fn test() -> Result { + unsafe { HomeserverCore::new(CoreConfig::test()) } + } + + // === Public Methods === + + pub async fn create_root_user(&mut self, keypair: &Keypair) -> Result { + let auth_token = AuthToken::sign(keypair, vec![Capability::root()]); + + let response = self + .call( + Request::builder() + .uri("/signup") + .header("host", keypair.public_key().to_string()) + .method(Method::POST) + .body(Body::from(auth_token.serialize())) + .unwrap(), + ) + .await?; + + let header_value = response + .headers() + .get(header::SET_COOKIE) + .and_then(|h| h.to_str().ok()) + .expect("should return a set-cookie header") + .to_string(); + + Ok(header_value) + } + + pub async fn call(&self, request: Request) -> Result { + Ok(self.router.clone().oneshot(request).await?) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Database configurations +pub struct CoreConfig { + /// Path to the storage directory. + /// + /// Defaults to a directory in the OS data directory + pub storage: PathBuf, + pub db_map_size: usize, + + /// The default limit of a list api if no `limit` query parameter is provided. + /// + /// Defaults to `100` + pub default_list_limit: u16, + /// The maximum limit of a list api, even if a `limit` query parameter is provided. + /// + /// Defaults to `1000` + pub max_list_limit: u16, +} + +impl Default for CoreConfig { + fn default() -> Self { + Self { + storage: storage(None) + .expect("operating environment provides no directory for application data"), + db_map_size: DEFAULT_MAP_SIZE, + + default_list_limit: DEFAULT_LIST_LIMIT, + max_list_limit: DEFAULT_MAX_LIST_LIMIT, + } + } +} + +impl CoreConfig { + pub fn test() -> Self { + let storage = std::env::temp_dir() + .join(pubky_common::timestamp::Timestamp::now().to_string()) + .join(DEFAULT_STORAGE_DIR); + + Self { + storage, + db_map_size: 10485760, + + ..Default::default() + } + } +} + +pub fn storage(storage: Option) -> Result { + let dir = if let Some(storage) = storage { + PathBuf::from(storage) + } else { + let path = dirs_next::data_dir().ok_or_else(|| { + anyhow::anyhow!("operating environment provides no directory for application data") + })?; + path.join(DEFAULT_STORAGE_DIR) + }; + + Ok(dir.join("homeserver")) +} +``` +./pubky-homeserver/src/core/layers/trace.rs +``` +use std::sync::Arc; + +use axum::{extract::Request, Router}; +use tower_http::trace::{ + DefaultOnFailure, DefaultOnRequest, DefaultOnResponse, OnFailure, OnRequest, OnResponse, + TraceLayer, +}; +use tracing::{Level, Span}; + +use crate::core::extractors::PubkyHost; + +pub fn with_trace_layer(router: Router, excluded_paths: &[&str]) -> Router { + let excluded_paths = Arc::new( + excluded_paths + .iter() + .map(|s| s.to_string()) + .collect::>(), + ); + + router.layer( + TraceLayer::new_for_http() + .make_span_with(move |request: &Request| { + if excluded_paths.contains(&request.uri().path().to_string()) { + // Skip logging for the noisy endpoint + tracing::span!(Level::INFO, "request", excluded = true) + } else { + // Use the default span for other endpoints + + let uri = if let Some(pubky_host) = request.extensions().get::() { + format!("pubky://{pubky_host}{}", request.uri()) + } else { + request.uri().to_string() + }; + + tracing::span!( + Level::INFO, + "request", + method = %request.method(), + uri = ?uri, + version = ?request.version(), + ) + } + }) + .on_request(|request: &Request, span: &Span| { + // Skip logging for excluded spans + if span.has_field("excluded") { + return; + } + // Use the default behavior for other spans + DefaultOnRequest::new().on_request(request, span); + }) + .on_response( + |response: &axum::response::Response, latency: std::time::Duration, span: &Span| { + // Skip logging for excluded spans + if span.has_field("excluded") { + return; + } + // Use the default behavior for other spans + DefaultOnResponse::new().on_response(response, latency, span); + }, + ) + .on_failure( + |error: tower_http::classify::ServerErrorsFailureClass, + latency: std::time::Duration, + span: &Span| { + // Skip logging for excluded spans + if span.has_field("excluded") { + return; + } + // Use the default behavior for other spans + DefaultOnFailure::new().on_failure(error, latency, span); + }, + ), + ) +} +``` +./pubky-homeserver/src/core/layers/pubky_host.rs +``` +use pkarr::PublicKey; + +use crate::core::extractors::PubkyHost; + +use axum::{body::Body, http::Request}; +use futures_util::future::BoxFuture; +use std::{convert::Infallible, task::Poll}; +use tower::{Layer, Service}; + +use crate::core::error::Result; + +/// A Tower Layer to handle authorization for write operations. +#[derive(Debug, Clone)] +pub struct PubkyHostLayer; + +impl Layer for PubkyHostLayer { + type Service = PubkyHostLayerMiddleware; + + fn layer(&self, inner: S) -> Self::Service { + PubkyHostLayerMiddleware { inner } + } +} + +/// Middleware that performs authorization checks for write operations. +#[derive(Debug, Clone)] +pub struct PubkyHostLayerMiddleware { + inner: S, +} + +impl Service> for PubkyHostLayerMiddleware +where + S: Service, Response = axum::response::Response, Error = Infallible> + + Send + + 'static + + Clone, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = Infallible; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(|_| unreachable!()) // `Infallible` conversion + } + + fn call(&mut self, req: Request) -> Self::Future { + let mut inner = self.inner.clone(); + let mut req = req; + + Box::pin(async move { + let headers_to_check = ["host", "pubky-host"]; + + for header in headers_to_check { + if let Some(Ok(pubky_host)) = req.headers().get(header).map(|h| h.to_str()) { + if let Ok(public_key) = PublicKey::try_from(pubky_host) { + req.extensions_mut().insert(PubkyHost(public_key)); + } + } + } + + inner.call(req).await.map_err(|_| unreachable!()) + }) + } +} +``` +./pubky-homeserver/src/core/layers/authz.rs +``` +use axum::http::Method; +use axum::response::IntoResponse; +use axum::{ + body::Body, + http::{Request, StatusCode}, +}; +use futures_util::future::BoxFuture; +use pkarr::PublicKey; +use std::{convert::Infallible, task::Poll}; +use tower::{Layer, Service}; +use tower_cookies::Cookies; + +use crate::core::{ + error::{Error, Result}, + extractors::PubkyHost, + AppState, +}; + +/// A Tower Layer to handle authorization for write operations. +#[derive(Debug, Clone)] +pub struct AuthorizationLayer { + state: AppState, +} + +impl AuthorizationLayer { + pub fn new(state: AppState) -> Self { + Self { state } + } +} + +impl Layer for AuthorizationLayer { + type Service = AuthorizationMiddleware; + + fn layer(&self, inner: S) -> Self::Service { + AuthorizationMiddleware { + inner, + state: self.state.clone(), + } + } +} + +/// Middleware that performs authorization checks for write operations. +#[derive(Debug, Clone)] +pub struct AuthorizationMiddleware { + inner: S, + state: AppState, +} + +impl Service> for AuthorizationMiddleware +where + S: Service, Response = axum::response::Response, Error = Infallible> + + Send + + 'static + + Clone, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = S::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(|_| unreachable!()) // `Infallible` conversion + } + + fn call(&mut self, req: Request) -> Self::Future { + let state = self.state.clone(); + let mut inner = self.inner.clone(); + + Box::pin(async move { + let path = req.uri().path(); + + let pubky = match req.extensions().get::() { + Some(pk) => pk, + None => { + return Ok( + Error::new(StatusCode::NOT_FOUND, "Pubky Host is missing".into()) + .into_response(), + ) + } + }; + + let cookies = req.extensions().get::(); + + // Authorize the request + if let Err(e) = authorize(&state, req.method(), cookies, pubky.public_key(), path) { + return Ok(e.into_response()); + } + + // If authorized, proceed to the inner service + inner.call(req).await.map_err(|_| unreachable!()) + }) + } +} + +/// Authorize write (PUT or DELETE) for Public paths. +fn authorize( + state: &AppState, + method: &Method, + cookies: Option<&Cookies>, + public_key: &PublicKey, + path: &str, +) -> Result<()> { + if path == "/session" { + // Checking (or deleting) one's session is ok for everyone + return Ok(()); + } else if path.starts_with("/pub/") { + if method == Method::GET { + return Ok(()); + } + } else { + return Err(Error::new( + StatusCode::FORBIDDEN, + "Writing to directories other than '/pub/' is forbidden".into(), + )); + } + + if let Some(cookies) = cookies { + let session_secret = session_secret_from_cookies(cookies, public_key) + .ok_or(Error::with_status(StatusCode::UNAUTHORIZED))?; + + let session = state + .db + .get_session(&session_secret)? + .ok_or(Error::with_status(StatusCode::UNAUTHORIZED))?; + + if session.pubky() == public_key + && session.capabilities().iter().any(|cap| { + path.starts_with(&cap.scope) + && cap + .actions + .contains(&pubky_common::capabilities::Action::Write) + }) + { + return Ok(()); + } + + return Err(Error::with_status(StatusCode::FORBIDDEN)); + } + + Err(Error::with_status(StatusCode::UNAUTHORIZED)) +} + +pub fn session_secret_from_cookies(cookies: &Cookies, public_key: &PublicKey) -> Option { + cookies + .get(&public_key.to_string()) + .map(|c| c.value().to_string()) +} +``` +./pubky-homeserver/src/core/layers/mod.rs +``` +pub mod authz; +pub mod pubky_host; +pub mod trace; +``` +./pubky-homeserver/src/core/database/tables.rs +``` +pub mod blobs; +pub mod entries; +pub mod events; +pub mod sessions; +pub mod users; + +use heed::{Env, RwTxn}; + +use blobs::{BlobsTable, BLOBS_TABLE}; +use entries::{EntriesTable, ENTRIES_TABLE}; + +use self::{ + events::{EventsTable, EVENTS_TABLE}, + sessions::{SessionsTable, SESSIONS_TABLE}, + users::{UsersTable, USERS_TABLE}, +}; + +pub const TABLES_COUNT: u32 = 5; + +#[derive(Debug, Clone)] +pub struct Tables { + pub users: UsersTable, + pub sessions: SessionsTable, + pub blobs: BlobsTable, + pub entries: EntriesTable, + pub events: EventsTable, +} + +impl Tables { + pub fn new(env: &Env, wtxn: &mut RwTxn) -> anyhow::Result { + Ok(Self { + users: env + .open_database(wtxn, Some(USERS_TABLE))? + .expect("Users table already created"), + sessions: env + .open_database(wtxn, Some(SESSIONS_TABLE))? + .expect("Sessions table already created"), + blobs: env + .open_database(wtxn, Some(BLOBS_TABLE))? + .expect("Blobs table already created"), + entries: env + .open_database(wtxn, Some(ENTRIES_TABLE))? + .expect("Entries table already created"), + events: env + .open_database(wtxn, Some(EVENTS_TABLE))? + .expect("Events table already created"), + }) + } +} +``` +./pubky-homeserver/src/core/database/migrations.rs +``` +use heed::Env; + +mod m0; + +use super::tables::Tables; + +pub fn run(env: &Env) -> anyhow::Result { + let mut wtxn = env.write_txn()?; + + m0::run(env, &mut wtxn)?; + + let tables = Tables::new(env, &mut wtxn)?; + + wtxn.commit()?; + + Ok(tables) +} +``` +./pubky-homeserver/src/core/database/mod.rs +``` +//! Internal database in [super::HomeserverCore] + +use std::{fs, path::PathBuf}; + +use heed::{Env, EnvOpenOptions}; + +mod migrations; +pub mod tables; + +use tables::{Tables, TABLES_COUNT}; + +pub use protected::DB; + +/// Protecting fields from being mutated by modules in crate::database +mod protected { + + use crate::core::CoreConfig; + + use super::*; + + #[derive(Debug, Clone)] + pub struct DB { + pub(crate) env: Env, + pub(crate) tables: Tables, + pub(crate) buffers_dir: PathBuf, + pub(crate) max_chunk_size: usize, + config: CoreConfig, + } + + impl DB { + /// # Safety + /// DB uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub unsafe fn open(config: CoreConfig) -> anyhow::Result { + let buffers_dir = config.storage.clone().join("buffers"); + + // Cleanup buffers. + let _ = fs::remove_dir(&buffers_dir); + fs::create_dir_all(&buffers_dir)?; + + let env = unsafe { + EnvOpenOptions::new() + .max_dbs(TABLES_COUNT) + .map_size(config.db_map_size) + .open(&config.storage) + }?; + + let tables = migrations::run(&env)?; + + let db = DB { + env, + tables, + config, + buffers_dir, + max_chunk_size: max_chunk_size(), + }; + + Ok(db) + } + + // Create an ephemeral database for testing purposes. + pub fn test() -> DB { + unsafe { DB::open(CoreConfig::test()).unwrap() } + } + + // === Getters === + + pub fn config(&self) -> &CoreConfig { + &self.config + } + } +} + +/// calculate optimal chunk size: +/// - +/// - +fn max_chunk_size() -> usize { + let page_size = page_size::get(); + + // - 16 bytes Header per page (LMDB) + // - Each page has to contain 2 records + // - 8 bytes per record (LMDB) (empirically, it seems to be 10 not 8) + // - 12 bytes key: + // - timestamp : 8 bytes + // - chunk index: 4 bytes + ((page_size - 16) / 2) - (8 + 2) - 12 +} +``` +./pubky-homeserver/src/core/database/tables/sessions.rs +``` +use heed::{ + types::{Bytes, Str}, + Database, +}; +use pubky_common::session::Session; + +use crate::core::database::DB; + +/// session secret => Session. +pub type SessionsTable = Database; + +pub const SESSIONS_TABLE: &str = "sessions"; + +impl DB { + pub fn get_session(&self, session_secret: &str) -> anyhow::Result> { + let rtxn = self.env.read_txn()?; + + let session = self + .tables + .sessions + .get(&rtxn, session_secret)? + .map(|s| s.to_vec()); + + rtxn.commit()?; + + if let Some(bytes) = session { + return Ok(Some(Session::deserialize(&bytes)?)); + }; + + Ok(None) + } + + pub fn delete_session(&mut self, secret: &str) -> anyhow::Result { + let mut wtxn = self.env.write_txn()?; + + let deleted = self.tables.sessions.delete(&mut wtxn, secret)?; + + wtxn.commit()?; + + Ok(deleted) + } +} +``` +./pubky-homeserver/src/core/database/tables/blobs.rs +``` +use heed::{types::Bytes, Database, RoTxn}; + +use crate::core::database::DB; + +use super::entries::Entry; + +/// (entry timestamp | chunk_index BE) => bytes +pub type BlobsTable = Database; + +pub const BLOBS_TABLE: &str = "blobs"; + +impl DB { + pub fn read_entry_content<'txn>( + &self, + rtxn: &'txn RoTxn, + entry: &Entry, + ) -> anyhow::Result> + 'txn> { + Ok(self + .tables + .blobs + .prefix_iter(rtxn, &entry.timestamp().to_bytes())? + .map(|i| i.map(|(_, bytes)| bytes))) + } +} +``` +./pubky-homeserver/src/core/database/tables/entries.rs +``` +use pkarr::PublicKey; +use postcard::{from_bytes, to_allocvec}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::File, + io::{Read, Write}, + path::PathBuf, +}; +use tracing::instrument; + +use heed::{ + types::{Bytes, Str}, + Database, RoTxn, +}; + +use pubky_common::{ + crypto::{Hash, Hasher}, + timestamp::Timestamp, +}; + +use crate::core::database::DB; + +use super::events::Event; + +/// full_path(pubky/*path) => Entry. +pub type EntriesTable = Database; + +pub const ENTRIES_TABLE: &str = "entries"; + +impl DB { + /// Write an entry by an author at a given path. + /// + /// The path has to start with a forward slash `/` + pub fn write_entry( + &mut self, + public_key: &PublicKey, + path: &str, + ) -> anyhow::Result { + EntryWriter::new(self, public_key, path) + } + + /// Delete an entry by an author at a given path. + /// + /// The path has to start with a forward slash `/` + pub fn delete_entry(&mut self, public_key: &PublicKey, path: &str) -> anyhow::Result { + let mut wtxn = self.env.write_txn()?; + + let key = format!("{public_key}{path}"); + + let deleted = if let Some(bytes) = self.tables.entries.get(&wtxn, &key)? { + let entry = Entry::deserialize(bytes)?; + + let mut deleted_chunks = false; + + { + let mut iter = self + .tables + .blobs + .prefix_iter_mut(&mut wtxn, &entry.timestamp.to_bytes())?; + + while iter.next().is_some() { + unsafe { + deleted_chunks = iter.del_current()?; + } + } + } + + let deleted_entry = self.tables.entries.delete(&mut wtxn, &key)?; + + // create DELETE event + if path.starts_with("/pub/") { + let url = format!("pubky://{key}"); + + let event = Event::delete(&url); + let value = event.serialize(); + + let key = Timestamp::now().to_string(); + + self.tables.events.put(&mut wtxn, &key, &value)?; + + // TODO: delete events older than a threshold. + // TODO: move to events.rs + } + + deleted_entry && deleted_chunks + } else { + false + }; + + wtxn.commit()?; + + Ok(deleted) + } + + pub fn get_entry( + &self, + txn: &RoTxn, + public_key: &PublicKey, + path: &str, + ) -> anyhow::Result> { + let key = format!("{public_key}{path}"); + + if let Some(bytes) = self.tables.entries.get(txn, &key)? { + return Ok(Some(Entry::deserialize(bytes)?)); + } + + Ok(None) + } + + pub fn contains_directory(&self, txn: &RoTxn, path: &str) -> anyhow::Result { + Ok(self.tables.entries.get_greater_than(txn, path)?.is_some()) + } + + /// Return a list of pubky urls. + /// + /// - limit defaults to [crate::config::DEFAULT_LIST_LIMIT] and capped by [crate::config::DEFAULT_MAX_LIST_LIMIT] + pub fn list( + &self, + txn: &RoTxn, + path: &str, + reverse: bool, + limit: Option, + cursor: Option, + shallow: bool, + ) -> anyhow::Result> { + // Vector to store results + let mut results = Vec::new(); + + let limit = limit + .unwrap_or(self.config().default_list_limit) + .min(self.config().max_list_limit); + + // TODO: make this more performant than split and allocations? + + let mut threshold = cursor + .map(|cursor| { + // Removing leading forward slashes + let mut file_or_directory = cursor.trim_start_matches('/'); + + if cursor.starts_with("pubky://") { + file_or_directory = cursor.split(path).last().expect("should not be reachable") + }; + + next_threshold( + path, + file_or_directory, + file_or_directory.ends_with('/'), + reverse, + shallow, + ) + }) + .unwrap_or(next_threshold(path, "", false, reverse, shallow)); + + for _ in 0..limit { + if let Some((key, _)) = if reverse { + self.tables.entries.get_lower_than(txn, &threshold)? + } else { + self.tables.entries.get_greater_than(txn, &threshold)? + } { + if !key.starts_with(path) { + break; + } + + if shallow { + let mut split = key[path.len()..].split('/'); + let file_or_directory = split.next().expect("should not be reachable"); + + let is_directory = split.next().is_some(); + + threshold = + next_threshold(path, file_or_directory, is_directory, reverse, shallow); + + results.push(format!( + "pubky://{path}{file_or_directory}{}", + if is_directory { "/" } else { "" } + )); + } else { + threshold = key.to_string(); + results.push(format!("pubky://{}", key)) + } + }; + } + + Ok(results) + } +} + +/// Calculate the next threshold +#[instrument] +fn next_threshold( + path: &str, + file_or_directory: &str, + is_directory: bool, + reverse: bool, + shallow: bool, +) -> String { + format!( + "{path}{file_or_directory}{}", + if file_or_directory.is_empty() { + // No file_or_directory, early return + if reverse { + // `path/to/dir/\x7f` to catch all paths than `path/to/dir/` + "\x7f" + } else { + "" + } + } else if shallow & is_directory { + if reverse { + // threshold = `path/to/dir\x2e`, since `\x2e` is lower than `/` + "\x2e" + } else { + //threshold = `path/to/dir\x7f`, since `\x7f` is greater than `/` + "\x7f" + } + } else { + "" + } + ) +} + +#[derive(Clone, Default, Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct Entry { + /// Encoding version + version: usize, + /// Modified at + timestamp: Timestamp, + content_hash: EntryHash, + content_length: usize, + content_type: String, + // user_metadata: ? +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct EntryHash(Hash); + +impl Default for EntryHash { + fn default() -> Self { + Self(Hash::from_bytes([0; 32])) + } +} + +impl Serialize for EntryHash { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let bytes = self.0.as_bytes(); + bytes.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for EntryHash { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes: [u8; 32] = Deserialize::deserialize(deserializer)?; + Ok(Self(Hash::from_bytes(bytes))) + } +} + +impl Entry { + pub fn new() -> Self { + Default::default() + } + + // === Setters === + + pub fn set_timestamp(&mut self, timestamp: &Timestamp) -> &mut Self { + self.timestamp = *timestamp; + self + } + + pub fn set_content_hash(&mut self, content_hash: Hash) -> &mut Self { + EntryHash(content_hash).clone_into(&mut self.content_hash); + self + } + + pub fn set_content_length(&mut self, content_length: usize) -> &mut Self { + self.content_length = content_length; + self + } + + // === Getters === + + pub fn timestamp(&self) -> &Timestamp { + &self.timestamp + } + + pub fn content_hash(&self) -> &Hash { + &self.content_hash.0 + } + + pub fn content_length(&self) -> usize { + self.content_length + } + + pub fn content_type(&self) -> &str { + &self.content_type + } + + // === Public Method === + + pub fn read_content<'txn>( + &self, + db: &'txn DB, + rtxn: &'txn RoTxn, + ) -> anyhow::Result> + 'txn> { + db.read_entry_content(rtxn, self) + } + + pub fn serialize(&self) -> Vec { + to_allocvec(self).expect("Session::serialize") + } + + pub fn deserialize(bytes: &[u8]) -> core::result::Result { + if bytes[0] > 0 { + panic!("Unknown Entry version"); + } + + from_bytes(bytes) + } +} + +pub struct EntryWriter<'db> { + db: &'db DB, + buffer: File, + hasher: Hasher, + buffer_path: PathBuf, + entry_key: String, + timestamp: Timestamp, + is_public: bool, +} + +impl<'db> EntryWriter<'db> { + pub fn new(db: &'db DB, public_key: &PublicKey, path: &str) -> anyhow::Result { + let hasher = Hasher::new(); + + let timestamp = Timestamp::now(); + + let buffer_path = db.buffers_dir.join(timestamp.to_string()); + + let buffer = File::create(&buffer_path)?; + + let entry_key = format!("{public_key}{path}"); + + Ok(Self { + db, + buffer, + hasher, + buffer_path, + entry_key, + timestamp, + is_public: path.starts_with("/pub/"), + }) + } + + /// Same ase [EntryWriter::write_all] but returns a Result of a mutable reference of itself + /// to enable chaining with [Self::commit]. + pub fn update(&mut self, chunk: &[u8]) -> Result<&mut Self, std::io::Error> { + self.write_all(chunk)?; + + Ok(self) + } + + /// Commit blob from the filesystem buffer to LMDB, + /// write the [Entry], and commit the write transaction. + pub fn commit(&self) -> anyhow::Result { + let hash = self.hasher.finalize(); + + let mut buffer = File::open(&self.buffer_path)?; + + let mut wtxn = self.db.env.write_txn()?; + + let mut chunk_key = [0; 12]; + chunk_key[0..8].copy_from_slice(&self.timestamp.to_bytes()); + + let mut chunk_index: u32 = 0; + + loop { + let mut chunk = vec![0_u8; self.db.max_chunk_size]; + + let bytes_read = buffer.read(&mut chunk)?; + + if bytes_read == 0 { + break; // EOF reached + } + + chunk_key[8..].copy_from_slice(&chunk_index.to_be_bytes()); + + self.db + .tables + .blobs + .put(&mut wtxn, &chunk_key, &chunk[..bytes_read])?; + + chunk_index += 1; + } + + let mut entry = Entry::new(); + entry.set_timestamp(&self.timestamp); + + entry.set_content_hash(hash); + + let length = buffer.metadata()?.len(); + entry.set_content_length(length as usize); + + self.db + .tables + .entries + .put(&mut wtxn, &self.entry_key, &entry.serialize())?; + + // Write a public [Event]. + if self.is_public { + let url = format!("pubky://{}", self.entry_key); + let event = Event::put(&url); + let value = event.serialize(); + + let key = entry.timestamp.to_string(); + + self.db.tables.events.put(&mut wtxn, &key, &value)?; + + // TODO: delete events older than a threshold. + // TODO: move to events.rs + } + + wtxn.commit()?; + + std::fs::remove_file(&self.buffer_path)?; + + Ok(entry) + } +} + +impl<'db> std::io::Write for EntryWriter<'db> { + /// Write a chunk to a Filesystem based buffer. + #[inline] + fn write(&mut self, chunk: &[u8]) -> std::io::Result { + self.hasher.update(chunk); + self.buffer.write_all(chunk)?; + + Ok(chunk.len()) + } + + /// Does not do anything, you need to call [Self::commit] + #[inline] + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use pkarr::Keypair; + + use super::DB; + + #[tokio::test] + async fn entries() -> anyhow::Result<()> { + let mut db = DB::test(); + + let keypair = Keypair::random(); + let public_key = keypair.public_key(); + let path = "/pub/foo.txt"; + + let chunk = Bytes::from(vec![1, 2, 3, 4, 5]); + + db.write_entry(&public_key, path)? + .update(&chunk)? + .commit()?; + + let rtxn = db.env.read_txn().unwrap(); + let entry = db.get_entry(&rtxn, &public_key, path).unwrap().unwrap(); + + assert_eq!( + entry.content_hash(), + &[ + 2, 79, 103, 192, 66, 90, 61, 192, 47, 186, 245, 140, 185, 61, 229, 19, 46, 61, 117, + 197, 25, 250, 160, 186, 218, 33, 73, 29, 136, 201, 112, 87 + ] + ); + + let mut blob = vec![]; + + { + let mut iter = entry.read_content(&db, &rtxn).unwrap(); + + while let Some(Ok(chunk)) = iter.next() { + blob.extend_from_slice(chunk); + } + } + + assert_eq!(blob, vec![1, 2, 3, 4, 5]); + + rtxn.commit().unwrap(); + + Ok(()) + } + + #[tokio::test] + async fn chunked_entry() -> anyhow::Result<()> { + let mut db = DB::test(); + + let keypair = Keypair::random(); + let public_key = keypair.public_key(); + let path = "/pub/foo.txt"; + + let chunk = Bytes::from(vec![0; 1024 * 1024]); + + db.write_entry(&public_key, path)? + .update(&chunk)? + .commit()?; + + let rtxn = db.env.read_txn().unwrap(); + let entry = db.get_entry(&rtxn, &public_key, path).unwrap().unwrap(); + + assert_eq!( + entry.content_hash(), + &[ + 72, 141, 226, 2, 247, 59, 217, 118, 222, 78, 112, 72, 244, 225, 243, 154, 119, 109, + 134, 213, 130, 183, 52, 143, 245, 59, 244, 50, 185, 135, 252, 168 + ] + ); + + let mut blob = vec![]; + + { + let mut iter = entry.read_content(&db, &rtxn).unwrap(); + + while let Some(Ok(chunk)) = iter.next() { + blob.extend_from_slice(chunk); + } + } + + assert_eq!(blob, vec![0; 1024 * 1024]); + + let stats = db.tables.blobs.stat(&rtxn).unwrap(); + assert_eq!(stats.overflow_pages, 0); + + rtxn.commit().unwrap(); + + Ok(()) + } +} +``` +./pubky-homeserver/src/core/database/tables/events.rs +``` +//! Server events (Put and Delete entries) +//! +//! Useful as a realtime sync with Indexers until +//! we implement more self-authenticated merkle data. + +use heed::{ + types::{Bytes, Str}, + Database, +}; +use postcard::{from_bytes, to_allocvec}; +use serde::{Deserialize, Serialize}; + +use crate::core::database::DB; + +/// Event [pkarr::Timestamp] base32 => Encoded event. +pub type EventsTable = Database; + +pub const EVENTS_TABLE: &str = "events"; + +#[derive(Clone, Serialize, Deserialize, Debug, Eq, PartialEq)] +pub enum Event { + Put(String), + Delete(String), +} + +impl Event { + pub fn put(url: &str) -> Self { + Self::Put(url.to_string()) + } + + pub fn delete(url: &str) -> Self { + Self::Delete(url.to_string()) + } + + pub fn serialize(&self) -> Vec { + to_allocvec(self).expect("Session::serialize") + } + + pub fn deserialize(bytes: &[u8]) -> core::result::Result { + if bytes[0] > 1 { + panic!("Unknown Event version"); + } + + from_bytes(bytes) + } + + pub fn url(&self) -> &str { + match self { + Event::Put(url) => url, + Event::Delete(url) => url, + } + } + + pub fn operation(&self) -> &str { + match self { + Event::Put(_) => "PUT", + Event::Delete(_) => "DEL", + } + } +} + +impl DB { + /// Returns a list of events formatted as ` `. + /// + /// - limit defaults to [crate::config::DEFAULT_LIST_LIMIT] and capped by [crate::config::DEFAULT_MAX_LIST_LIMIT] + /// - cursor is a 13 character string encoding of a timestamp + pub fn list_events( + &self, + limit: Option, + cursor: Option, + ) -> anyhow::Result> { + let txn = self.env.read_txn()?; + + let limit = limit + .unwrap_or(self.config().default_list_limit) + .min(self.config().max_list_limit); + + let cursor = cursor.unwrap_or("0000000000000".to_string()); + + let mut result: Vec = vec![]; + let mut next_cursor = cursor.to_string(); + + for _ in 0..limit { + match self.tables.events.get_greater_than(&txn, &next_cursor)? { + Some((timestamp, event_bytes)) => { + let event = Event::deserialize(event_bytes)?; + + let line = format!("{} {}", event.operation(), event.url()); + next_cursor = timestamp.to_string(); + + result.push(line); + } + None => break, + }; + } + + if !result.is_empty() { + result.push(format!("cursor: {next_cursor}")) + } + + txn.commit()?; + + Ok(result) + } +} +``` +./pubky-homeserver/src/core/database/tables/users.rs +``` +use std::borrow::Cow; + +use postcard::{from_bytes, to_allocvec}; +use serde::{Deserialize, Serialize}; + +use heed::{BoxedError, BytesDecode, BytesEncode, Database}; +use pkarr::PublicKey; + +extern crate alloc; + +/// PublicKey => User. +pub type UsersTable = Database; + +pub const USERS_TABLE: &str = "users"; + +// TODO: add more adminstration metadata like quota, invitation links, etc.. +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct User { + pub created_at: u64, +} + +impl<'a> BytesEncode<'a> for User { + type EItem = Self; + + fn bytes_encode(user: &Self::EItem) -> Result, BoxedError> { + let vec = to_allocvec(user).unwrap(); + + Ok(Cow::Owned(vec)) + } +} + +impl<'a> BytesDecode<'a> for User { + type DItem = Self; + + fn bytes_decode(bytes: &'a [u8]) -> Result { + let user: User = from_bytes(bytes).unwrap(); + + Ok(user) + } +} + +pub struct PublicKeyCodec {} + +impl<'a> BytesEncode<'a> for PublicKeyCodec { + type EItem = PublicKey; + + fn bytes_encode(pubky: &Self::EItem) -> Result, BoxedError> { + Ok(Cow::Borrowed(pubky.as_bytes())) + } +} + +impl<'a> BytesDecode<'a> for PublicKeyCodec { + type DItem = PublicKey; + + fn bytes_decode(bytes: &'a [u8]) -> Result { + Ok(PublicKey::try_from(bytes)?) + } +} +``` +./pubky-homeserver/src/core/database/migrations/m0.rs +``` +use heed::{Env, RwTxn}; + +use crate::core::database::tables::{blobs, entries, events, sessions, users}; + +pub fn run(env: &Env, wtxn: &mut RwTxn) -> anyhow::Result<()> { + let _: users::UsersTable = env.create_database(wtxn, Some(users::USERS_TABLE))?; + + let _: sessions::SessionsTable = env.create_database(wtxn, Some(sessions::SESSIONS_TABLE))?; + + let _: blobs::BlobsTable = env.create_database(wtxn, Some(blobs::BLOBS_TABLE))?; + + let _: entries::EntriesTable = env.create_database(wtxn, Some(entries::ENTRIES_TABLE))?; + + let _: events::EventsTable = env.create_database(wtxn, Some(events::EVENTS_TABLE))?; + + Ok(()) +} +``` +./pubky-homeserver/src/core/error.rs +``` +//! Server error + +use axum::{ + extract::rejection::{ExtensionRejection, PathRejection, QueryRejection}, + http::StatusCode, + response::IntoResponse, +}; +use tokio::task::JoinError; +use tracing::debug; + +pub type Result = core::result::Result; + +#[derive(Debug, Clone)] +pub struct Error { + // #[serde(with = "serde_status_code")] + status: StatusCode, + detail: Option, +} + +impl Default for Error { + fn default() -> Self { + Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + detail: None, + } + } +} + +impl Error { + pub fn with_status(status: StatusCode) -> Error { + Self { + status, + detail: None, + } + } + + /// Create a new [`Error`]. + pub fn new(status_code: StatusCode, message: Option) -> Error { + Self { + status: status_code, + detail: message.map(|m| m.to_string()), + } + } +} + +impl IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + match self.detail { + Some(detail) => (self.status, detail).into_response(), + _ => (self.status,).into_response(), + } + } +} + +impl From for Error { + fn from(error: QueryRejection) -> Self { + Self::new(StatusCode::BAD_REQUEST, error.into()) + } +} + +impl From for Error { + fn from(error: ExtensionRejection) -> Self { + Self::new(StatusCode::BAD_REQUEST, error.into()) + } +} + +impl From for Error { + fn from(error: PathRejection) -> Self { + Self::new(StatusCode::BAD_REQUEST, error.into()) + } +} + +// === Pubky specific errors === + +impl From for Error { + fn from(error: pubky_common::auth::Error) -> Self { + Self::new(StatusCode::BAD_REQUEST, Some(error)) + } +} + +impl From for Error { + fn from(error: pkarr::errors::SignedPacketVerifyError) -> Self { + Self::new(StatusCode::BAD_REQUEST, Some(error)) + } +} + +impl From for Error { + fn from(error: pkarr::errors::PublishError) -> Self { + Self::new(StatusCode::BAD_REQUEST, Some(error)) + } +} + +impl From for Error { + fn from(error: pkarr::errors::PublicKeyError) -> Self { + Self::new(StatusCode::BAD_REQUEST, Some(error)) + } +} + +// === INTERNAL_SERVER_ERROR === + +impl From for Error { + fn from(error: std::io::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: heed::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: anyhow::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: postcard::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: axum::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From> for Error { + fn from(error: flume::SendError) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: flume::RecvError) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: JoinError) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: axum::http::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} +``` +./pubky-homeserver/src/config.example.toml +``` +# Secret key (in hex) to generate the Homeserver's Keypair +# secret_key = "0000000000000000000000000000000000000000000000000000000000000000" + +[database] +# Storage directory Defaults to +# +# Storage path can be relative or absolute. +storage = "./storage/" + +[io] +# The port number to run an HTTP (clear text) server on. +http_port = 6286 +# The port number to run an HTTPs (Pkarr TLS) server on. +https_port = 6287 + +# The public IP of this server. +# +# This address will be mentioned in the Pkarr records of this +# Homeserver that is published on its public key (derivde from `secret_key`) +public_ip = "127.0.0.1" + +# If you are running this server behind a reverse proxy, +# you need to provide some extra configurations. +[io.reverse_proxy] +# The public port should be mapped to the `io::https_port` +# and you should setup tcp forwarding (don't terminate TLS on that port). +public_port = 6287 + +# If you want your server to be accessible from legacy browsers, +# you need to provide some extra configurations. +[io.legacy_browsers] +# An ICANN domain name is necessary to support legacy browsers +# +# Make sure to setup a domain name and point it the IP +# address of this machine where you are running this server. +# +# This domain should point to the `:`. +# +# Currently we don't support ICANN TLS, so you should be running +# a reverse proxy and managing certificates there for this endpoint. +domain = "example.com" +``` +./pubky-homeserver/src/lib.rs +``` +#![doc = include_str!("../README.md")] +//! + +#![deny(missing_docs)] +#![deny(rustdoc::broken_intra_doc_links)] +#![cfg_attr(any(), deny(clippy::unwrap_used))] + +mod config; +mod core; +mod io; + +pub use io::Homeserver; +pub use io::HomeserverBuilder; +``` +./pubky-homeserver/src/io/pkarr.rs +``` +//! Pkarr related task + +use anyhow::Result; +use pkarr::{dns::rdata::SVCB, Keypair, SignedPacket}; + +use super::IoConfig; + +pub struct PkarrServer { + client: pkarr::Client, + signed_packet: SignedPacket, +} + +impl PkarrServer { + pub fn new( + keypair: &Keypair, + config: &IoConfig, + https_port: u16, + http_port: u16, + ) -> Result { + let mut builder = pkarr::Client::builder(); + + // TODO: should we enable relays in homeservers for udp restricted environments? + builder.no_relays(); + + if let Some(bootstrap) = &config.bootstrap { + builder.bootstrap(bootstrap); + } + + if let Some(request_timeout) = config.dht_request_timeout { + builder.request_timeout(request_timeout); + } + + let client = builder.build()?; + + let signed_packet = create_signed_packet(keypair, config, https_port, http_port)?; + + Ok(Self { + client, + signed_packet, + }) + } + + pub async fn publish_server_packet(&self) -> anyhow::Result<()> { + // TODO: warn if packet is not most recent, which means the + // user is publishing a Packet from somewhere else. + + self.client.publish(&self.signed_packet, None).await?; + + Ok(()) + } +} + +pub fn create_signed_packet( + keypair: &Keypair, + config: &IoConfig, + https_port: u16, + http_port: u16, +) -> Result { + // TODO: Try to resolve first before publishing. + + let mut signed_packet_builder = SignedPacket::builder(); + + let mut svcb = SVCB::new(0, ".".try_into()?); + + // Set the public Ip or the loclahost + signed_packet_builder = signed_packet_builder.address( + ".".try_into().unwrap(), + config + .public_addr + .map(|addr| addr.ip()) + .unwrap_or("127.0.0.1".parse().expect("localhost is valid ip")), + 60 * 60, + ); + + // Set the public port or the local https_port + svcb.set_port( + config + .public_addr + .map(|addr| addr.port()) + .unwrap_or(https_port), + ); + + signed_packet_builder = signed_packet_builder.https(".".try_into().unwrap(), svcb, 60 * 60); + + // Set low priority https record for legacy browsers support + if let Some(ref domain) = config.domain { + let mut svcb = SVCB::new(10, ".".try_into()?); + + let http_port_be_bytes = http_port.to_be_bytes(); + if domain == "localhost" { + svcb.set_param( + pubky_common::constants::reserved_param_keys::HTTP_PORT, + &http_port_be_bytes, + )?; + } + + svcb.target = domain.as_str().try_into()?; + + signed_packet_builder = signed_packet_builder.https(".".try_into().unwrap(), svcb, 60 * 60); + } + + Ok(signed_packet_builder.build(keypair)?) +} +``` +./pubky-homeserver/src/io/mod.rs +``` +use std::{ + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; + +use ::pkarr::{Keypair, PublicKey}; +use anyhow::Result; +use http::HttpServers; +use pkarr::PkarrServer; +use tracing::info; + +use crate::{ + config::{Config, DEFAULT_HTTPS_PORT, DEFAULT_HTTP_PORT}, + core::HomeserverCore, +}; + +mod http; +mod pkarr; + +#[derive(Debug, Default)] +/// Builder for [Homeserver]. +pub struct HomeserverBuilder(Config); + +impl HomeserverBuilder { + /// Set the Homeserver's keypair + pub fn keypair(&mut self, keypair: Keypair) -> &mut Self { + self.0.keypair = keypair; + + self + } + + /// Configure the storage path of the Homeserver + pub fn storage(&mut self, storage: PathBuf) -> &mut Self { + self.0.core.storage = storage; + + self + } + + /// Configure the DHT bootstrapping nodes that this Homeserver is connected to. + pub fn bootstrap(&mut self, bootstrap: &[String]) -> &mut Self { + self.0.io.bootstrap = Some(bootstrap.to_vec()); + + self + } + + /// Configure Pkarr relays used by this Homeserver + pub fn relays(&mut self, _relays: &[url::Url]) -> &mut Self { + // TODO: make it not a noop if we are going to support relays in homeservers. + + self + } + + /// Set the public domain of this Homeserver + pub fn domain(&mut self, domain: &str) -> &mut Self { + self.0.io.domain = Some(domain.to_string()); + + self + } + + /// Run a Homeserver + /// + /// # Safety + /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub async unsafe fn run(self) -> Result { + Homeserver::run(self.0).await + } +} + +#[derive(Debug)] +/// Homeserver Core + I/O (http server and pkarr publishing). +pub struct Homeserver { + http_servers: HttpServers, + keypair: Keypair, +} + +impl Homeserver { + /// Returns a Homeserver builder. + pub fn builder() -> HomeserverBuilder { + HomeserverBuilder::default() + } + + /// Run a Homeserver with a configuration file path. + /// + /// # Safety + /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub async fn run_with_config_file(config_path: impl AsRef) -> Result { + unsafe { Self::run(Config::load(config_path).await?) }.await + } + + /// Run a Homeserver with configurations suitable for ephemeral tests. + pub async fn run_test(bootstrap: &[String]) -> Result { + let config = Config::test(bootstrap); + + unsafe { Self::run(config) }.await + } + + /// Run a Homeserver + /// + /// # Safety + /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + async unsafe fn run(config: Config) -> Result { + tracing::debug!(?config, "Running homeserver with configurations"); + + let keypair = config.keypair; + + let core = unsafe { HomeserverCore::new(config.core)? }; + + let http_servers = HttpServers::run(&keypair, &config.io, &core.router).await?; + + info!( + "Homeserver listening on http://localhost:{}", + http_servers.http_address().port() + ); + + info!("Publishing Pkarr packet.."); + + let pkarr_server = PkarrServer::new( + &keypair, + &config.io, + http_servers.https_address().port(), + http_servers.http_address().port(), + )?; + pkarr_server.publish_server_packet().await?; + + info!("Homeserver listening on https://{}", keypair.public_key()); + + Ok(Self { + http_servers, + keypair, + }) + } + + // === Getters === + + /// Returns the public_key of this server. + pub fn public_key(&self) -> PublicKey { + self.keypair.public_key() + } + + /// Returns the `https://` url + pub fn url(&self) -> url::Url { + url::Url::parse(&format!("https://{}", self.public_key())).expect("valid url") + } + + // === Public Methods === + + /// Send a shutdown signal to all open resources + pub fn shutdown(&self) { + self.http_servers.shutdown(); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IoConfig { + pub http_port: u16, + pub https_port: u16, + pub public_addr: Option, + pub domain: Option, + + /// Bootstrapping DHT nodes. + /// + /// Helpful to run the server locally or in testnet. + pub bootstrap: Option>, + pub dht_request_timeout: Option, +} + +impl Default for IoConfig { + fn default() -> Self { + IoConfig { + https_port: DEFAULT_HTTPS_PORT, + http_port: DEFAULT_HTTP_PORT, + + public_addr: None, + domain: None, + bootstrap: None, + dht_request_timeout: None, + } + } +} +``` +./pubky-homeserver/src/io/http.rs +``` +//! Http server around the HomeserverCore + +use std::{ + net::{SocketAddr, TcpListener}, + sync::Arc, +}; + +use anyhow::Result; +use axum::Router; +use axum_server::{ + tls_rustls::{RustlsAcceptor, RustlsConfig}, + Handle, +}; +use futures_util::TryFutureExt; +use pkarr::Keypair; + +use super::IoConfig; + +#[derive(Debug)] +pub struct HttpServers { + /// Handle for the HTTP server + pub(crate) http_handle: Handle, + /// Handle for the HTTPS server using Pkarr TLS + pub(crate) https_handle: Handle, + + http_address: SocketAddr, + https_address: SocketAddr, +} + +impl HttpServers { + pub async fn run(keypair: &Keypair, config: &IoConfig, router: &Router) -> Result { + let http_listener = TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], config.http_port)))?; + let http_address = http_listener.local_addr()?; + + let http_handle = Handle::new(); + + tokio::spawn( + axum_server::from_tcp(http_listener) + .handle(http_handle.clone()) + .serve( + router + .clone() + .into_make_service_with_connect_info::(), + ) + .map_err(|error| tracing::error!(?error, "Homeserver http server error")), + ); + + let https_listener = + TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], config.https_port)))?; + let https_address = https_listener.local_addr()?; + + let https_handle = Handle::new(); + + tokio::spawn( + axum_server::from_tcp(https_listener) + .acceptor(RustlsAcceptor::new(RustlsConfig::from_config(Arc::new( + keypair.to_rpk_rustls_server_config(), + )))) + .handle(https_handle.clone()) + .serve( + router + .clone() + .into_make_service_with_connect_info::(), + ) + .map_err(|error| tracing::error!(?error, "Homeserver https server error")), + ); + + Ok(Self { + http_handle, + https_handle, + + http_address, + https_address, + }) + } + + pub fn http_address(&self) -> SocketAddr { + self.http_address + } + + pub fn https_address(&self) -> SocketAddr { + self.https_address + } + + /// Shutdown all HTTP servers. + pub fn shutdown(&self) { + self.http_handle.shutdown(); + self.https_handle.shutdown(); + } +} +``` +./pubky-homeserver/src/config.rs +``` +//! Configuration for the server + +use anyhow::{anyhow, Context, Result}; +use pkarr::Keypair; +use serde::{Deserialize, Serialize}; +use std::{ + fmt::Debug, + fs, + net::{IpAddr, SocketAddr}, + path::{Path, PathBuf}, +}; + +use crate::{core::CoreConfig, io::IoConfig}; + +// === Core == +pub const DEFAULT_STORAGE_DIR: &str = "pubky"; +pub const DEFAULT_MAP_SIZE: usize = 10995116277760; // 10TB (not = disk-space used) + +pub const DEFAULT_LIST_LIMIT: u16 = 100; +pub const DEFAULT_MAX_LIST_LIMIT: u16 = 1000; + +// === IO === +pub const DEFAULT_HTTP_PORT: u16 = 6286; +pub const DEFAULT_HTTPS_PORT: u16 = 6287; + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +struct DatabaseToml { + storage: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +struct ReverseProxyToml { + pub public_port: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +struct LegacyBrowsersTompl { + pub domain: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq)] +struct IoToml { + pub http_port: Option, + pub https_port: Option, + pub public_ip: Option, + + pub reverse_proxy: Option, + pub legacy_browsers: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +struct ConfigToml { + secret_key: Option, + + database: Option, + io: Option, +} + +/// Server configuration +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + /// Server keypair. + /// + /// Defaults to a random keypair. + pub keypair: Keypair, + + pub io: IoConfig, + pub core: CoreConfig, +} + +impl Config { + fn try_from_str(value: &str) -> Result { + let config_toml: ConfigToml = toml::from_str(value)?; + + config_toml.try_into() + } + + /// Load the config from a file. + pub async fn load(path: impl AsRef) -> Result { + let config_file_path = path.as_ref(); + + let s = tokio::fs::read_to_string(config_file_path) + .await + .with_context(|| format!("failed to read {}", path.as_ref().to_string_lossy()))?; + + let mut config = Config::try_from_str(&s)?; + + // support relative path. + if config.core.storage.is_relative() { + config.core.storage = config_file_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(config.core.storage.clone()); + } + + fs::create_dir_all(&config.core.storage)?; + config.core.storage = config.core.storage.canonicalize()?; + + Ok(config) + } + + /// Create test configurations + pub fn test(bootstrap: &[String]) -> Self { + let bootstrap = Some(bootstrap.to_vec()); + + Self { + io: IoConfig { + bootstrap, + http_port: 0, + https_port: 0, + + ..Default::default() + }, + core: CoreConfig::test(), + ..Default::default() + } + } +} + +impl Default for Config { + fn default() -> Self { + Self { + keypair: Keypair::random(), + io: IoConfig::default(), + core: CoreConfig::default(), + } + } +} + +impl TryFrom for Config { + type Error = anyhow::Error; + + fn try_from(value: ConfigToml) -> std::result::Result { + let keypair = if let Some(secret_key) = value.secret_key { + let secret_key = deserialize_secret_key(secret_key)?; + Keypair::from_secret_key(&secret_key) + } else { + Keypair::random() + }; + + let storage = { + let dir = + if let Some(storage) = value.database.as_ref().and_then(|db| db.storage.clone()) { + storage + } else { + let path = dirs_next::data_dir().ok_or_else(|| { + anyhow!("operating environment provides no directory for application data") + })?; + path.join(DEFAULT_STORAGE_DIR) + }; + + dir.join("homeserver") + }; + + let io = if let Some(io) = value.io { + IoConfig { + http_port: io.http_port.unwrap_or(DEFAULT_HTTP_PORT), + https_port: io.https_port.unwrap_or(DEFAULT_HTTPS_PORT), + domain: io.legacy_browsers.and_then(|l| l.domain), + public_addr: io.public_ip.map(|ip| { + SocketAddr::from(( + ip, + io.reverse_proxy + .and_then(|r| r.public_port) + .unwrap_or(io.https_port.unwrap_or(0)), + )) + }), + ..Default::default() + } + } else { + IoConfig { + http_port: DEFAULT_HTTP_PORT, + https_port: DEFAULT_HTTPS_PORT, + ..Default::default() + } + }; + + Ok(Config { + keypair, + + io, + core: CoreConfig { + storage, + ..Default::default() + }, + }) + } +} + +fn deserialize_secret_key(s: String) -> anyhow::Result<[u8; 32]> { + let bytes = + hex::decode(s).map_err(|_| anyhow!("secret_key in config.toml should hex encoded"))?; + + if bytes.len() != 32 { + return Err(anyhow!(format!( + "secret_key in config.toml should be 32 bytes in hex (64 characters), got: {}", + bytes.len() + ))); + } + + let mut arr = [0u8; 32]; + arr.copy_from_slice(&bytes); + + Ok(arr) +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn parse_empty() { + let config = Config::try_from_str("").unwrap(); + + assert_eq!( + config, + Config { + keypair: config.keypair.clone(), + ..Default::default() + } + ) + } + + #[tokio::test] + async fn config_load() { + let crate_dir = std::env::current_dir().unwrap(); + let config_file_path = crate_dir.join("./src/config.example.toml"); + let canonical_file_path = config_file_path.canonicalize().unwrap(); + + let config = Config::load(canonical_file_path).await.unwrap(); + + assert!(config + .core + .storage + .ends_with("pubky-homeserver/src/storage/homeserver")); + } + + #[test] + fn config_test() { + let config = Config::test(&[]); + + assert_eq!( + config, + Config { + keypair: config.keypair.clone(), + + io: IoConfig { + bootstrap: Some(vec![]), + http_port: 0, + https_port: 0, + + ..Default::default() + }, + core: CoreConfig { + db_map_size: 10485760, + storage: config.core.storage.clone(), + + ..Default::default() + }, + } + ) + } + + #[test] + fn parse() { + let config = Config::try_from_str( + r#" +# Secret key (in hex) to generate the Homeserver's Keypair +secret_key = "0000000000000000000000000000000000000000000000000000000000000000" + +[database] +# Storage directory Defaults to +# storage = "" + +[io] +# The port number to run an HTTP (clear text) server on. +http_port = 6286 +# The port number to run an HTTPs (Pkarr TLS) server on. +https_port = 6287 + +# The public IP of this server. +# +# This address will be mentioned in the Pkarr records of this +# Homeserver that is published on its public key (derivde from `secret_key`) +public_ip = "127.0.0.1" + +# If you are running this server behind a reverse proxy, +# you need to provide some extra configurations. +[io.reverse_proxy] +# The public port should be mapped to the `io::https_port` +# and you should setup tcp forwarding (don't terminate TLS on that port). +public_port = 6287 + +# If you want your server to be accessible from legacy browsers, +# you need to provide some extra configurations. +[io.legacy_browsers] +# An ICANN domain name is necessary to support legacy browsers +# +# Make sure to setup a domain name and point it the IP +# address of this machine where you are running this server. +# +# This domain should point to the `:`. +# +# Currently we don't support ICANN TLS, so you should be running +# a reverse proxy and managing certificates there for this endpoint. +domain = "example.com" + "#, + ) + .unwrap(); + + assert_eq!(config.keypair, Keypair::from_secret_key(&[0; 32])); + assert_eq!(config.io.https_port, 6287); + assert_eq!( + config.io.public_addr, + Some(SocketAddr::from(([127, 0, 0, 1], 6287))) + ); + assert_eq!(config.io.domain, Some("example.com".to_string())); + } +} +``` +./pubky-homeserver/Cargo.toml +``` +[package] +name = "pubky-homeserver" +version = "0.1.1" +edition = "2021" +authors = ["Nuh "] +description = "Pubky core's homeserver." +license = "MIT" +homepage = "https://github.com/pubky/pubky-core" +repository = "https://github.com/pubky/pubky-core" +keywords = ["pkarr", "sovereign", "web", "pkarr", "datastore"] +categories = [ + "network-programming", + "cryptography", + "web-programming", + "authentication", +] + +[dependencies] +anyhow = "1.0.95" +axum = { version = "0.8.1", features = ["macros"] } +axum-extra = { version = "0.10.0", features = [ + "typed-header", + "async-read-body", +] } +base32 = "0.5.1" +bytes = "^1.10.0" +clap = { version = "4.5.29", features = ["derive"] } +dirs-next = "2.0.0" +flume = "0.11.1" +futures-util = "0.3.31" +heed = "0.21.0" +hex = "0.4.3" +httpdate = "1.0.3" +postcard = { version = "1.1.1", features = ["alloc"] } +pkarr = { version = "3.3.3", features = ["dht", "lmdb-cache", "tls"] } +pubky-common = { version = "0.3.1", path = "../pubky-common" } +serde = { version = "1.0.217", features = ["derive"] } +tokio = { version = "1.43.0", features = ["full"] } +toml = "0.8.20" +tower-cookies = "0.11.0" +tower-http = { version = "0.6.2", features = ["cors", "trace"] } +tracing = "0.1.41" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +url = "2.5.4" +axum-server = { version = "0.7.1", features = ["tls-rustls-no-provider"] } +tower = "0.5.2" +page_size = "0.6.0" +``` diff --git a/print_files.sh b/print_files.sh new file mode 100755 index 0000000..ecd38bb --- /dev/null +++ b/print_files.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Array of directories to skip +skip_dirs=(./target ./benches ./examples ./pubky ./http-relay ./pubky-testnet ./docs ./.svg) + +# Build the find command with exclusion patterns +find_cmd="find ." + +for dir in "${skip_dirs[@]}"; do + find_cmd+=" -path $dir -prune -o" +done + +# Add the file types to include and the actions to perform +find_cmd+=" \( -name '*.rs' -o -name '*.toml' \) -print" + +# Execute the constructed find command +eval $find_cmd | while read -r file; do + # Print the path to the file + echo "$file" + echo '```' + # Print the content of the file + cat "$file" + echo '```' +done diff --git a/pubky-homeserver/a.txt b/pubky-homeserver/a.txt new file mode 100644 index 0000000..f6c042f --- /dev/null +++ b/pubky-homeserver/a.txt @@ -0,0 +1,3242 @@ +./src/main.rs +``` +use std::path::PathBuf; + +use anyhow::Result; +use pubky_homeserver::Homeserver; + +use clap::Parser; + +#[derive(Parser, Debug)] +struct Cli { + /// [tracing_subscriber::EnvFilter] + #[clap(short, long)] + tracing_env_filter: Option, + + /// Optional Path to config file. + #[clap(short, long)] + config: Option, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Cli::parse(); + + tracing_subscriber::fmt() + .with_env_filter( + args.tracing_env_filter + .unwrap_or("pubky_homeserver=debug,tower_http=debug".to_string()), + ) + .init(); + + let server = unsafe { + if let Some(config_path) = args.config { + Homeserver::run_with_config_file(config_path).await? + } else { + Homeserver::builder().run().await? + } + }; + + tokio::signal::ctrl_c().await?; + + tracing::info!("Shutting down Homeserver"); + + server.shutdown(); + + Ok(()) +} +``` +./src/core/routes/feed.rs +``` +use axum::{ + body::Body, + extract::State, + http::{header, Response, StatusCode}, + response::IntoResponse, +}; +use pubky_common::timestamp::Timestamp; + +use crate::core::{ + error::{Error, Result}, + extractors::ListQueryParams, + AppState, +}; + +pub async fn feed( + State(state): State, + params: ListQueryParams, +) -> Result { + if let Some(ref cursor) = params.cursor { + if Timestamp::try_from(cursor.to_string()).is_err() { + Err(Error::new( + StatusCode::BAD_REQUEST, + "Cursor should be valid base32 Crockford encoding of a timestamp".into(), + ))? + } + } + + let result = state.db.list_events(params.limit, params.cursor)?; + + Ok(Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/plain") + .body(Body::from(result.join("\n"))) + .unwrap()) +} +``` +./src/core/routes/tenants/write.rs +``` +use std::io::Write; + +use futures_util::stream::StreamExt; + +use axum::{ + body::Body, + extract::{OriginalUri, State}, + http::StatusCode, + response::IntoResponse, +}; + +use crate::core::{ + error::{Error, Result}, + extractors::PubkyHost, + AppState, +}; + +pub async fn delete( + State(mut state): State, + pubky: PubkyHost, + path: OriginalUri, +) -> Result { + let public_key = pubky.public_key().clone(); + + // TODO: should we wrap this with `tokio::task::spawn_blocking` in case it takes too long? + let deleted = state.db.delete_entry(&public_key, path.0.path())?; + + if !deleted { + // TODO: if the path ends with `/` return a `CONFLICT` error? + return Err(Error::with_status(StatusCode::NOT_FOUND)); + }; + + Ok(()) +} + +pub async fn put( + State(mut state): State, + pubky: PubkyHost, + path: OriginalUri, + body: Body, +) -> Result { + let public_key = pubky.public_key().clone(); + + let mut entry_writer = state.db.write_entry(&public_key, path.0.path())?; + + let mut stream = body.into_data_stream(); + while let Some(next) = stream.next().await { + let chunk = next?; + entry_writer.write_all(&chunk)?; + } + + let _entry = entry_writer.commit()?; + + // TODO: return relevant headers, like Etag? + + Ok(()) +} +``` +./src/core/routes/tenants/read.rs +``` +use axum::{ + body::Body, + extract::{OriginalUri, State}, + http::{header, HeaderMap, HeaderValue, Response, StatusCode}, + response::IntoResponse, +}; +use httpdate::HttpDate; +use pkarr::PublicKey; +use std::str::FromStr; + +use crate::core::{ + database::tables::entries::Entry, + error::{Error, Result}, + extractors::{ListQueryParams, PubkyHost}, + AppState, +}; + +pub async fn head( + State(state): State, + pubky: PubkyHost, + headers: HeaderMap, + path: OriginalUri, +) -> Result { + let rtxn = state.db.env.read_txn()?; + + get_entry( + headers, + state + .db + .get_entry(&rtxn, pubky.public_key(), path.0.path())?, + None, + ) +} + +pub async fn get( + State(state): State, + headers: HeaderMap, + pubky: PubkyHost, + path: OriginalUri, + params: ListQueryParams, +) -> Result { + let public_key = pubky.public_key().clone(); + let path = path.0.path().to_string(); + + if path.ends_with('/') { + return list(state, &public_key, &path, params); + } + + let (entry_tx, entry_rx) = flume::bounded::>(1); + let (chunks_tx, chunks_rx) = flume::unbounded::, heed::Error>>(); + + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let rtxn = state.db.env.read_txn()?; + + let option = state.db.get_entry(&rtxn, &public_key, &path)?; + + if let Some(entry) = option { + let iter = entry.read_content(&state.db, &rtxn)?; + + entry_tx.send(Some(entry))?; + + for next in iter { + chunks_tx.send(next.map(|b| b.to_vec()))?; + } + }; + + entry_tx.send(None)?; + + Ok(()) + }); + + get_entry( + headers, + entry_rx.recv_async().await?, + Some(Body::from_stream(chunks_rx.into_stream())), + ) +} + +pub fn list( + state: AppState, + public_key: &PublicKey, + path: &str, + params: ListQueryParams, +) -> Result> { + let txn = state.db.env.read_txn()?; + + let path = format!("{public_key}{path}"); + + if !state.db.contains_directory(&txn, &path)? { + return Err(Error::new( + StatusCode::NOT_FOUND, + "Directory Not Found".into(), + )); + } + + // Handle listing + let vec = state.db.list( + &txn, + &path, + params.reverse, + params.limit, + params.cursor, + params.shallow, + )?; + + Ok(Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/plain") + .body(Body::from(vec.join("\n")))?) +} + +pub fn get_entry( + headers: HeaderMap, + entry: Option, + body: Option, +) -> Result> { + if let Some(entry) = entry { + // TODO: Enable seek API (range requests) + // TODO: Gzip? or brotli? + + let mut response = HeaderMap::from(&entry).into_response(); + + // Handle IF_MODIFIED_SINCE + if let Some(condition_http_date) = headers + .get(header::IF_MODIFIED_SINCE) + .and_then(|h| h.to_str().ok()) + .and_then(|s| HttpDate::from_str(s).ok()) + { + let entry_http_date: HttpDate = entry.timestamp().to_owned().into(); + + if condition_http_date >= entry_http_date { + *response.status_mut() = StatusCode::NOT_MODIFIED; + } + }; + + // Handle IF_NONE_MATCH + if let Some(str) = headers + .get(header::IF_NONE_MATCH) + .and_then(|h| h.to_str().ok()) + { + let etag = format!("\"{}\"", entry.content_hash()); + if str + .trim() + .split(',') + .collect::>() + .contains(&etag.as_str()) + { + *response.status_mut() = StatusCode::NOT_MODIFIED; + }; + } + + if let Some(body) = body { + *response.body_mut() = body; + }; + + Ok(response) + } else { + Err(Error::with_status(StatusCode::NOT_FOUND))? + } +} + +impl From<&Entry> for HeaderMap { + fn from(entry: &Entry) -> Self { + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_LENGTH, entry.content_length().into()); + headers.insert( + header::LAST_MODIFIED, + HeaderValue::from_str(&entry.timestamp().format_http_date()) + .expect("http date is valid header value"), + ); + headers.insert( + header::CONTENT_TYPE, + // TODO: when setting content type from user input, we should validate it as a HeaderValue + entry + .content_type() + .try_into() + .or(HeaderValue::from_str("")) + .expect("valid header value"), + ); + headers.insert( + header::ETAG, + format!("\"{}\"", entry.content_hash()) + .try_into() + .expect("hex string is valid"), + ); + + headers + } +} + +#[cfg(test)] +mod tests { + use axum::{ + body::Body, + http::{header, Method, Request, StatusCode}, + }; + use pkarr::Keypair; + + use crate::core::HomeserverCore; + + #[tokio::test] + async fn if_last_modified() { + let mut server = HomeserverCore::test().unwrap(); + + let keypair = Keypair::random(); + let public_key = keypair.public_key(); + let cookie = server.create_root_user(&keypair).await.unwrap().to_string(); + + let data = vec![1_u8, 2, 3, 4, 5]; + + let response = server + .call( + Request::builder() + .header("host", public_key.to_string()) + .uri("/pub/foo") + .method(Method::PUT) + .header(header::COOKIE, cookie) + .body(Body::from(data)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + let response = server + .call( + Request::builder() + .header("host", public_key.to_string()) + .uri("/pub/foo") + .method(Method::GET) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + let response = server + .call( + Request::builder() + .header("host", public_key.to_string()) + .uri("/pub/foo") + .method(Method::GET) + .header( + header::IF_MODIFIED_SINCE, + response.headers().get(header::LAST_MODIFIED).unwrap(), + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + } + + #[tokio::test] + async fn if_none_match() { + let mut server = HomeserverCore::test().unwrap(); + + let keypair = Keypair::random(); + let public_key = keypair.public_key(); + + let cookie = server.create_root_user(&keypair).await.unwrap().to_string(); + + let data = vec![1_u8, 2, 3, 4, 5]; + + let response = server + .call( + Request::builder() + .uri("/pub/foo") + .header("host", public_key.to_string()) + .method(Method::PUT) + .header(header::COOKIE, cookie) + .body(Body::from(data)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + let response = server + .call( + Request::builder() + .uri("/pub/foo") + .header("host", public_key.to_string()) + .method(Method::GET) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + let response = server + .call( + Request::builder() + .uri("/pub/foo") + .header("host", public_key.to_string()) + .method(Method::GET) + .header( + header::IF_NONE_MATCH, + response.headers().get(header::ETAG).unwrap(), + ) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + } +} +``` +./src/core/routes/tenants/session.rs +``` +use axum::{extract::State, http::StatusCode, response::IntoResponse}; +use tower_cookies::Cookies; + +use crate::core::{ + error::{Error, Result}, + extractors::PubkyHost, + layers::authz::session_secret_from_cookies, + AppState, +}; + +pub async fn session( + State(state): State, + cookies: Cookies, + pubky: PubkyHost, +) -> Result { + if let Some(secret) = session_secret_from_cookies(&cookies, pubky.public_key()) { + if let Some(session) = state.db.get_session(&secret)? { + // TODO: add content-type + return Ok(session.serialize()); + }; + } + + Err(Error::with_status(StatusCode::NOT_FOUND)) +} +pub async fn signout( + State(mut state): State, + cookies: Cookies, + pubky: PubkyHost, +) -> Result { + // TODO: Set expired cookie to delete the cookie on client side. + + if let Some(secret) = session_secret_from_cookies(&cookies, pubky.public_key()) { + state.db.delete_session(&secret)?; + } + + // Idempotent Success Response (200 OK) + Ok(()) +} +``` +./src/core/routes/tenants/mod.rs +``` +//! Per Tenant (user / Pubky) routes. +//! +//! Every route here is relative to a tenant's Pubky host, +//! as opposed to routes relative to the Homeserver's owner. + +use axum::{ + extract::DefaultBodyLimit, + routing::{delete, get, head, put}, + Router, +}; + +use crate::core::{layers::authz::AuthorizationLayer, AppState}; + +pub mod read; +pub mod session; +pub mod write; + +pub fn router(state: AppState) -> Router { + Router::new() + // - Datastore routes + .route("/pub/", get(read::get)) + .route("/pub/{*path}", get(read::get)) + .route("/pub/{*path}", head(read::head)) + .route("/pub/{*path}", put(write::put)) + .route("/pub/{*path}", delete(write::delete)) + // - Session routes + .route("/session", get(session::session)) + .route("/session", delete(session::signout)) + // Layers + // TODO: different max size for sessions and other routes? + .layer(DefaultBodyLimit::max(100 * 1024 * 1024)) + .layer(AuthorizationLayer::new(state.clone())) +} +``` +./src/core/routes/auth.rs +``` +use axum::{extract::State, response::IntoResponse}; +use axum_extra::{extract::Host, headers::UserAgent, TypedHeader}; +use bytes::Bytes; +use tower_cookies::{cookie::SameSite, Cookie, Cookies}; + +use pubky_common::{crypto::random_bytes, session::Session, timestamp::Timestamp}; + +use crate::core::{database::tables::users::User, error::Result, AppState}; + +pub async fn signup( + State(state): State, + user_agent: Option>, + cookies: Cookies, + host: Host, + body: Bytes, +) -> Result { + // TODO: Verify invitation link. + // TODO: add errors in case of already axisting user. + signin(State(state), user_agent, cookies, host, body).await +} + +pub async fn signin( + State(state): State, + user_agent: Option>, + cookies: Cookies, + Host(host): Host, + body: Bytes, +) -> Result { + let token = state.verifier.verify(&body)?; + + let public_key = token.pubky(); + + let mut wtxn = state.db.env.write_txn()?; + + let users = state.db.tables.users; + if let Some(existing) = users.get(&wtxn, public_key)? { + // TODO: why do we need this? + users.put(&mut wtxn, public_key, &existing)?; + } else { + users.put( + &mut wtxn, + public_key, + &User { + created_at: Timestamp::now().as_u64(), + }, + )?; + } + + let session_secret = base32::encode(base32::Alphabet::Crockford, &random_bytes::<16>()); + + let session = Session::new( + token.pubky(), + token.capabilities(), + user_agent.map(|ua| ua.to_string()), + ) + .serialize(); + + state + .db + .tables + .sessions + .put(&mut wtxn, &session_secret, &session)?; + + wtxn.commit()?; + + let mut cookie = Cookie::new(public_key.to_string(), session_secret); + + cookie.set_path("/"); + + // TODO: do we even have insecure anymore? + if is_secure(&host) { + cookie.set_secure(true); + cookie.set_same_site(SameSite::None); + } + cookie.set_http_only(true); + + cookies.add(cookie); + + Ok(session) +} + +/// Assuming that if the server is addressed by anything other than +/// localhost, or IP addresses, it is not addressed from a browser in an +/// secure (HTTPs) window, thus it no need to `secure` and `same_site=none` to cookies +fn is_secure(host: &str) -> bool { + url::Host::parse(host) + .map(|host| match host { + url::Host::Domain(domain) => domain != "localhost", + _ => false, + }) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use pkarr::Keypair; + + use super::*; + + #[test] + fn test_is_secure() { + assert!(!is_secure("")); + assert!(!is_secure("127.0.0.1")); + assert!(!is_secure("167.86.102.121")); + assert!(!is_secure("[2001:0db8:0000:0000:0000:ff00:0042:8329]")); + assert!(!is_secure("localhost")); + assert!(!is_secure("localhost:23423")); + assert!(is_secure(&Keypair::random().public_key().to_string())); + assert!(is_secure("example.com")); + } +} +``` +./src/core/routes/mod.rs +``` +//! The controller part of the [super::HomeserverCore] + +use axum::{ + body::Body, + extract::Request, + http::{header, HeaderValue}, + middleware::{self, Next}, + response::Response, + routing::{get, post}, + Router, +}; +use tower::ServiceBuilder; +use tower_cookies::CookieManagerLayer; +use tower_http::cors::CorsLayer; + +use crate::core::AppState; + +use super::layers::{pubky_host::PubkyHostLayer, trace::with_trace_layer}; + +mod auth; +mod feed; +mod root; +mod tenants; + +static HOMESERVER_VERSION: &str = concat!("pubky.org", "@", env!("CARGO_PKG_VERSION"),); +const TRACING_EXCLUDED_PATHS: [&str; 1] = ["/events/"]; + +fn base() -> Router { + Router::new() + .route("/", get(root::handler)) + .route("/signup", post(auth::signup)) + .route("/session", post(auth::signin)) + // Events + .route("/events/", get(feed::feed)) + // TODO: add size limit + // TODO: revisit if we enable streaming big payloads + // TODO: maybe add to a separate router (drive router?). +} + +pub fn create_app(state: AppState) -> Router { + let app = base() + .merge(tenants::router(state.clone())) + .layer(CookieManagerLayer::new()) + .layer(CorsLayer::very_permissive()) + .layer(ServiceBuilder::new().layer(middleware::from_fn(add_server_header))) + .with_state(state); + + with_trace_layer(app, &TRACING_EXCLUDED_PATHS).layer(PubkyHostLayer) +} + +// Middleware to add a `Server` header to all responses +async fn add_server_header(request: Request, next: Next) -> Response { + let mut response = next.run(request).await; + + // Add a custom header to the response + response + .headers_mut() + .insert(header::SERVER, HeaderValue::from_static(HOMESERVER_VERSION)); + + response +} +``` +./src/core/routes/root.rs +``` +use axum::response::IntoResponse; + +pub async fn handler() -> Result { + Ok("This a Pubky homeserver.".to_string()) +} +``` +./src/core/extractors.rs +``` +use std::{collections::HashMap, fmt::Display}; + +use axum::{ + extract::{FromRequestParts, Query}, + http::{request::Parts, StatusCode}, + response::{IntoResponse, Response}, + RequestPartsExt, +}; + +use pkarr::PublicKey; + +use crate::core::error::Result; + +#[derive(Debug, Clone)] +pub struct PubkyHost(pub(crate) PublicKey); + +impl PubkyHost { + pub fn public_key(&self) -> &PublicKey { + &self.0 + } +} + +impl Display for PubkyHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl FromRequestParts for PubkyHost +where + S: Sync + Send, +{ + type Rejection = Response; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + let pubky_host = parts + .extensions + .get::() + .cloned() + .ok_or(( + StatusCode::INTERNAL_SERVER_ERROR, + "Can't extract PubkyHost. Is `PubkyHostLayer` enabled?", + )) + .map_err(|e| e.into_response())?; + + Ok(pubky_host) + } +} + +#[derive(Debug)] +pub struct ListQueryParams { + pub limit: Option, + pub cursor: Option, + pub reverse: bool, + pub shallow: bool, +} + +impl FromRequestParts for ListQueryParams +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + let params: Query> = + parts.extract().await.map_err(IntoResponse::into_response)?; + + let reverse = params.contains_key("reverse"); + let shallow = params.contains_key("shallow"); + let limit = params + .get("limit") + // Treat `limit=` as None + .and_then(|l| if l.is_empty() { None } else { Some(l) }) + .and_then(|l| l.parse::().ok()); + let cursor = params + .get("cursor") + .map(|c| c.as_str()) + // Treat `cursor=` as None + .and_then(|c| { + if c.is_empty() { + None + } else { + Some(c.to_string()) + } + }); + + Ok(ListQueryParams { + reverse, + shallow, + limit, + cursor, + }) + } +} +``` +./src/core/mod.rs +``` +use std::path::PathBuf; + +use anyhow::Result; +use axum::Router; +use pubky_common::auth::AuthVerifier; + +pub mod database; +mod error; +mod extractors; +mod layers; +mod routes; + +use crate::config::{ + DEFAULT_LIST_LIMIT, DEFAULT_MAP_SIZE, DEFAULT_MAX_LIST_LIMIT, DEFAULT_STORAGE_DIR, +}; + +use database::DB; + +#[derive(Clone, Debug)] +pub(crate) struct AppState { + pub(crate) verifier: AuthVerifier, + pub(crate) db: DB, +} + +#[derive(Debug, Clone)] +/// A side-effect-free Core of the [crate::Homeserver]. +pub struct HomeserverCore { + pub(crate) router: Router, +} + +impl HomeserverCore { + /// Create a side-effect-free Homeserver core. + /// + /// # Safety + /// HomeserverCore uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub unsafe fn new(config: CoreConfig) -> Result { + let db = unsafe { DB::open(config.clone())? }; + + let state = AppState { + verifier: AuthVerifier::default(), + db, + }; + + let router = routes::create_app(state.clone()); + + Ok(Self { router }) + } +} + +#[cfg(test)] +mod tests { + + use anyhow::Result; + use axum::{ + body::Body, + extract::Request, + http::{header, Method}, + response::Response, + }; + use pkarr::Keypair; + use pubky_common::{auth::AuthToken, capabilities::Capability}; + use tower::ServiceExt; + + use super::*; + + impl HomeserverCore { + /// Test version of [HomeserverCore::new], using an ephemeral small storage. + pub fn test() -> Result { + unsafe { HomeserverCore::new(CoreConfig::test()) } + } + + // === Public Methods === + + pub async fn create_root_user(&mut self, keypair: &Keypair) -> Result { + let auth_token = AuthToken::sign(keypair, vec![Capability::root()]); + + let response = self + .call( + Request::builder() + .uri("/signup") + .header("host", keypair.public_key().to_string()) + .method(Method::POST) + .body(Body::from(auth_token.serialize())) + .unwrap(), + ) + .await?; + + let header_value = response + .headers() + .get(header::SET_COOKIE) + .and_then(|h| h.to_str().ok()) + .expect("should return a set-cookie header") + .to_string(); + + Ok(header_value) + } + + pub async fn call(&self, request: Request) -> Result { + Ok(self.router.clone().oneshot(request).await?) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// Database configurations +pub struct CoreConfig { + /// Path to the storage directory. + /// + /// Defaults to a directory in the OS data directory + pub storage: PathBuf, + pub db_map_size: usize, + + /// The default limit of a list api if no `limit` query parameter is provided. + /// + /// Defaults to `100` + pub default_list_limit: u16, + /// The maximum limit of a list api, even if a `limit` query parameter is provided. + /// + /// Defaults to `1000` + pub max_list_limit: u16, +} + +impl Default for CoreConfig { + fn default() -> Self { + Self { + storage: storage(None) + .expect("operating environment provides no directory for application data"), + db_map_size: DEFAULT_MAP_SIZE, + + default_list_limit: DEFAULT_LIST_LIMIT, + max_list_limit: DEFAULT_MAX_LIST_LIMIT, + } + } +} + +impl CoreConfig { + pub fn test() -> Self { + let storage = std::env::temp_dir() + .join(pubky_common::timestamp::Timestamp::now().to_string()) + .join(DEFAULT_STORAGE_DIR); + + Self { + storage, + db_map_size: 10485760, + + ..Default::default() + } + } +} + +pub fn storage(storage: Option) -> Result { + let dir = if let Some(storage) = storage { + PathBuf::from(storage) + } else { + let path = dirs_next::data_dir().ok_or_else(|| { + anyhow::anyhow!("operating environment provides no directory for application data") + })?; + path.join(DEFAULT_STORAGE_DIR) + }; + + Ok(dir.join("homeserver")) +} +``` +./src/core/layers/trace.rs +``` +use std::sync::Arc; + +use axum::{extract::Request, Router}; +use tower_http::trace::{ + DefaultOnFailure, DefaultOnRequest, DefaultOnResponse, OnFailure, OnRequest, OnResponse, + TraceLayer, +}; +use tracing::{Level, Span}; + +use crate::core::extractors::PubkyHost; + +pub fn with_trace_layer(router: Router, excluded_paths: &[&str]) -> Router { + let excluded_paths = Arc::new( + excluded_paths + .iter() + .map(|s| s.to_string()) + .collect::>(), + ); + + router.layer( + TraceLayer::new_for_http() + .make_span_with(move |request: &Request| { + if excluded_paths.contains(&request.uri().path().to_string()) { + // Skip logging for the noisy endpoint + tracing::span!(Level::INFO, "request", excluded = true) + } else { + // Use the default span for other endpoints + + let uri = if let Some(pubky_host) = request.extensions().get::() { + format!("pubky://{pubky_host}{}", request.uri()) + } else { + request.uri().to_string() + }; + + tracing::span!( + Level::INFO, + "request", + method = %request.method(), + uri = ?uri, + version = ?request.version(), + ) + } + }) + .on_request(|request: &Request, span: &Span| { + // Skip logging for excluded spans + if span.has_field("excluded") { + return; + } + // Use the default behavior for other spans + DefaultOnRequest::new().on_request(request, span); + }) + .on_response( + |response: &axum::response::Response, latency: std::time::Duration, span: &Span| { + // Skip logging for excluded spans + if span.has_field("excluded") { + return; + } + // Use the default behavior for other spans + DefaultOnResponse::new().on_response(response, latency, span); + }, + ) + .on_failure( + |error: tower_http::classify::ServerErrorsFailureClass, + latency: std::time::Duration, + span: &Span| { + // Skip logging for excluded spans + if span.has_field("excluded") { + return; + } + // Use the default behavior for other spans + DefaultOnFailure::new().on_failure(error, latency, span); + }, + ), + ) +} +``` +./src/core/layers/pubky_host.rs +``` +use pkarr::PublicKey; + +use crate::core::extractors::PubkyHost; + +use axum::{body::Body, http::Request}; +use futures_util::future::BoxFuture; +use std::{convert::Infallible, task::Poll}; +use tower::{Layer, Service}; + +use crate::core::error::Result; + +/// A Tower Layer to handle authorization for write operations. +#[derive(Debug, Clone)] +pub struct PubkyHostLayer; + +impl Layer for PubkyHostLayer { + type Service = PubkyHostLayerMiddleware; + + fn layer(&self, inner: S) -> Self::Service { + PubkyHostLayerMiddleware { inner } + } +} + +/// Middleware that performs authorization checks for write operations. +#[derive(Debug, Clone)] +pub struct PubkyHostLayerMiddleware { + inner: S, +} + +impl Service> for PubkyHostLayerMiddleware +where + S: Service, Response = axum::response::Response, Error = Infallible> + + Send + + 'static + + Clone, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = Infallible; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(|_| unreachable!()) // `Infallible` conversion + } + + fn call(&mut self, req: Request) -> Self::Future { + let mut inner = self.inner.clone(); + let mut req = req; + + Box::pin(async move { + let headers_to_check = ["host", "pubky-host"]; + + for header in headers_to_check { + if let Some(Ok(pubky_host)) = req.headers().get(header).map(|h| h.to_str()) { + if let Ok(public_key) = PublicKey::try_from(pubky_host) { + req.extensions_mut().insert(PubkyHost(public_key)); + } + } + } + + inner.call(req).await.map_err(|_| unreachable!()) + }) + } +} +``` +./src/core/layers/authz.rs +``` +use axum::http::Method; +use axum::response::IntoResponse; +use axum::{ + body::Body, + http::{Request, StatusCode}, +}; +use futures_util::future::BoxFuture; +use pkarr::PublicKey; +use std::{convert::Infallible, task::Poll}; +use tower::{Layer, Service}; +use tower_cookies::Cookies; + +use crate::core::{ + error::{Error, Result}, + extractors::PubkyHost, + AppState, +}; + +/// A Tower Layer to handle authorization for write operations. +#[derive(Debug, Clone)] +pub struct AuthorizationLayer { + state: AppState, +} + +impl AuthorizationLayer { + pub fn new(state: AppState) -> Self { + Self { state } + } +} + +impl Layer for AuthorizationLayer { + type Service = AuthorizationMiddleware; + + fn layer(&self, inner: S) -> Self::Service { + AuthorizationMiddleware { + inner, + state: self.state.clone(), + } + } +} + +/// Middleware that performs authorization checks for write operations. +#[derive(Debug, Clone)] +pub struct AuthorizationMiddleware { + inner: S, + state: AppState, +} + +impl Service> for AuthorizationMiddleware +where + S: Service, Response = axum::response::Response, Error = Infallible> + + Send + + 'static + + Clone, + S::Future: Send + 'static, +{ + type Response = S::Response; + type Error = S::Error; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + self.inner.poll_ready(cx).map_err(|_| unreachable!()) // `Infallible` conversion + } + + fn call(&mut self, req: Request) -> Self::Future { + let state = self.state.clone(); + let mut inner = self.inner.clone(); + + Box::pin(async move { + let path = req.uri().path(); + + let pubky = match req.extensions().get::() { + Some(pk) => pk, + None => { + return Ok( + Error::new(StatusCode::NOT_FOUND, "Pubky Host is missing".into()) + .into_response(), + ) + } + }; + + let cookies = req.extensions().get::(); + + // Authorize the request + if let Err(e) = authorize(&state, req.method(), cookies, pubky.public_key(), path) { + return Ok(e.into_response()); + } + + // If authorized, proceed to the inner service + inner.call(req).await.map_err(|_| unreachable!()) + }) + } +} + +/// Authorize write (PUT or DELETE) for Public paths. +fn authorize( + state: &AppState, + method: &Method, + cookies: Option<&Cookies>, + public_key: &PublicKey, + path: &str, +) -> Result<()> { + if path == "/session" { + // Checking (or deleting) one's session is ok for everyone + return Ok(()); + } else if path.starts_with("/pub/") { + if method == Method::GET { + return Ok(()); + } + } else { + return Err(Error::new( + StatusCode::FORBIDDEN, + "Writing to directories other than '/pub/' is forbidden".into(), + )); + } + + if let Some(cookies) = cookies { + let session_secret = session_secret_from_cookies(cookies, public_key) + .ok_or(Error::with_status(StatusCode::UNAUTHORIZED))?; + + let session = state + .db + .get_session(&session_secret)? + .ok_or(Error::with_status(StatusCode::UNAUTHORIZED))?; + + if session.pubky() == public_key + && session.capabilities().iter().any(|cap| { + path.starts_with(&cap.scope) + && cap + .actions + .contains(&pubky_common::capabilities::Action::Write) + }) + { + return Ok(()); + } + + return Err(Error::with_status(StatusCode::FORBIDDEN)); + } + + Err(Error::with_status(StatusCode::UNAUTHORIZED)) +} + +pub fn session_secret_from_cookies(cookies: &Cookies, public_key: &PublicKey) -> Option { + cookies + .get(&public_key.to_string()) + .map(|c| c.value().to_string()) +} +``` +./src/core/layers/mod.rs +``` +pub mod authz; +pub mod pubky_host; +pub mod trace; +``` +./src/core/database/tables.rs +``` +pub mod blobs; +pub mod entries; +pub mod events; +pub mod sessions; +pub mod users; + +use heed::{Env, RwTxn}; + +use blobs::{BlobsTable, BLOBS_TABLE}; +use entries::{EntriesTable, ENTRIES_TABLE}; + +use self::{ + events::{EventsTable, EVENTS_TABLE}, + sessions::{SessionsTable, SESSIONS_TABLE}, + users::{UsersTable, USERS_TABLE}, +}; + +pub const TABLES_COUNT: u32 = 5; + +#[derive(Debug, Clone)] +pub struct Tables { + pub users: UsersTable, + pub sessions: SessionsTable, + pub blobs: BlobsTable, + pub entries: EntriesTable, + pub events: EventsTable, +} + +impl Tables { + pub fn new(env: &Env, wtxn: &mut RwTxn) -> anyhow::Result { + Ok(Self { + users: env + .open_database(wtxn, Some(USERS_TABLE))? + .expect("Users table already created"), + sessions: env + .open_database(wtxn, Some(SESSIONS_TABLE))? + .expect("Sessions table already created"), + blobs: env + .open_database(wtxn, Some(BLOBS_TABLE))? + .expect("Blobs table already created"), + entries: env + .open_database(wtxn, Some(ENTRIES_TABLE))? + .expect("Entries table already created"), + events: env + .open_database(wtxn, Some(EVENTS_TABLE))? + .expect("Events table already created"), + }) + } +} +``` +./src/core/database/migrations.rs +``` +use heed::Env; + +mod m0; + +use super::tables::Tables; + +pub fn run(env: &Env) -> anyhow::Result { + let mut wtxn = env.write_txn()?; + + m0::run(env, &mut wtxn)?; + + let tables = Tables::new(env, &mut wtxn)?; + + wtxn.commit()?; + + Ok(tables) +} +``` +./src/core/database/mod.rs +``` +//! Internal database in [super::HomeserverCore] + +use std::{fs, path::PathBuf}; + +use heed::{Env, EnvOpenOptions}; + +mod migrations; +pub mod tables; + +use tables::{Tables, TABLES_COUNT}; + +pub use protected::DB; + +/// Protecting fields from being mutated by modules in crate::database +mod protected { + + use crate::core::CoreConfig; + + use super::*; + + #[derive(Debug, Clone)] + pub struct DB { + pub(crate) env: Env, + pub(crate) tables: Tables, + pub(crate) buffers_dir: PathBuf, + pub(crate) max_chunk_size: usize, + config: CoreConfig, + } + + impl DB { + /// # Safety + /// DB uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub unsafe fn open(config: CoreConfig) -> anyhow::Result { + let buffers_dir = config.storage.clone().join("buffers"); + + // Cleanup buffers. + let _ = fs::remove_dir(&buffers_dir); + fs::create_dir_all(&buffers_dir)?; + + let env = unsafe { + EnvOpenOptions::new() + .max_dbs(TABLES_COUNT) + .map_size(config.db_map_size) + .open(&config.storage) + }?; + + let tables = migrations::run(&env)?; + + let db = DB { + env, + tables, + config, + buffers_dir, + max_chunk_size: max_chunk_size(), + }; + + Ok(db) + } + + // Create an ephemeral database for testing purposes. + pub fn test() -> DB { + unsafe { DB::open(CoreConfig::test()).unwrap() } + } + + // === Getters === + + pub fn config(&self) -> &CoreConfig { + &self.config + } + } +} + +/// calculate optimal chunk size: +/// - +/// - +fn max_chunk_size() -> usize { + let page_size = page_size::get(); + + // - 16 bytes Header per page (LMDB) + // - Each page has to contain 2 records + // - 8 bytes per record (LMDB) (empirically, it seems to be 10 not 8) + // - 12 bytes key: + // - timestamp : 8 bytes + // - chunk index: 4 bytes + ((page_size - 16) / 2) - (8 + 2) - 12 +} +``` +./src/core/database/tables/sessions.rs +``` +use heed::{ + types::{Bytes, Str}, + Database, +}; +use pubky_common::session::Session; + +use crate::core::database::DB; + +/// session secret => Session. +pub type SessionsTable = Database; + +pub const SESSIONS_TABLE: &str = "sessions"; + +impl DB { + pub fn get_session(&self, session_secret: &str) -> anyhow::Result> { + let rtxn = self.env.read_txn()?; + + let session = self + .tables + .sessions + .get(&rtxn, session_secret)? + .map(|s| s.to_vec()); + + rtxn.commit()?; + + if let Some(bytes) = session { + return Ok(Some(Session::deserialize(&bytes)?)); + }; + + Ok(None) + } + + pub fn delete_session(&mut self, secret: &str) -> anyhow::Result { + let mut wtxn = self.env.write_txn()?; + + let deleted = self.tables.sessions.delete(&mut wtxn, secret)?; + + wtxn.commit()?; + + Ok(deleted) + } +} +``` +./src/core/database/tables/blobs.rs +``` +use heed::{types::Bytes, Database, RoTxn}; + +use crate::core::database::DB; + +use super::entries::Entry; + +/// (entry timestamp | chunk_index BE) => bytes +pub type BlobsTable = Database; + +pub const BLOBS_TABLE: &str = "blobs"; + +impl DB { + pub fn read_entry_content<'txn>( + &self, + rtxn: &'txn RoTxn, + entry: &Entry, + ) -> anyhow::Result> + 'txn> { + Ok(self + .tables + .blobs + .prefix_iter(rtxn, &entry.timestamp().to_bytes())? + .map(|i| i.map(|(_, bytes)| bytes))) + } +} +``` +./src/core/database/tables/entries.rs +``` +use pkarr::PublicKey; +use postcard::{from_bytes, to_allocvec}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::File, + io::{Read, Write}, + path::PathBuf, +}; +use tracing::instrument; + +use heed::{ + types::{Bytes, Str}, + Database, RoTxn, +}; + +use pubky_common::{ + crypto::{Hash, Hasher}, + timestamp::Timestamp, +}; + +use crate::core::database::DB; + +use super::events::Event; + +/// full_path(pubky/*path) => Entry. +pub type EntriesTable = Database; + +pub const ENTRIES_TABLE: &str = "entries"; + +impl DB { + /// Write an entry by an author at a given path. + /// + /// The path has to start with a forward slash `/` + pub fn write_entry( + &mut self, + public_key: &PublicKey, + path: &str, + ) -> anyhow::Result { + EntryWriter::new(self, public_key, path) + } + + /// Delete an entry by an author at a given path. + /// + /// The path has to start with a forward slash `/` + pub fn delete_entry(&mut self, public_key: &PublicKey, path: &str) -> anyhow::Result { + let mut wtxn = self.env.write_txn()?; + + let key = format!("{public_key}{path}"); + + let deleted = if let Some(bytes) = self.tables.entries.get(&wtxn, &key)? { + let entry = Entry::deserialize(bytes)?; + + let mut deleted_chunks = false; + + { + let mut iter = self + .tables + .blobs + .prefix_iter_mut(&mut wtxn, &entry.timestamp.to_bytes())?; + + while iter.next().is_some() { + unsafe { + deleted_chunks = iter.del_current()?; + } + } + } + + let deleted_entry = self.tables.entries.delete(&mut wtxn, &key)?; + + // create DELETE event + if path.starts_with("/pub/") { + let url = format!("pubky://{key}"); + + let event = Event::delete(&url); + let value = event.serialize(); + + let key = Timestamp::now().to_string(); + + self.tables.events.put(&mut wtxn, &key, &value)?; + + // TODO: delete events older than a threshold. + // TODO: move to events.rs + } + + deleted_entry && deleted_chunks + } else { + false + }; + + wtxn.commit()?; + + Ok(deleted) + } + + pub fn get_entry( + &self, + txn: &RoTxn, + public_key: &PublicKey, + path: &str, + ) -> anyhow::Result> { + let key = format!("{public_key}{path}"); + + if let Some(bytes) = self.tables.entries.get(txn, &key)? { + return Ok(Some(Entry::deserialize(bytes)?)); + } + + Ok(None) + } + + pub fn contains_directory(&self, txn: &RoTxn, path: &str) -> anyhow::Result { + Ok(self.tables.entries.get_greater_than(txn, path)?.is_some()) + } + + /// Return a list of pubky urls. + /// + /// - limit defaults to [crate::config::DEFAULT_LIST_LIMIT] and capped by [crate::config::DEFAULT_MAX_LIST_LIMIT] + pub fn list( + &self, + txn: &RoTxn, + path: &str, + reverse: bool, + limit: Option, + cursor: Option, + shallow: bool, + ) -> anyhow::Result> { + // Vector to store results + let mut results = Vec::new(); + + let limit = limit + .unwrap_or(self.config().default_list_limit) + .min(self.config().max_list_limit); + + // TODO: make this more performant than split and allocations? + + let mut threshold = cursor + .map(|cursor| { + // Removing leading forward slashes + let mut file_or_directory = cursor.trim_start_matches('/'); + + if cursor.starts_with("pubky://") { + file_or_directory = cursor.split(path).last().expect("should not be reachable") + }; + + next_threshold( + path, + file_or_directory, + file_or_directory.ends_with('/'), + reverse, + shallow, + ) + }) + .unwrap_or(next_threshold(path, "", false, reverse, shallow)); + + for _ in 0..limit { + if let Some((key, _)) = if reverse { + self.tables.entries.get_lower_than(txn, &threshold)? + } else { + self.tables.entries.get_greater_than(txn, &threshold)? + } { + if !key.starts_with(path) { + break; + } + + if shallow { + let mut split = key[path.len()..].split('/'); + let file_or_directory = split.next().expect("should not be reachable"); + + let is_directory = split.next().is_some(); + + threshold = + next_threshold(path, file_or_directory, is_directory, reverse, shallow); + + results.push(format!( + "pubky://{path}{file_or_directory}{}", + if is_directory { "/" } else { "" } + )); + } else { + threshold = key.to_string(); + results.push(format!("pubky://{}", key)) + } + }; + } + + Ok(results) + } +} + +/// Calculate the next threshold +#[instrument] +fn next_threshold( + path: &str, + file_or_directory: &str, + is_directory: bool, + reverse: bool, + shallow: bool, +) -> String { + format!( + "{path}{file_or_directory}{}", + if file_or_directory.is_empty() { + // No file_or_directory, early return + if reverse { + // `path/to/dir/\x7f` to catch all paths than `path/to/dir/` + "\x7f" + } else { + "" + } + } else if shallow & is_directory { + if reverse { + // threshold = `path/to/dir\x2e`, since `\x2e` is lower than `/` + "\x2e" + } else { + //threshold = `path/to/dir\x7f`, since `\x7f` is greater than `/` + "\x7f" + } + } else { + "" + } + ) +} + +#[derive(Clone, Default, Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct Entry { + /// Encoding version + version: usize, + /// Modified at + timestamp: Timestamp, + content_hash: EntryHash, + content_length: usize, + content_type: String, + // user_metadata: ? +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct EntryHash(Hash); + +impl Default for EntryHash { + fn default() -> Self { + Self(Hash::from_bytes([0; 32])) + } +} + +impl Serialize for EntryHash { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let bytes = self.0.as_bytes(); + bytes.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for EntryHash { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes: [u8; 32] = Deserialize::deserialize(deserializer)?; + Ok(Self(Hash::from_bytes(bytes))) + } +} + +impl Entry { + pub fn new() -> Self { + Default::default() + } + + // === Setters === + + pub fn set_timestamp(&mut self, timestamp: &Timestamp) -> &mut Self { + self.timestamp = *timestamp; + self + } + + pub fn set_content_hash(&mut self, content_hash: Hash) -> &mut Self { + EntryHash(content_hash).clone_into(&mut self.content_hash); + self + } + + pub fn set_content_length(&mut self, content_length: usize) -> &mut Self { + self.content_length = content_length; + self + } + + // === Getters === + + pub fn timestamp(&self) -> &Timestamp { + &self.timestamp + } + + pub fn content_hash(&self) -> &Hash { + &self.content_hash.0 + } + + pub fn content_length(&self) -> usize { + self.content_length + } + + pub fn content_type(&self) -> &str { + &self.content_type + } + + // === Public Method === + + pub fn read_content<'txn>( + &self, + db: &'txn DB, + rtxn: &'txn RoTxn, + ) -> anyhow::Result> + 'txn> { + db.read_entry_content(rtxn, self) + } + + pub fn serialize(&self) -> Vec { + to_allocvec(self).expect("Session::serialize") + } + + pub fn deserialize(bytes: &[u8]) -> core::result::Result { + if bytes[0] > 0 { + panic!("Unknown Entry version"); + } + + from_bytes(bytes) + } +} + +pub struct EntryWriter<'db> { + db: &'db DB, + buffer: File, + hasher: Hasher, + buffer_path: PathBuf, + entry_key: String, + timestamp: Timestamp, + is_public: bool, +} + +impl<'db> EntryWriter<'db> { + pub fn new(db: &'db DB, public_key: &PublicKey, path: &str) -> anyhow::Result { + let hasher = Hasher::new(); + + let timestamp = Timestamp::now(); + + let buffer_path = db.buffers_dir.join(timestamp.to_string()); + + let buffer = File::create(&buffer_path)?; + + let entry_key = format!("{public_key}{path}"); + + Ok(Self { + db, + buffer, + hasher, + buffer_path, + entry_key, + timestamp, + is_public: path.starts_with("/pub/"), + }) + } + + /// Same ase [EntryWriter::write_all] but returns a Result of a mutable reference of itself + /// to enable chaining with [Self::commit]. + pub fn update(&mut self, chunk: &[u8]) -> Result<&mut Self, std::io::Error> { + self.write_all(chunk)?; + + Ok(self) + } + + /// Commit blob from the filesystem buffer to LMDB, + /// write the [Entry], and commit the write transaction. + pub fn commit(&self) -> anyhow::Result { + let hash = self.hasher.finalize(); + + let mut buffer = File::open(&self.buffer_path)?; + + let mut wtxn = self.db.env.write_txn()?; + + let mut chunk_key = [0; 12]; + chunk_key[0..8].copy_from_slice(&self.timestamp.to_bytes()); + + let mut chunk_index: u32 = 0; + + loop { + let mut chunk = vec![0_u8; self.db.max_chunk_size]; + + let bytes_read = buffer.read(&mut chunk)?; + + if bytes_read == 0 { + break; // EOF reached + } + + chunk_key[8..].copy_from_slice(&chunk_index.to_be_bytes()); + + self.db + .tables + .blobs + .put(&mut wtxn, &chunk_key, &chunk[..bytes_read])?; + + chunk_index += 1; + } + + let mut entry = Entry::new(); + entry.set_timestamp(&self.timestamp); + + entry.set_content_hash(hash); + + let length = buffer.metadata()?.len(); + entry.set_content_length(length as usize); + + self.db + .tables + .entries + .put(&mut wtxn, &self.entry_key, &entry.serialize())?; + + // Write a public [Event]. + if self.is_public { + let url = format!("pubky://{}", self.entry_key); + let event = Event::put(&url); + let value = event.serialize(); + + let key = entry.timestamp.to_string(); + + self.db.tables.events.put(&mut wtxn, &key, &value)?; + + // TODO: delete events older than a threshold. + // TODO: move to events.rs + } + + wtxn.commit()?; + + std::fs::remove_file(&self.buffer_path)?; + + Ok(entry) + } +} + +impl<'db> std::io::Write for EntryWriter<'db> { + /// Write a chunk to a Filesystem based buffer. + #[inline] + fn write(&mut self, chunk: &[u8]) -> std::io::Result { + self.hasher.update(chunk); + self.buffer.write_all(chunk)?; + + Ok(chunk.len()) + } + + /// Does not do anything, you need to call [Self::commit] + #[inline] + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use pkarr::Keypair; + + use super::DB; + + #[tokio::test] + async fn entries() -> anyhow::Result<()> { + let mut db = DB::test(); + + let keypair = Keypair::random(); + let public_key = keypair.public_key(); + let path = "/pub/foo.txt"; + + let chunk = Bytes::from(vec![1, 2, 3, 4, 5]); + + db.write_entry(&public_key, path)? + .update(&chunk)? + .commit()?; + + let rtxn = db.env.read_txn().unwrap(); + let entry = db.get_entry(&rtxn, &public_key, path).unwrap().unwrap(); + + assert_eq!( + entry.content_hash(), + &[ + 2, 79, 103, 192, 66, 90, 61, 192, 47, 186, 245, 140, 185, 61, 229, 19, 46, 61, 117, + 197, 25, 250, 160, 186, 218, 33, 73, 29, 136, 201, 112, 87 + ] + ); + + let mut blob = vec![]; + + { + let mut iter = entry.read_content(&db, &rtxn).unwrap(); + + while let Some(Ok(chunk)) = iter.next() { + blob.extend_from_slice(chunk); + } + } + + assert_eq!(blob, vec![1, 2, 3, 4, 5]); + + rtxn.commit().unwrap(); + + Ok(()) + } + + #[tokio::test] + async fn chunked_entry() -> anyhow::Result<()> { + let mut db = DB::test(); + + let keypair = Keypair::random(); + let public_key = keypair.public_key(); + let path = "/pub/foo.txt"; + + let chunk = Bytes::from(vec![0; 1024 * 1024]); + + db.write_entry(&public_key, path)? + .update(&chunk)? + .commit()?; + + let rtxn = db.env.read_txn().unwrap(); + let entry = db.get_entry(&rtxn, &public_key, path).unwrap().unwrap(); + + assert_eq!( + entry.content_hash(), + &[ + 72, 141, 226, 2, 247, 59, 217, 118, 222, 78, 112, 72, 244, 225, 243, 154, 119, 109, + 134, 213, 130, 183, 52, 143, 245, 59, 244, 50, 185, 135, 252, 168 + ] + ); + + let mut blob = vec![]; + + { + let mut iter = entry.read_content(&db, &rtxn).unwrap(); + + while let Some(Ok(chunk)) = iter.next() { + blob.extend_from_slice(chunk); + } + } + + assert_eq!(blob, vec![0; 1024 * 1024]); + + let stats = db.tables.blobs.stat(&rtxn).unwrap(); + assert_eq!(stats.overflow_pages, 0); + + rtxn.commit().unwrap(); + + Ok(()) + } +} +``` +./src/core/database/tables/events.rs +``` +//! Server events (Put and Delete entries) +//! +//! Useful as a realtime sync with Indexers until +//! we implement more self-authenticated merkle data. + +use heed::{ + types::{Bytes, Str}, + Database, +}; +use postcard::{from_bytes, to_allocvec}; +use serde::{Deserialize, Serialize}; + +use crate::core::database::DB; + +/// Event [pkarr::Timestamp] base32 => Encoded event. +pub type EventsTable = Database; + +pub const EVENTS_TABLE: &str = "events"; + +#[derive(Clone, Serialize, Deserialize, Debug, Eq, PartialEq)] +pub enum Event { + Put(String), + Delete(String), +} + +impl Event { + pub fn put(url: &str) -> Self { + Self::Put(url.to_string()) + } + + pub fn delete(url: &str) -> Self { + Self::Delete(url.to_string()) + } + + pub fn serialize(&self) -> Vec { + to_allocvec(self).expect("Session::serialize") + } + + pub fn deserialize(bytes: &[u8]) -> core::result::Result { + if bytes[0] > 1 { + panic!("Unknown Event version"); + } + + from_bytes(bytes) + } + + pub fn url(&self) -> &str { + match self { + Event::Put(url) => url, + Event::Delete(url) => url, + } + } + + pub fn operation(&self) -> &str { + match self { + Event::Put(_) => "PUT", + Event::Delete(_) => "DEL", + } + } +} + +impl DB { + /// Returns a list of events formatted as ` `. + /// + /// - limit defaults to [crate::config::DEFAULT_LIST_LIMIT] and capped by [crate::config::DEFAULT_MAX_LIST_LIMIT] + /// - cursor is a 13 character string encoding of a timestamp + pub fn list_events( + &self, + limit: Option, + cursor: Option, + ) -> anyhow::Result> { + let txn = self.env.read_txn()?; + + let limit = limit + .unwrap_or(self.config().default_list_limit) + .min(self.config().max_list_limit); + + let cursor = cursor.unwrap_or("0000000000000".to_string()); + + let mut result: Vec = vec![]; + let mut next_cursor = cursor.to_string(); + + for _ in 0..limit { + match self.tables.events.get_greater_than(&txn, &next_cursor)? { + Some((timestamp, event_bytes)) => { + let event = Event::deserialize(event_bytes)?; + + let line = format!("{} {}", event.operation(), event.url()); + next_cursor = timestamp.to_string(); + + result.push(line); + } + None => break, + }; + } + + if !result.is_empty() { + result.push(format!("cursor: {next_cursor}")) + } + + txn.commit()?; + + Ok(result) + } +} +``` +./src/core/database/tables/users.rs +``` +use std::borrow::Cow; + +use postcard::{from_bytes, to_allocvec}; +use serde::{Deserialize, Serialize}; + +use heed::{BoxedError, BytesDecode, BytesEncode, Database}; +use pkarr::PublicKey; + +extern crate alloc; + +/// PublicKey => User. +pub type UsersTable = Database; + +pub const USERS_TABLE: &str = "users"; + +// TODO: add more adminstration metadata like quota, invitation links, etc.. +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] +pub struct User { + pub created_at: u64, +} + +impl<'a> BytesEncode<'a> for User { + type EItem = Self; + + fn bytes_encode(user: &Self::EItem) -> Result, BoxedError> { + let vec = to_allocvec(user).unwrap(); + + Ok(Cow::Owned(vec)) + } +} + +impl<'a> BytesDecode<'a> for User { + type DItem = Self; + + fn bytes_decode(bytes: &'a [u8]) -> Result { + let user: User = from_bytes(bytes).unwrap(); + + Ok(user) + } +} + +pub struct PublicKeyCodec {} + +impl<'a> BytesEncode<'a> for PublicKeyCodec { + type EItem = PublicKey; + + fn bytes_encode(pubky: &Self::EItem) -> Result, BoxedError> { + Ok(Cow::Borrowed(pubky.as_bytes())) + } +} + +impl<'a> BytesDecode<'a> for PublicKeyCodec { + type DItem = PublicKey; + + fn bytes_decode(bytes: &'a [u8]) -> Result { + Ok(PublicKey::try_from(bytes)?) + } +} +``` +./src/core/database/migrations/m0.rs +``` +use heed::{Env, RwTxn}; + +use crate::core::database::tables::{blobs, entries, events, sessions, users}; + +pub fn run(env: &Env, wtxn: &mut RwTxn) -> anyhow::Result<()> { + let _: users::UsersTable = env.create_database(wtxn, Some(users::USERS_TABLE))?; + + let _: sessions::SessionsTable = env.create_database(wtxn, Some(sessions::SESSIONS_TABLE))?; + + let _: blobs::BlobsTable = env.create_database(wtxn, Some(blobs::BLOBS_TABLE))?; + + let _: entries::EntriesTable = env.create_database(wtxn, Some(entries::ENTRIES_TABLE))?; + + let _: events::EventsTable = env.create_database(wtxn, Some(events::EVENTS_TABLE))?; + + Ok(()) +} +``` +./src/core/error.rs +``` +//! Server error + +use axum::{ + extract::rejection::{ExtensionRejection, PathRejection, QueryRejection}, + http::StatusCode, + response::IntoResponse, +}; +use tokio::task::JoinError; +use tracing::debug; + +pub type Result = core::result::Result; + +#[derive(Debug, Clone)] +pub struct Error { + // #[serde(with = "serde_status_code")] + status: StatusCode, + detail: Option, +} + +impl Default for Error { + fn default() -> Self { + Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + detail: None, + } + } +} + +impl Error { + pub fn with_status(status: StatusCode) -> Error { + Self { + status, + detail: None, + } + } + + /// Create a new [`Error`]. + pub fn new(status_code: StatusCode, message: Option) -> Error { + Self { + status: status_code, + detail: message.map(|m| m.to_string()), + } + } +} + +impl IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + match self.detail { + Some(detail) => (self.status, detail).into_response(), + _ => (self.status,).into_response(), + } + } +} + +impl From for Error { + fn from(error: QueryRejection) -> Self { + Self::new(StatusCode::BAD_REQUEST, error.into()) + } +} + +impl From for Error { + fn from(error: ExtensionRejection) -> Self { + Self::new(StatusCode::BAD_REQUEST, error.into()) + } +} + +impl From for Error { + fn from(error: PathRejection) -> Self { + Self::new(StatusCode::BAD_REQUEST, error.into()) + } +} + +// === Pubky specific errors === + +impl From for Error { + fn from(error: pubky_common::auth::Error) -> Self { + Self::new(StatusCode::BAD_REQUEST, Some(error)) + } +} + +impl From for Error { + fn from(error: pkarr::errors::SignedPacketVerifyError) -> Self { + Self::new(StatusCode::BAD_REQUEST, Some(error)) + } +} + +impl From for Error { + fn from(error: pkarr::errors::PublishError) -> Self { + Self::new(StatusCode::BAD_REQUEST, Some(error)) + } +} + +impl From for Error { + fn from(error: pkarr::errors::PublicKeyError) -> Self { + Self::new(StatusCode::BAD_REQUEST, Some(error)) + } +} + +// === INTERNAL_SERVER_ERROR === + +impl From for Error { + fn from(error: std::io::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: heed::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: anyhow::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: postcard::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: axum::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From> for Error { + fn from(error: flume::SendError) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: flume::RecvError) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: JoinError) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} + +impl From for Error { + fn from(error: axum::http::Error) -> Self { + debug!(?error); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, error.into()) + } +} +``` +./src/config.example.toml +``` +# Secret key (in hex) to generate the Homeserver's Keypair +# secret_key = "0000000000000000000000000000000000000000000000000000000000000000" + +[database] +# Storage directory Defaults to +# +# Storage path can be relative or absolute. +storage = "./storage/" + +[io] +# The port number to run an HTTP (clear text) server on. +http_port = 6286 +# The port number to run an HTTPs (Pkarr TLS) server on. +https_port = 6287 + +# The public IP of this server. +# +# This address will be mentioned in the Pkarr records of this +# Homeserver that is published on its public key (derivde from `secret_key`) +public_ip = "127.0.0.1" + +# If you are running this server behind a reverse proxy, +# you need to provide some extra configurations. +[io.reverse_proxy] +# The public port should be mapped to the `io::https_port` +# and you should setup tcp forwarding (don't terminate TLS on that port). +public_port = 6287 + +# If you want your server to be accessible from legacy browsers, +# you need to provide some extra configurations. +[io.legacy_browsers] +# An ICANN domain name is necessary to support legacy browsers +# +# Make sure to setup a domain name and point it the IP +# address of this machine where you are running this server. +# +# This domain should point to the `:`. +# +# Currently we don't support ICANN TLS, so you should be running +# a reverse proxy and managing certificates there for this endpoint. +domain = "example.com" +``` +./src/lib.rs +``` +#![doc = include_str!("../README.md")] +//! + +#![deny(missing_docs)] +#![deny(rustdoc::broken_intra_doc_links)] +#![cfg_attr(any(), deny(clippy::unwrap_used))] + +mod config; +mod core; +mod io; + +pub use io::Homeserver; +pub use io::HomeserverBuilder; +``` +./src/io/pkarr.rs +``` +//! Pkarr related task + +use anyhow::Result; +use pkarr::{dns::rdata::SVCB, Keypair, SignedPacket}; + +use super::IoConfig; + +pub struct PkarrServer { + client: pkarr::Client, + signed_packet: SignedPacket, +} + +impl PkarrServer { + pub fn new( + keypair: &Keypair, + config: &IoConfig, + https_port: u16, + http_port: u16, + ) -> Result { + let mut builder = pkarr::Client::builder(); + + // TODO: should we enable relays in homeservers for udp restricted environments? + builder.no_relays(); + + if let Some(bootstrap) = &config.bootstrap { + builder.bootstrap(bootstrap); + } + + if let Some(request_timeout) = config.dht_request_timeout { + builder.request_timeout(request_timeout); + } + + let client = builder.build()?; + + let signed_packet = create_signed_packet(keypair, config, https_port, http_port)?; + + Ok(Self { + client, + signed_packet, + }) + } + + pub async fn publish_server_packet(&self) -> anyhow::Result<()> { + // TODO: warn if packet is not most recent, which means the + // user is publishing a Packet from somewhere else. + + self.client.publish(&self.signed_packet, None).await?; + + Ok(()) + } +} + +pub fn create_signed_packet( + keypair: &Keypair, + config: &IoConfig, + https_port: u16, + http_port: u16, +) -> Result { + // TODO: Try to resolve first before publishing. + + let mut signed_packet_builder = SignedPacket::builder(); + + let mut svcb = SVCB::new(0, ".".try_into()?); + + // Set the public Ip or the loclahost + signed_packet_builder = signed_packet_builder.address( + ".".try_into().unwrap(), + config + .public_addr + .map(|addr| addr.ip()) + .unwrap_or("127.0.0.1".parse().expect("localhost is valid ip")), + 60 * 60, + ); + + // Set the public port or the local https_port + svcb.set_port( + config + .public_addr + .map(|addr| addr.port()) + .unwrap_or(https_port), + ); + + signed_packet_builder = signed_packet_builder.https(".".try_into().unwrap(), svcb, 60 * 60); + + // Set low priority https record for legacy browsers support + if let Some(ref domain) = config.domain { + let mut svcb = SVCB::new(10, ".".try_into()?); + + let http_port_be_bytes = http_port.to_be_bytes(); + if domain == "localhost" { + svcb.set_param( + pubky_common::constants::reserved_param_keys::HTTP_PORT, + &http_port_be_bytes, + )?; + } + + svcb.target = domain.as_str().try_into()?; + + signed_packet_builder = signed_packet_builder.https(".".try_into().unwrap(), svcb, 60 * 60); + } + + Ok(signed_packet_builder.build(keypair)?) +} +``` +./src/io/mod.rs +``` +use std::{ + net::SocketAddr, + path::{Path, PathBuf}, + time::Duration, +}; + +use ::pkarr::{Keypair, PublicKey}; +use anyhow::Result; +use http::HttpServers; +use pkarr::PkarrServer; +use tracing::info; + +use crate::{ + config::{Config, DEFAULT_HTTPS_PORT, DEFAULT_HTTP_PORT}, + core::HomeserverCore, +}; + +mod http; +mod pkarr; + +#[derive(Debug, Default)] +/// Builder for [Homeserver]. +pub struct HomeserverBuilder(Config); + +impl HomeserverBuilder { + /// Set the Homeserver's keypair + pub fn keypair(&mut self, keypair: Keypair) -> &mut Self { + self.0.keypair = keypair; + + self + } + + /// Configure the storage path of the Homeserver + pub fn storage(&mut self, storage: PathBuf) -> &mut Self { + self.0.core.storage = storage; + + self + } + + /// Configure the DHT bootstrapping nodes that this Homeserver is connected to. + pub fn bootstrap(&mut self, bootstrap: &[String]) -> &mut Self { + self.0.io.bootstrap = Some(bootstrap.to_vec()); + + self + } + + /// Configure Pkarr relays used by this Homeserver + pub fn relays(&mut self, _relays: &[url::Url]) -> &mut Self { + // TODO: make it not a noop if we are going to support relays in homeservers. + + self + } + + /// Set the public domain of this Homeserver + pub fn domain(&mut self, domain: &str) -> &mut Self { + self.0.io.domain = Some(domain.to_string()); + + self + } + + /// Run a Homeserver + /// + /// # Safety + /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub async unsafe fn run(self) -> Result { + Homeserver::run(self.0).await + } +} + +#[derive(Debug)] +/// Homeserver Core + I/O (http server and pkarr publishing). +pub struct Homeserver { + http_servers: HttpServers, + keypair: Keypair, +} + +impl Homeserver { + /// Returns a Homeserver builder. + pub fn builder() -> HomeserverBuilder { + HomeserverBuilder::default() + } + + /// Run a Homeserver with a configuration file path. + /// + /// # Safety + /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + pub async fn run_with_config_file(config_path: impl AsRef) -> Result { + unsafe { Self::run(Config::load(config_path).await?) }.await + } + + /// Run a Homeserver with configurations suitable for ephemeral tests. + pub async fn run_test(bootstrap: &[String]) -> Result { + let config = Config::test(bootstrap); + + unsafe { Self::run(config) }.await + } + + /// Run a Homeserver + /// + /// # Safety + /// Homeserver uses LMDB, [opening][heed::EnvOpenOptions::open] which is marked unsafe, + /// because the possible Undefined Behavior (UB) if the lock file is broken. + async unsafe fn run(config: Config) -> Result { + tracing::debug!(?config, "Running homeserver with configurations"); + + let keypair = config.keypair; + + let core = unsafe { HomeserverCore::new(config.core)? }; + + let http_servers = HttpServers::run(&keypair, &config.io, &core.router).await?; + + info!( + "Homeserver listening on http://localhost:{}", + http_servers.http_address().port() + ); + + info!("Publishing Pkarr packet.."); + + let pkarr_server = PkarrServer::new( + &keypair, + &config.io, + http_servers.https_address().port(), + http_servers.http_address().port(), + )?; + pkarr_server.publish_server_packet().await?; + + info!("Homeserver listening on https://{}", keypair.public_key()); + + Ok(Self { + http_servers, + keypair, + }) + } + + // === Getters === + + /// Returns the public_key of this server. + pub fn public_key(&self) -> PublicKey { + self.keypair.public_key() + } + + /// Returns the `https://` url + pub fn url(&self) -> url::Url { + url::Url::parse(&format!("https://{}", self.public_key())).expect("valid url") + } + + // === Public Methods === + + /// Send a shutdown signal to all open resources + pub fn shutdown(&self) { + self.http_servers.shutdown(); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IoConfig { + pub http_port: u16, + pub https_port: u16, + pub public_addr: Option, + pub domain: Option, + + /// Bootstrapping DHT nodes. + /// + /// Helpful to run the server locally or in testnet. + pub bootstrap: Option>, + pub dht_request_timeout: Option, +} + +impl Default for IoConfig { + fn default() -> Self { + IoConfig { + https_port: DEFAULT_HTTPS_PORT, + http_port: DEFAULT_HTTP_PORT, + + public_addr: None, + domain: None, + bootstrap: None, + dht_request_timeout: None, + } + } +} +``` +./src/io/http.rs +``` +//! Http server around the HomeserverCore + +use std::{ + net::{SocketAddr, TcpListener}, + sync::Arc, +}; + +use anyhow::Result; +use axum::Router; +use axum_server::{ + tls_rustls::{RustlsAcceptor, RustlsConfig}, + Handle, +}; +use futures_util::TryFutureExt; +use pkarr::Keypair; + +use super::IoConfig; + +#[derive(Debug)] +pub struct HttpServers { + /// Handle for the HTTP server + pub(crate) http_handle: Handle, + /// Handle for the HTTPS server using Pkarr TLS + pub(crate) https_handle: Handle, + + http_address: SocketAddr, + https_address: SocketAddr, +} + +impl HttpServers { + pub async fn run(keypair: &Keypair, config: &IoConfig, router: &Router) -> Result { + let http_listener = TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], config.http_port)))?; + let http_address = http_listener.local_addr()?; + + let http_handle = Handle::new(); + + tokio::spawn( + axum_server::from_tcp(http_listener) + .handle(http_handle.clone()) + .serve( + router + .clone() + .into_make_service_with_connect_info::(), + ) + .map_err(|error| tracing::error!(?error, "Homeserver http server error")), + ); + + let https_listener = + TcpListener::bind(SocketAddr::from(([0, 0, 0, 0], config.https_port)))?; + let https_address = https_listener.local_addr()?; + + let https_handle = Handle::new(); + + tokio::spawn( + axum_server::from_tcp(https_listener) + .acceptor(RustlsAcceptor::new(RustlsConfig::from_config(Arc::new( + keypair.to_rpk_rustls_server_config(), + )))) + .handle(https_handle.clone()) + .serve( + router + .clone() + .into_make_service_with_connect_info::(), + ) + .map_err(|error| tracing::error!(?error, "Homeserver https server error")), + ); + + Ok(Self { + http_handle, + https_handle, + + http_address, + https_address, + }) + } + + pub fn http_address(&self) -> SocketAddr { + self.http_address + } + + pub fn https_address(&self) -> SocketAddr { + self.https_address + } + + /// Shutdown all HTTP servers. + pub fn shutdown(&self) { + self.http_handle.shutdown(); + self.https_handle.shutdown(); + } +} +``` +./src/config.rs +``` +//! Configuration for the server + +use anyhow::{anyhow, Context, Result}; +use pkarr::Keypair; +use serde::{Deserialize, Serialize}; +use std::{ + fmt::Debug, + fs, + net::{IpAddr, SocketAddr}, + path::{Path, PathBuf}, +}; + +use crate::{core::CoreConfig, io::IoConfig}; + +// === Core == +pub const DEFAULT_STORAGE_DIR: &str = "pubky"; +pub const DEFAULT_MAP_SIZE: usize = 10995116277760; // 10TB (not = disk-space used) + +pub const DEFAULT_LIST_LIMIT: u16 = 100; +pub const DEFAULT_MAX_LIST_LIMIT: u16 = 1000; + +// === IO === +pub const DEFAULT_HTTP_PORT: u16 = 6286; +pub const DEFAULT_HTTPS_PORT: u16 = 6287; + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +struct DatabaseToml { + storage: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +struct ReverseProxyToml { + pub public_port: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +struct LegacyBrowsersTompl { + pub domain: Option, +} + +#[derive(Debug, Default, Serialize, Deserialize, Clone, PartialEq)] +struct IoToml { + pub http_port: Option, + pub https_port: Option, + pub public_ip: Option, + + pub reverse_proxy: Option, + pub legacy_browsers: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +struct ConfigToml { + secret_key: Option, + + database: Option, + io: Option, +} + +/// Server configuration +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + /// Server keypair. + /// + /// Defaults to a random keypair. + pub keypair: Keypair, + + pub io: IoConfig, + pub core: CoreConfig, +} + +impl Config { + fn try_from_str(value: &str) -> Result { + let config_toml: ConfigToml = toml::from_str(value)?; + + config_toml.try_into() + } + + /// Load the config from a file. + pub async fn load(path: impl AsRef) -> Result { + let config_file_path = path.as_ref(); + + let s = tokio::fs::read_to_string(config_file_path) + .await + .with_context(|| format!("failed to read {}", path.as_ref().to_string_lossy()))?; + + let mut config = Config::try_from_str(&s)?; + + // support relative path. + if config.core.storage.is_relative() { + config.core.storage = config_file_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(config.core.storage.clone()); + } + + fs::create_dir_all(&config.core.storage)?; + config.core.storage = config.core.storage.canonicalize()?; + + Ok(config) + } + + /// Create test configurations + pub fn test(bootstrap: &[String]) -> Self { + let bootstrap = Some(bootstrap.to_vec()); + + Self { + io: IoConfig { + bootstrap, + http_port: 0, + https_port: 0, + + ..Default::default() + }, + core: CoreConfig::test(), + ..Default::default() + } + } +} + +impl Default for Config { + fn default() -> Self { + Self { + keypair: Keypair::random(), + io: IoConfig::default(), + core: CoreConfig::default(), + } + } +} + +impl TryFrom for Config { + type Error = anyhow::Error; + + fn try_from(value: ConfigToml) -> std::result::Result { + let keypair = if let Some(secret_key) = value.secret_key { + let secret_key = deserialize_secret_key(secret_key)?; + Keypair::from_secret_key(&secret_key) + } else { + Keypair::random() + }; + + let storage = { + let dir = + if let Some(storage) = value.database.as_ref().and_then(|db| db.storage.clone()) { + storage + } else { + let path = dirs_next::data_dir().ok_or_else(|| { + anyhow!("operating environment provides no directory for application data") + })?; + path.join(DEFAULT_STORAGE_DIR) + }; + + dir.join("homeserver") + }; + + let io = if let Some(io) = value.io { + IoConfig { + http_port: io.http_port.unwrap_or(DEFAULT_HTTP_PORT), + https_port: io.https_port.unwrap_or(DEFAULT_HTTPS_PORT), + domain: io.legacy_browsers.and_then(|l| l.domain), + public_addr: io.public_ip.map(|ip| { + SocketAddr::from(( + ip, + io.reverse_proxy + .and_then(|r| r.public_port) + .unwrap_or(io.https_port.unwrap_or(0)), + )) + }), + ..Default::default() + } + } else { + IoConfig { + http_port: DEFAULT_HTTP_PORT, + https_port: DEFAULT_HTTPS_PORT, + ..Default::default() + } + }; + + Ok(Config { + keypair, + + io, + core: CoreConfig { + storage, + ..Default::default() + }, + }) + } +} + +fn deserialize_secret_key(s: String) -> anyhow::Result<[u8; 32]> { + let bytes = + hex::decode(s).map_err(|_| anyhow!("secret_key in config.toml should hex encoded"))?; + + if bytes.len() != 32 { + return Err(anyhow!(format!( + "secret_key in config.toml should be 32 bytes in hex (64 characters), got: {}", + bytes.len() + ))); + } + + let mut arr = [0u8; 32]; + arr.copy_from_slice(&bytes); + + Ok(arr) +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn parse_empty() { + let config = Config::try_from_str("").unwrap(); + + assert_eq!( + config, + Config { + keypair: config.keypair.clone(), + ..Default::default() + } + ) + } + + #[tokio::test] + async fn config_load() { + let crate_dir = std::env::current_dir().unwrap(); + let config_file_path = crate_dir.join("./src/config.example.toml"); + let canonical_file_path = config_file_path.canonicalize().unwrap(); + + let config = Config::load(canonical_file_path).await.unwrap(); + + assert!(config + .core + .storage + .ends_with("pubky-homeserver/src/storage/homeserver")); + } + + #[test] + fn config_test() { + let config = Config::test(&[]); + + assert_eq!( + config, + Config { + keypair: config.keypair.clone(), + + io: IoConfig { + bootstrap: Some(vec![]), + http_port: 0, + https_port: 0, + + ..Default::default() + }, + core: CoreConfig { + db_map_size: 10485760, + storage: config.core.storage.clone(), + + ..Default::default() + }, + } + ) + } + + #[test] + fn parse() { + let config = Config::try_from_str( + r#" +# Secret key (in hex) to generate the Homeserver's Keypair +secret_key = "0000000000000000000000000000000000000000000000000000000000000000" + +[database] +# Storage directory Defaults to +# storage = "" + +[io] +# The port number to run an HTTP (clear text) server on. +http_port = 6286 +# The port number to run an HTTPs (Pkarr TLS) server on. +https_port = 6287 + +# The public IP of this server. +# +# This address will be mentioned in the Pkarr records of this +# Homeserver that is published on its public key (derivde from `secret_key`) +public_ip = "127.0.0.1" + +# If you are running this server behind a reverse proxy, +# you need to provide some extra configurations. +[io.reverse_proxy] +# The public port should be mapped to the `io::https_port` +# and you should setup tcp forwarding (don't terminate TLS on that port). +public_port = 6287 + +# If you want your server to be accessible from legacy browsers, +# you need to provide some extra configurations. +[io.legacy_browsers] +# An ICANN domain name is necessary to support legacy browsers +# +# Make sure to setup a domain name and point it the IP +# address of this machine where you are running this server. +# +# This domain should point to the `:`. +# +# Currently we don't support ICANN TLS, so you should be running +# a reverse proxy and managing certificates there for this endpoint. +domain = "example.com" + "#, + ) + .unwrap(); + + assert_eq!(config.keypair, Keypair::from_secret_key(&[0; 32])); + assert_eq!(config.io.https_port, 6287); + assert_eq!( + config.io.public_addr, + Some(SocketAddr::from(([127, 0, 0, 1], 6287))) + ); + assert_eq!(config.io.domain, Some("example.com".to_string())); + } +} +``` +./Cargo.toml +``` +[package] +name = "pubky-homeserver" +version = "0.1.1" +edition = "2021" +authors = ["Nuh "] +description = "Pubky core's homeserver." +license = "MIT" +homepage = "https://github.com/pubky/pubky-core" +repository = "https://github.com/pubky/pubky-core" +keywords = ["pkarr", "sovereign", "web", "pkarr", "datastore"] +categories = [ + "network-programming", + "cryptography", + "web-programming", + "authentication", +] + +[dependencies] +anyhow = "1.0.95" +axum = { version = "0.8.1", features = ["macros"] } +axum-extra = { version = "0.10.0", features = [ + "typed-header", + "async-read-body", +] } +base32 = "0.5.1" +bytes = "^1.10.0" +clap = { version = "4.5.29", features = ["derive"] } +dirs-next = "2.0.0" +flume = "0.11.1" +futures-util = "0.3.31" +heed = "0.21.0" +hex = "0.4.3" +httpdate = "1.0.3" +postcard = { version = "1.1.1", features = ["alloc"] } +pkarr = { version = "3.3.3", features = ["dht", "lmdb-cache", "tls"] } +pubky-common = { version = "0.3.1", path = "../pubky-common" } +serde = { version = "1.0.217", features = ["derive"] } +tokio = { version = "1.43.0", features = ["full"] } +toml = "0.8.20" +tower-cookies = "0.11.0" +tower-http = { version = "0.6.2", features = ["cors", "trace"] } +tracing = "0.1.41" +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +url = "2.5.4" +axum-server = { version = "0.7.1", features = ["tls-rustls-no-provider"] } +tower = "0.5.2" +page_size = "0.6.0" +``` diff --git a/pubky-homeserver/print_files.sh b/pubky-homeserver/print_files.sh new file mode 100755 index 0000000..ecd38bb --- /dev/null +++ b/pubky-homeserver/print_files.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Array of directories to skip +skip_dirs=(./target ./benches ./examples ./pubky ./http-relay ./pubky-testnet ./docs ./.svg) + +# Build the find command with exclusion patterns +find_cmd="find ." + +for dir in "${skip_dirs[@]}"; do + find_cmd+=" -path $dir -prune -o" +done + +# Add the file types to include and the actions to perform +find_cmd+=" \( -name '*.rs' -o -name '*.toml' \) -print" + +# Execute the constructed find command +eval $find_cmd | while read -r file; do + # Print the path to the file + echo "$file" + echo '```' + # Print the content of the file + cat "$file" + echo '```' +done diff --git a/pubky-testnet/src/lib.rs b/pubky-testnet/src/lib.rs index 7a41b5e..3844241 100644 --- a/pubky-testnet/src/lib.rs +++ b/pubky-testnet/src/lib.rs @@ -23,7 +23,7 @@ pub struct Testnet { impl Testnet { /// Run a new testnet. pub async fn run() -> Result { - let dht = mainline::Testnet::new(10)?; + let dht = mainline::Testnet::new(3)?; let mut testnet = Self { dht, @@ -42,7 +42,7 @@ impl Testnet { /// 2. A Homeserver with address is hardcoded to `8pinxxgqs41n4aididenw5apqp1urfmzdztr8jt4abrkdn435ewo` /// 4. An HTTP relay running on port [15412](pubky_common::constants::testnet_ports::HTTP_RELAY) pub async fn run_with_hardcoded_configurations() -> Result { - let dht = mainline::Testnet::new(10)?; + let dht = mainline::Testnet::new(3)?; dht.leak();