cleanup + tune LRU size

src/bin/page_explorer.rs

@@ -8,7 +8,9 @@ use std::{
 
 use clap::Parser;
 use innodb::innodb::{
-    buffer_manager::{lru::LRUBufferManager, simple::SimpleBufferManager, BufferManager, DummyBufferMangaer},
+    buffer_manager::{
+        lru::LRUBufferManager, simple::SimpleBufferManager, BufferManager, DummyBufferMangaer,
+    },
     page::{
         index::{record::RecordType, IndexPage},
         Page, PageType, FIL_PAGE_SIZE,
@@ -62,10 +64,12 @@ struct PageExplorer {
     buffer_mgr: Box<dyn BufferManager>,
     total_records: usize,
     missing_records: usize,
+    incomplete_records: usize,
 }
 
 impl PageExplorer {
-    fn write_row(&mut self, row: &Row, values: &[FieldValue]) -> Result<()> {
+    fn write_row(&mut self, values: &[FieldValue]) -> Result<()> {
+        let mut has_missing = false;
         if let Some(writer) = &mut self.output_writer {
             writer.begin_object()?;
 
@@ -82,11 +86,19 @@ impl PageExplorer {
                     FieldValue::UnsignedInt(v) => writer.number_value(*v)?,
                     FieldValue::String(s) => writer.string_value(s)?,
                     FieldValue::Null => writer.null_value()?,
+                    FieldValue::Skipped => {
+                        has_missing = true;
+                        writer.null_value()?;
+                    }
                     _ => panic!("Unsupported Field Value for writing JSON"),
                 };
             }
             writer.end_object()?;
         }
+
+        if has_missing {
+            self.incomplete_records += 1;
+        }
         Ok(())
     }
 
@@ -111,7 +123,7 @@ impl PageExplorer {
                         let values = row.parse_values(self.buffer_mgr.as_mut());
                         assert_eq!(values.len(), table.field_count());
                         debug!("{:?}", values);
-                        self.write_row(&row, &values).expect("Failed to write row");
+                        self.write_row(&values).expect("Failed to write row");
                     }
                 }
                 RecordType::NodePointer => {
@@ -145,6 +157,9 @@ impl PageExplorer {
     }
 
     fn explore_page(&mut self, file_offset: usize, page: Page) {
+        if page.header.page_type == PageType::Allocated {
+            return;
+        }
         if page.crc32_checksum() == page.header.new_checksum {
             trace!("Page @ {:#x} byte has valid CRC32c checksum", file_offset);
         } else if page.innodb_checksum() == page.header.new_checksum {
@@ -173,6 +188,7 @@ impl PageExplorer {
                 }
                 self.explore_index(&index_page);
             }
+            PageType::Blob | PageType::LobFirst | PageType::LobData => {}
             _ => warn!("Unknown page type: {:?}", page.header.page_type),
         }
     }
@@ -223,8 +239,8 @@ impl PageExplorer {
         }
 
         info!(
-            "Processed {} pages, total records: {}, potentially missing: {}",
-            counter, self.total_records, self.missing_records
+            "Processed {} pages, total records: {}, potentially missing: {}, Incomplete: {}",
+            counter, self.total_records, self.missing_records, self.incomplete_records
         );
     }
 }
@@ -258,6 +274,7 @@ fn main() {
         output_writer: None,
         total_records: 0,
         missing_records: 0,
+        incomplete_records: 0,
     };
 
     if let Some(tablespace) = &args.tablespce_dir {

src/bin/page_extractor.rs

@@ -224,8 +224,7 @@ fn main() {
             PageValidationResult::InvalidChecksum => {
                 failed_checksum += 1;
             }
-            PageValidationResult::NotAPage |
-            PageValidationResult::EmptyPage => {}
+            PageValidationResult::NotAPage | PageValidationResult::EmptyPage => {}
         }
 
         head_pointer += step_size;

src/innodb/buffer_manager/lru.rs

@@ -19,7 +19,7 @@ use crate::innodb::{
 use anyhow::{anyhow, Result};
 use tracing::{trace, warn};
 
-const LRU_PAGE_COUNT: usize = 128;
+const LRU_PAGE_COUNT: usize = 16;
 
 pub struct LRUBufferManager {
     backing_store: Vec<[u8; FIL_PAGE_SIZE]>,
@@ -74,10 +74,14 @@ impl LRUBufferManager {
             let ((space_id, offset), _) = borrowed_pin_map
                 .iter()
                 .find(|(_, val)| **val == result_frame)
-                .expect("can't find the frame")
+                .expect(&format!(
+                    "can't find the frame({result_frame}), {:#?}, pinmap: {:#?}",
+                    self, borrowed_pin_map
+                ))
                 .to_owned();
             let (space_id, offset) = (*space_id, *offset);
             borrowed_pin_map.remove(&(space_id, offset));
+            self.lru_list.borrow_mut()[result_frame] = 0;
             return result_frame;
         } else {
             panic!("pin too many pages, \nState: {:#?}", self);
@@ -100,13 +104,16 @@ impl BufferManager for LRUBufferManager {
     fn pin(&self, space_id: u32, offset: u32) -> Result<PageGuard> {
         trace!("Pinning {}, {}", space_id, offset);
         let cur_sys_time = SystemTime::now();
-        let current_time = cur_sys_time.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_nanos() as u64;
+        let current_time = cur_sys_time
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap()
+            .as_nanos() as u64;
 
         // If we have the page already pinned
         if let Some(frame_number) = self.page_pin_map.borrow().get(&(space_id, offset)) {
             self.page_pin_counter.borrow_mut()[*frame_number] += 1;
             self.lru_list.borrow_mut()[*frame_number] = current_time;
-            let page = Page::from_bytes(&self.backing_store[offset as usize])?;
+            let page = Page::from_bytes(&self.backing_store[*frame_number])?;
             return Ok(PageGuard::new(page, self));
         }
 
@@ -119,18 +126,19 @@ impl BufferManager for LRUBufferManager {
             slice::from_raw_parts_mut(selected_frame.as_ptr() as *mut u8, FIL_PAGE_SIZE)
         })?;
 
-        self.lru_list.borrow_mut()[free_frame] = current_time;
+        // Validate page *FIRST*
         let page = Page::from_bytes(&self.backing_store[free_frame as usize])?;
         if page.header.space_id == 0 && page.header.offset == 0 {
             return Err(anyhow!(InnoDBError::PageNotFound));
         }
-
         assert_eq!(page.header.space_id, space_id);
         assert_eq!(page.header.offset, offset);
         assert_eq!(page.header.new_checksum, page.crc32_checksum());
 
-        self.page_pin_counter.borrow_mut()[free_frame] += 1;
+        // Can't fail from this point on, so we update internal state
 
+        self.lru_list.borrow_mut()[free_frame] = current_time;
+        self.page_pin_counter.borrow_mut()[free_frame] += 1;
         self.page_pin_map
             .borrow_mut()
             .insert((space_id, offset), free_frame);

src/innodb/buffer_manager/mod.rs

@@ -1,7 +1,7 @@
 use std::ops::Deref;
 
 use super::page::Page;
-use anyhow::{Result, anyhow};
+use anyhow::{anyhow, Result};
 
 pub mod lru;
 pub mod simple;
@@ -13,10 +13,10 @@ pub trait BufferManager {
 
 pub struct PageGuard<'a> {
     page: Page<'a>,
-    buffer_manager: &'a dyn BufferManager
+    buffer_manager: &'a dyn BufferManager,
 }
 
-impl <'a> PageGuard<'a> {
+impl<'a> PageGuard<'a> {
     pub fn new(page: Page<'a>, buffer_manager: &'a dyn BufferManager) -> Self {
         PageGuard {
             page,
@@ -25,15 +25,15 @@ impl <'a> PageGuard<'a> {
     }
 }
 
-impl <'a> Deref for PageGuard<'a> {
+impl<'a> Deref for PageGuard<'a> {
     type Target = Page<'a>;
 
     fn deref(&self) -> &Self::Target {
         &self.page
     }
 }
 
-impl <'a> Drop for PageGuard<'a> {
+impl<'a> Drop for PageGuard<'a> {
     fn drop(&mut self) {
         self.buffer_manager.unpin(std::mem::take(&mut self.page));
     }

src/innodb/buffer_manager/simple.rs

@@ -1,5 +1,4 @@
 use anyhow::Result;
-use tracing::trace;
 use std::{
     cell::RefCell,
     collections::HashMap,
@@ -8,6 +7,7 @@ use std::{
     path::{Path, PathBuf},
     slice,
 };
+use tracing::trace;
 
 use crate::innodb::page::{Page, FIL_PAGE_SIZE};
 
@@ -60,6 +60,10 @@ impl BufferManager for SimpleBufferManager {
     }
 
     fn unpin(&self, page: Page) {
-        trace!("Closed ({:?}, {})", page.header.space_id, page.header.offset);
+        trace!(
+            "Closed ({:?}, {})",
+            page.header.space_id,
+            page.header.offset
+        );
     }
 }

src/innodb/file_list.rs

@@ -32,7 +32,7 @@ impl FileAddress {
     pub fn is_null(&self) -> bool {
         self.page_number == FIL_NULL
     }
-    
+
     fn size() -> usize {
         6
     }
@@ -58,10 +58,10 @@ impl FileListBaseNode {
         Ok(FileListBaseNode {
             list_len,
             first_node,
-            last_node
+            last_node,
         })
     }
-    
+
     pub(crate) fn size() -> usize {
         4 + FileAddress::size() + FileAddress::size()
     }
@@ -79,14 +79,10 @@ impl FileListInnerNode {
             return Err(anyhow!("Buffer is too small"));
         }
 
-
         let prev = FileAddress::try_from_bytes(&buf[0..6])?;
         let next = FileAddress::try_from_bytes(&buf[6..12])?;
 
-        Ok(FileListInnerNode {
-            prev,
-            next
-        })
+        Ok(FileListInnerNode { prev, next })
     }
 
     pub fn size() -> usize {

src/innodb/mod.rs

@@ -1,8 +1,8 @@
-pub mod charset;
 pub mod buffer_manager;
+pub mod charset;
+pub mod file_list;
 pub mod page;
 pub mod table;
-pub mod file_list;
 
 use std::{
     error::Error,

src/innodb/page/lob/data_page.rs

@@ -1,6 +1,9 @@
 use anyhow::{anyhow, Result};
 
-use crate::innodb::{page::{Page, PageType}, InnoDBError};
+use crate::innodb::{
+    page::{Page, PageType},
+    InnoDBError,
+};
 
 #[derive(Debug, Clone)]
 pub struct LobDataHeader {
@@ -39,10 +42,10 @@ pub struct LobData<'a> {
     pub header: LobDataHeader,
 }
 
-impl <'a> LobData<'a> {
+impl<'a> LobData<'a> {
     pub fn try_from_page(p: &'a Page<'a>) -> Result<Self> {
         match p.header.page_type {
-            PageType::LobData => Ok(LobData{
+            PageType::LobData => Ok(LobData {
                 header: LobDataHeader::try_from_bytes(p.body())?,
                 page: p,
             }),

src/innodb/page/mod.rs

@@ -200,7 +200,7 @@ impl FILHeader {
         let page_type = match PageType::try_from_primitive(page_type_value) {
             Ok(page_type) => page_type,
             Err(e) => {
-                debug!("Invalid FIL PageType: {:?}", e);      
+                debug!("Invalid FIL PageType: {:?}", e);
                 PageType::Unknown
             }
         };

src/innodb/table/blob_header.rs

@@ -17,21 +17,14 @@ impl ExternReference {
             anyhow::bail!("Insufficient bytes to construct BlobHeader");
         }
 
-        let space_id = u32::from_be_bytes([
-            bytes[0], bytes[1], bytes[2], bytes[3]
-        ]);
+        let space_id = u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]);
 
-        let page_number = u32::from_be_bytes([
-            bytes[4], bytes[5], bytes[6], bytes[7]
-        ]);
+        let page_number = u32::from_be_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
 
-        let offset = u32::from_be_bytes([
-            bytes[8], bytes[9], bytes[10], bytes[11]
-        ]);
+        let offset = u32::from_be_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]);
 
         let length = u64::from_be_bytes([
-            bytes[12], bytes[13], bytes[14], bytes[15],
-            bytes[16], bytes[17], bytes[18], bytes[19]
+            bytes[12], bytes[13], bytes[14], bytes[15], bytes[16], bytes[17], bytes[18], bytes[19],
         ]);
 
         Ok(ExternReference {

src/innodb/table/field.rs

@@ -49,6 +49,7 @@ pub enum FieldValue {
     SignedInt(i64),
     UnsignedInt(u64),
     String(String),
+    PartialString { partial: String, total_len: usize },
     Null,
     Skipped,
 }
@@ -106,25 +107,23 @@ impl Field {
                 ),
                 len as usize,
             ),
-            FieldType::Text(max_len, _) => {
-                match length_opt {
-                    None => (FieldValue::Null, 0),
-                    Some(length) => {
-                        assert!(
-                            length <= self.field_type.max_len(),
-                            "Length larger than expected max? {} > {} in field {:?}",
-                            length,
-                            max_len,
-                            self
-                        );
-                        let str = String::from_utf8(buf[..length as usize].into())
-                            .expect("Failed parsing UTF-8")
-                            .trim_end()
-                            .to_string();
-                        (FieldValue::String(str), length as usize)
-                    }
+            FieldType::Text(max_len, _) => match length_opt {
+                None => (FieldValue::Null, 0),
+                Some(length) => {
+                    assert!(
+                        length <= self.field_type.max_len(),
+                        "Length larger than expected max? {} > {} in field {:?}",
+                        length,
+                        max_len,
+                        self
+                    );
+                    let str = String::from_utf8(buf[..length as usize].into())
+                        .expect("Failed parsing UTF-8")
+                        .trim_end()
+                        .to_string();
+                    (FieldValue::String(str), length as usize)
                 }
-            }
+            },
             #[allow(unreachable_patterns)]
             _ => {
                 unimplemented!("type = {:?}", self.field_type);

src/innodb/table/mod.rs

@@ -1,6 +1,6 @@
+pub mod blob_header;
 pub mod field;
 pub mod row;
-pub mod blob_header;
 
 use anyhow::{anyhow, Result};
 use field::{Field, FieldType};