Analyze method and page count of btree to determine potential io of full scan

Author: Alex Gaetano Padula

pager.go

@@ -21,9 +21,11 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 )
 
 const PAGE_SIZE = 1024  // Page size
@@ -37,6 +39,10 @@ type Pager struct {
 	deletedPagesFile *os.File                // file to store deleted pages
 	pageLocks        map[int64]*sync.RWMutex // locks for pages
 	pageLocksLock    *sync.RWMutex           // lock for pagesLocks
+	StatLock         *sync.RWMutex           // lock for stats
+	cachedCount      int64                   // cached count of pages
+	lastCacheTime    time.Time               // last time the cache was updated
+	interval         time.Duration           // interval to update the cache
 }
 
 // OpenPager opens a file for page management
@@ -70,7 +76,7 @@ func OpenPager(filename string, flag int, perm os.FileMode) (*Pager, error) {
 		pgLocks[i] = &sync.RWMutex{}
 	}
 
-	return &Pager{file: file, deletedPages: deletedPages, deletedPagesFile: deletedPagesFile, deletedPagesLock: &sync.Mutex{}, pageLocks: pgLocks, pageLocksLock: &sync.RWMutex{}}, nil
+	return &Pager{file: file, deletedPages: deletedPages, deletedPagesFile: deletedPagesFile, deletedPagesLock: &sync.Mutex{}, pageLocks: pgLocks, pageLocksLock: &sync.RWMutex{}, StatLock: &sync.RWMutex{}, interval: time.Minute * 10, lastCacheTime: time.Now().Add(time.Minute * 10)}, nil
 }
 
 // writeDelPages writes the deleted pages that are in-memory to the deleted pages file
@@ -150,6 +156,10 @@ func splitDataIntoChunks(data []byte) [][]byte {
 
 // WriteTo writes data to a specific page
 func (p *Pager) WriteTo(pageID int64, data []byte) error {
+	// lock the page
+	p.getPageLock(pageID).Lock()
+	defer p.getPageLock(pageID).Unlock()
+
 	p.DeletePage(pageID)
 	// remove from deleted pages
 	p.deletedPagesLock.Lock()
@@ -161,6 +171,7 @@ func (p *Pager) WriteTo(pageID int64, data []byte) error {
 		}
 
 	}
+
 	// the reason we are doing this is because we are going to write to the page thus having any overflowed pages which are linked to the page may not be needed
 
 	// check if data is larger than the page size
@@ -317,6 +328,19 @@ func (p *Pager) Close() error {
 // GetPage gets a page and returns the data
 // Will gather all the pages that are linked together
 func (p *Pager) GetPage(pageID int64) ([]byte, error) {
+
+	// lock the page
+	p.getPageLock(pageID).Lock()
+	defer p.getPageLock(pageID).Unlock()
+
+	p.deletedPagesLock.Lock()
+	// Check if in deleted pages, if so return nil
+	if slices.Contains(p.deletedPages, pageID) {
+		p.deletedPagesLock.Unlock()
+		return nil, nil
+	}
+	p.deletedPagesLock.Unlock()
+
 	result := make([]byte, 0)
 
 	// get the page
@@ -389,6 +413,13 @@ func (p *Pager) GetPage(pageID int64) ([]byte, error) {
 	return result, nil
 }
 
+// GetDeletedPages returns the list of deleted pages
+func (p *Pager) GetDeletedPages() []int64 {
+	p.deletedPagesLock.Lock()
+	defer p.deletedPagesLock.Unlock()
+	return p.deletedPages
+}
+
 // DeletePage deletes a page
 func (p *Pager) DeletePage(pageID int64) error {
 	p.deletedPagesLock.Lock()
@@ -406,7 +437,70 @@ func (p *Pager) DeletePage(pageID int64) error {
 	return nil
 }
 
-func (p *Pager) Count() int64 {
+// Analyze forces the pager to analyze the file
+func (p *Pager) Analyze() error {
+	p.StatLock.Lock()
+	defer p.StatLock.Unlock()
+
+	// Initialize a counter for the pages
+	var pageCount int64 = 0
+
+	// Get the size of the file
 	stat, _ := p.file.Stat()
-	return stat.Size() / (PAGE_SIZE + HEADER_SIZE)
+	fileSize := stat.Size()
+
+	// Initialize a counter for the bytes read
+	var bytesRead int64 = 0
+
+	// Read through the file in chunks of PAGE_SIZE + HEADER_SIZE bytes
+	for bytesRead < fileSize {
+		bytesRead += PAGE_SIZE + HEADER_SIZE
+		pageCount++
+	}
+
+	p.cachedCount = pageCount
+
+	return nil
+}
+
+// Count returns the number of pages
+func (p *Pager) Count() int64 {
+
+	if p.cachedCount == 0 {
+		p.Analyze()
+
+	}
+
+	if time.Since(p.lastCacheTime) > p.interval {
+		time.Now()
+
+		p.StatLock.Lock()
+		defer p.StatLock.Unlock()
+
+		// Initialize a counter for the pages
+		var pageCount int64 = 0
+
+		// Get the size of the file
+		stat, _ := p.file.Stat()
+		fileSize := stat.Size()
+
+		// Initialize a counter for the bytes read
+		var bytesRead int64 = 0
+
+		// Read through the file in chunks of PAGE_SIZE + HEADER_SIZE bytes
+		for bytesRead < fileSize {
+			bytesRead += PAGE_SIZE + HEADER_SIZE
+			pageCount++
+		}
+
+		p.cachedCount = pageCount
+
+		// Subtract the number of deleted pages
+		//pageCount -= int64(len(p.GetDeletedPages()))
+
+		return pageCount
+	} else {
+		return p.cachedCount
+	}
+
 }