merged_modules_and_tests.txt

Folder Structure:


=== Begin File Contents ===




=== Folder: modules/chain ===


--- File: modules/chain/index.js ---

// modules/chain/index.js
export { EnhancedPatternOptimizer } from './optimizer';
export { ChainUtil } from './utils/chainUtil';
--- End of modules/chain/index.js ---


--- File: modules/chain/optimizer.js ---

const { AsyncLock } = require('../concurrency');
const { LRUCache } = require('../memory/cache/lru');
const { ThoughtError } = require('../errors/thoughtError');
const { ChainUtil } = require("./utils/chainUtil")

class EnhancedPatternOptimizer {
    constructor(performanceTracker) {
        this.performanceTracker = performanceTracker;
        this.patterns = new Map();
        this.cache = new LRUCache(1000);
        this.learningRate = 0.1;

        // Add AI configuration
        this.aiConfig = {
            endpoint: null,
            apiKey: null,
            optimizerModel: null
        };
    }

    async optimizeChain(thoughtChain) {
        const chainHash = this.hashChain(thoughtChain);
        const cachedOptimization = this.cache.get(chainHash);

        if (cachedOptimization) {
            // Validate cached optimization
            try {
                await this.validateOptimization(cachedOptimization, thoughtChain);
                return cachedOptimization;
            } catch (error) {
                console.warn('Cached optimization invalid:', error);
                this.cache.delete(chainHash); // Remove invalid cache entry
            }
        }

        const optimizedChain = await this.analyzeAndOptimize(thoughtChain);
        this.cache.set(chainHash, optimizedChain);
        return optimizedChain;
    }

    async validateOptimization(optimization, originalChain) {
        // Ensure all thoughts exist
        const thoughtIds = new Set(originalChain.map(t => t.id));
        for (const thought of optimization) {
            if (!thoughtIds.has(thought.id)) {
                throw new Error(`Optimization references non-existent thought: ${thought.id}`);
            }
        }

        // Validate dependencies
        for (const thought of optimization) {
            if (thought.dependencies) {
                for (const depId of thought.dependencies) {
                    if (!thoughtIds.has(depId)) {
                        throw new Error(`Invalid dependency: ${depId} in thought ${thought.id}`);
                    }
                }
            }
        }

        // Ensure no circular dependencies
        this.checkForCircularDependencies(optimization);

        return true;
    }

    checkForCircularDependencies(chain) {
        const visited = new Set();
        const recursionStack = new Set();

        const dfs = (thoughtId) => {
            if (recursionStack.has(thoughtId)) {
                throw new Error(`Circular dependency detected: ${thoughtId}`);
            }
            if (visited.has(thoughtId)) {
                return;
            }

            visited.add(thoughtId);
            recursionStack.add(thoughtId);

            const thought = chain.find(t => t.id === thoughtId);
            if (thought && thought.dependencies) {
                for (const depId of thought.dependencies) {
                    dfs(depId);
                }
            }

            recursionStack.delete(thoughtId);
        };

        for (const thought of chain) {
            dfs(thought.id);
        }
    }

    async analyzeAndOptimize(thoughtChain) {
        if (!this.aiConfig || !this.aiConfig.optimizerModel) {
            return this.baselineOptimization(thoughtChain);
        }
    
        let attempts = 2;
        while (attempts > 0) {
            try {
                const response = await fetch(`${this.aiConfig.endpoint}/optimize`, {
                    method: 'POST',
                    headers: {
                        'Authorization': `Bearer ${this.aiConfig.apiKey}`,
                        'Content-Type': 'application/json'
                    },
                    body: JSON.stringify({
                        chain: thoughtChain,
                        model: this.aiConfig.optimizerModel
                    })
                });
    
                if (!response.ok) {
                    const text = await response.text();
                    if (text.includes('Rate limited') && attempts > 1) {
                        attempts--;
                        continue; // retry once
                    } else {
                        throw new Error('API Error');
                    }
                }
    
                const aiOptimization = await response.json();
                const merged = this.mergeWithBaselineOptimization(
                    aiOptimization,
                    await this.baselineOptimization(thoughtChain)
                );
                return merged;
            } catch (error) {
                console.error('AI optimization failed, using baseline:', error);
                return this.baselineOptimization(thoughtChain);
            }
        }
    }
    
    

    async baselineOptimization(thoughtChain) {
        try {
            // Build dependency graph
            const graph = this.buildDependencyGraph(thoughtChain);
            
            // Find all possible execution paths
            const paths = this.findExecutionPaths(graph);
            
            // Optimize each path
            const optimizedPaths = await Promise.all(
                paths.map(path => this.optimizePath(path))
            );

            // Analyze metrics for each optimized path
            const pathMetrics = await Promise.all(
                optimizedPaths.map(async path => ({
                    path,
                    metrics: await this.calculatePathMetrics(path)
                }))
            );

            // Select best path based on metrics
            const bestPath = this.selectOptimalPath(pathMetrics);
            
            // Apply performance patterns
            const patternOptimizedPath = await this.applyPerformancePatterns(bestPath);
            
            // Validate final optimization
            await this.validateOptimization(patternOptimizedPath, thoughtChain);
            
            return patternOptimizedPath;
        } catch (error) {
            console.error('Baseline optimization failed:', error);
            // Return original chain if optimization fails
            return thoughtChain;
        }
    }

    buildDependencyGraph(thoughtChain) {
        const graph = new Map();

        thoughtChain.forEach(thought => {
            graph.set(thought.id, {
                thought,
                dependencies: new Set(thought.dependencies || []),
                dependents: new Set()
            });
        });

        // Build reverse dependencies
        thoughtChain.forEach(thought => {
            (thought.dependencies || []).forEach(depId => {
                const node = graph.get(depId);
                if (node) {
                    node.dependents.add(thought.id);
                }
            });
        });

        return graph;
    }

    findExecutionPaths(graph) {
        const paths = [];
        const visited = new Set();

        // Find root nodes (nodes with no dependencies)
        const roots = Array.from(graph.entries())
            .filter(([_, node]) => node.dependencies.size === 0)
            .map(([id]) => id);

        // DFS from each root
        roots.forEach(root => {
            const path = ChainUtil.explorePath(root, graph, visited, []);
            if (path.length > 0) {
                paths.push(path);
            }
        });

        return paths;
    }

    mergeWithBaselineOptimization(aiOptimization, baselineOptimization) {
        if (!Array.isArray(aiOptimization) || !Array.isArray(baselineOptimization)) {
            return baselineOptimization || [];
        }
    
        // Merge by taking unique thoughts keyed by 'id'
        const thoughtMap = new Map();
        baselineOptimization.forEach(t => thoughtMap.set(t.id, t));
        aiOptimization.forEach(t => thoughtMap.set(t.id, t));
    
        // Return only a filtered set of final thoughts
        return Array.from(thoughtMap.values());
    }
    

    async optimizePath(path) {
        const optimizedPath = [];
        let currentBatch = [];

        for (const thoughtId of path) {
            const thought = this.getThought(thoughtId);
            const metrics = await this.analyzeThought(thought);

            // Check if thought can be batched
            if (this.canBatch(thought, currentBatch, metrics)) {
                currentBatch.push(thought);
            } else {
                if (currentBatch.length > 0) {
                    optimizedPath.push(this.createBatch(currentBatch));
                    currentBatch = [thought];
                } else {
                    optimizedPath.push(thought);
                }
            }
        }

        // Handle remaining batch
        if (currentBatch.length > 0) {
            optimizedPath.push(this.createBatch(currentBatch));
        }

        return optimizedPath;
    }

    canBatch(thought, batch, metrics) {
        if (batch.length === 0) return true;

        // Check resource usage
        const batchMetrics = batch.map(t => this.getThoughtMetrics(t.id));
        const totalResources = this.calculateTotalResources(batchMetrics);

        // Check if adding this thought would exceed resource limits
        return this.checkResourceLimits(totalResources, metrics);
    }

    createBatch(thoughts) {
        return {
            id: `batch_${thoughts.map(t => t.id).join('_')}`,
            type: 'batch',
            thoughts: thoughts,
            execute: async (context) => {
                return Promise.all(thoughts.map(t => t.execute(context)));
            }
        };
    }

    checkResourceLimits(currentTotal, newMetrics) {
        // Implement resource limit checks
        const limits = {
            memory: 1000000000, // 1GB
            cpu: 0.8, // 80% CPU
            time: 1000 // 1 second
        };

        return Object.entries(limits).every(([resource, limit]) => {
            return (currentTotal[resource] || 0) + (newMetrics[resource] || 0) <= limit;
        });
    }

    async calculatePathMetrics(path) {
        const metrics = {
            totalExecutionTime: 0,
            peakMemoryUsage: 0,
            cpuUtilization: 0,
            resourceEfficiency: 0
        };

        for (const thought of path) {
            const thoughtMetrics = await this.analyzeThought(thought);
            metrics.totalExecutionTime += thoughtMetrics.executionTime || 0;
            metrics.peakMemoryUsage = Math.max(metrics.peakMemoryUsage, thoughtMetrics.memoryUsage || 0);
            metrics.cpuUtilization = Math.max(metrics.cpuUtilization, thoughtMetrics.cpuUsage || 0);
        }

        // Calculate resource efficiency score
        metrics.resourceEfficiency = this.calculateEfficiencyScore(metrics);
        return metrics;
    }

    calculateEfficiencyScore(metrics) {
        const weights = {
            executionTime: 0.4,
            memoryUsage: 0.3,
            cpuUtilization: 0.3
        };

        return (
            (1 / (metrics.totalExecutionTime + 1)) * weights.executionTime +
            (1 / (metrics.peakMemoryUsage + 1)) * weights.memoryUsage +
            (1 / (metrics.cpuUtilization + 1)) * weights.cpuUtilization
        );
    }

    selectOptimalPath(pathMetrics) {
        return pathMetrics.reduce((best, current) => {
            if (!best || current.metrics.resourceEfficiency > best.metrics.resourceEfficiency) {
                return current;
            }
            return best;
        }).path;
    }

    async applyPerformancePatterns(path) {
        const optimizedPath = [...path];
        
        // Apply known optimization patterns
        for (const [patternName, pattern] of this.patterns) {
            try {
                const patternResult = await pattern.apply(optimizedPath);
                if (patternResult.improved) {
                    optimizedPath.splice(0, optimizedPath.length, ...patternResult.path);
                }
            } catch (error) {
                console.error(`Failed to apply pattern ${patternName}:`, error);
            }
        }

        return optimizedPath;
    }

    async hashChain(thoughtChain) {
        const chainData = thoughtChain.map(t => ({
            id: t.id,
            dependencies: t.dependencies || []
        }));
        
        const encoder = new TextEncoder();
        const data = encoder.encode(JSON.stringify(chainData));
        const hashBuffer = await crypto.subtle.digest('SHA-256', data);
        const hashArray = Array.from(new Uint8Array(hashBuffer));
        return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
    }

    async getThought(thoughtId) {
        // Implementation would depend on how thoughts are stored
        // This is a placeholder that should be implemented based on the actual storage mechanism
        return this.performanceTracker.getThought(thoughtId);
    }

    async analyzeThought(thought) {
        // This would be implemented based on the actual metrics collection system
        return this.performanceTracker.getMetrics(thought.id);
    }

    calculateTotalResources(metrics) {
        return metrics.reduce((total, metric) => {
            Object.entries(metric).forEach(([key, value]) => {
                total[key] = (total[key] || 0) + value;
            });
            return total;
        }, {});
    }

    async getThoughtMetrics(thoughtId) {
        // This would be implemented based on the actual metrics collection system
        return this.performanceTracker.getThoughtMetrics(thoughtId);
    }
}

module.exports = { EnhancedPatternOptimizer };
--- End of modules/chain/optimizer.js ---



=== Folder: modules/chain/utils ===


--- File: modules/chain/utils/chainUtil.js ---

class ChainUtil {
    static hashChain(thoughtChain) {
        const chainString = JSON.stringify(thoughtChain.map(t => ({
            id: t.id,
            dependencies: t.dependencies || []
        })));

        // Create hash using available API
        if (typeof crypto !== 'undefined' && crypto.subtle) {
            return crypto.subtle.digest('SHA-256', new TextEncoder().encode(chainString))
                .then(hash => Array.from(new Uint8Array(hash))
                    .map(b => b.toString(16).padStart(2, '0'))
                    .join(''));
        }

        // Simple fallback hash
        let hash = 0;
        for (let i = 0; i < chainString.length; i++) {
            const char = chainString.charCodeAt(i);
            hash = ((hash << 5) - hash) + char;
            hash = hash & hash;
        }
        return hash.toString(36);
    }

    static explorePath(nodeId, graph, visited = new Set(), currentPath = [], paths = []) {
        currentPath.push(nodeId);
    
        const node = graph.get(nodeId);
        if (!node) {
            currentPath.pop();
            return paths;
        }
    
        let hasDependents = false;
        for (const dependentId of node.dependents) {
            if (currentPath.includes(dependentId)) {
                throw new Error(`Circular dependency detected: ${nodeId} -> ${dependentId}`);
            }
    
            if (!visited.has(dependentId)) {
                hasDependents = true;
                ChainUtil.explorePath(dependentId, graph, visited, currentPath, paths);
            }
        }
    
        // If this is a leaf node (no dependents) or all dependents are visited
        if (!hasDependents) {
            // Make a copy of currentPath and add to paths
            paths.push([...currentPath]);
        }
    
        visited.add(nodeId);
        currentPath.pop();
        return paths;
    }
    
    

    static combinePaths(paths) {
        const merged = {
            parallel: new Set(),
            sequential: new Map()
        };
    
        paths.forEach(path => {
            if (Array.isArray(path)) {
                // Single linear path
                path.forEach((nodeId, index) => {
                    if (index === 0) {
                        merged.parallel.add(nodeId);
                    } else {
                        const existingDeps = merged.sequential.get(nodeId) || [];
                        const newDeps = path.slice(0, index);
                        merged.sequential.set(nodeId, [...new Set([...existingDeps, ...newDeps])]);
                    }
                });
            } else {
                // Pre-optimized path: { parallel: [...], sequential: [[nodeId, deps], ...] }
                path.parallel.forEach(nodeId => merged.parallel.add(nodeId));
                path.sequential.forEach(([nodeId, deps]) => {
                    const existingDeps = merged.sequential.get(nodeId) || [];
                    merged.sequential.set(nodeId, [...new Set([...existingDeps, ...deps])]);
                });
            }
        });
    
        return {
            parallel: Array.from(merged.parallel),
            sequential: Array.from(merged.sequential.entries())
        };
    }
    
    
}

export { ChainUtil };
--- End of modules/chain/utils/chainUtil.js ---



=== Folder: modules/concurrency ===


--- File: modules/concurrency/asyncLock.js ---

class AsyncLock {
    constructor() {
        this.locks = new Map();
        this.waiting = new Map();
        this.debug = false;
        this.timeout = 10000;
        this.metrics = {
            acquireCount: 0,
            timeouts: 0,
            contentionCount: 0,
            totalWaitTime: 0
        };
    }

    async acquire(key, fn, timeout = this.timeout) {
        const startTime = Date.now();

        // Initialize waiting list if needed
        if (!this.waiting.has(key)) {
            this.waiting.set(key, []);
        }

        // Create waiting promise for this acquisition attempt
        let waitResolve;
        const waitPromise = new Promise(resolve => {
            waitResolve = resolve;
        });

        // Add to waiting list
        const waiters = this.waiting.get(key);
        waiters.push(waitResolve);

        try {
            // If there's an existing lock, wait for it or timeout
            if (this.locks.has(key)) {
                const timeoutPromise = this.createTimeout(timeout);
                try {
                    await Promise.race([this.locks.get(key), timeoutPromise]);
                } catch (error) {
                    // Remove from waiting list before throwing timeout error
                    const index = waiters.indexOf(waitResolve);
                    if (index !== -1) {
                        waiters.splice(index, 1);
                    }
                    if (waiters.length === 0) {
                        this.waiting.delete(key);
                    }
                    this.metrics.timeouts++;
                    throw error;
                }
            }

            // Create new lock promise
            let lockResolve;
            const lockPromise = new Promise(resolve => {
                lockResolve = resolve;
            });
            this.locks.set(key, lockPromise);

            // Update metrics
            this.metrics.acquireCount++;
            if (waiters.length > 1) {
                this.metrics.contentionCount++;
            }

            try {
                // Execute function
                const result = await fn();
                return result;
            } catch (error) {
                // For function execution errors, ensure cleanup before rethrowing
                const index = waiters.indexOf(waitResolve);
                if (index !== -1) {
                    waiters.splice(index, 1);
                }
                if (waiters.length === 0) {
                    this.waiting.delete(key);
                }
                throw error;
            } finally {
                const waitTime = Date.now() - startTime;
                this.metrics.totalWaitTime += waitTime;

                // Clean up waiting list
                const index = waiters.indexOf(waitResolve);
                if (index !== -1) {
                    waiters.splice(index, 1);
                }
                if (waiters.length === 0) {
                    this.waiting.delete(key);
                }

                // Release lock
                lockResolve();
                if (this.locks.get(key) === lockPromise) {
                    this.locks.delete(key);
                }
            }
        } catch (error) {
            // Handle any unexpected errors
            throw error;
        }
    }

    async tryAcquire(key, fn, timeout = 0) {
        if (this.isLocked(key)) {
            return null;
        }
        try {
            return await this.acquire(key, fn, timeout);
        } catch (error) {
            if (error.name === 'LockTimeoutError') {
                return null;
            }
            throw error;
        }
    }

    async acquireMultiple(keys, fn, timeout = this.timeout) {
        // Sort keys to prevent deadlocks
        const sortedKeys = [...new Set(keys)].sort();
        
        // Acquire all locks in order
        const acquire = async (index) => {
            if (index >= sortedKeys.length) {
                return await fn();
            }
            
            return await this.acquire(sortedKeys[index], async () => {
                return await acquire(index + 1);
            }, timeout);
        };
        
        return await acquire(0);
    }

    async withLock(key, fn, timeout = this.timeout) {
        return await this.acquire(key, fn, timeout);
    }

    createTimeout(timeout) {
        return new Promise((_, reject) => {
            setTimeout(() => {
                const error = new Error('Lock acquisition timed out');
                error.name = 'LockTimeoutError';
                reject(error);
            }, timeout);
        });
    }

    getWaitingCount(key) {
        const waiters = this.waiting.get(key);
        if (!waiters) return 0;
        // If there's an active lock, don't count the first waiter as it's currently executing
        return this.locks.has(key) ? waiters.length - 1 : waiters.length;
    }
    
    isLocked(key) {
        return this.locks.has(key) || (this.waiting.get(key)?.length > 0) || false;
    }

    async isBusy() {
        return this.locks.size > 0 || Array.from(this.waiting.values()).some(list => list.length > 0);
    }

    getMetrics() {
        const totalOperations = this.metrics.acquireCount || 1;
        return {
            ...this.metrics,
            averageWaitTime: this.metrics.totalWaitTime / totalOperations,
            contentionRate: this.metrics.contentionCount / totalOperations,
            timeoutRate: this.metrics.timeouts / totalOperations
        };
    }


    async releaseAll() {
        // Copy all keys to avoid modification during iteration
        const lockKeys = Array.from(this.locks.keys());
        const waitingKeys = Array.from(this.waiting.keys());
    
        // Release all active locks
        for (const key of lockKeys) {
            const lockPromise = this.locks.get(key);
            if (lockPromise) {
                // Resolve the lock promise
                await lockPromise;
                this.locks.delete(key);
            }
        }
    
        // Release all waiting operations
        for (const key of waitingKeys) {
            const waiters = this.waiting.get(key);
            if (waiters) {
                // Copy waiters array to avoid modification during iteration
                const waitersCopy = [...waiters];
                // Resolve all waiters
                for (const resolve of waitersCopy) {
                    resolve();
                }
                this.waiting.delete(key);
            }
        }
    }

    reset() {
        this.releaseAll();
        this.metrics = {
            acquireCount: 0,
            timeouts: 0,
            contentionCount: 0,
            totalWaitTime: 0
        };
    }

    setDebug(enabled) {
        this.debug = enabled;
    }

    setTimeout(timeout) {
        if (timeout <= 0) {
            throw new Error('Timeout must be greater than 0');
        }
        this.timeout = timeout;
    }

    async isBusy() {
        return this.locks.size > 0 || Array.from(this.waiting.values())
            .some(list => list.length > 0);
    }
}

export {AsyncLock}
--- End of modules/concurrency/asyncLock.js ---


--- File: modules/concurrency/index.js ---

import { AsyncLock } from './asyncLock.js';

export { AsyncLock };

// For CommonJS compatibility
if (typeof module !== 'undefined' && module.exports) {
    module.exports = { AsyncLock };
}
--- End of modules/concurrency/index.js ---



=== Folder: modules/config ===


--- File: modules/config/aiConfig.js ---

const aiConfig = {
    endpoints: {
        embedding: 'https://api.openai.com/v1/embeddings',
        compression: 'https://api.example.com/v1/compress',
        optimization: 'https://api.example.com/v1/optimize'
    },
    models: {
        embedding: 'text-embedding-ada-002',
        compression: 'compression-model',
        optimization: 'chain-optimizer-model'
    },
    apiKey: 'your-api-key-here',
    options: {
        maxRetries: 3,
        timeout: 10000,
        batchSize: 32
    }
};

export default aiConfig;
--- End of modules/config/aiConfig.js ---


--- File: modules/config/index.js ---

// modules/config/index.js
export { default as aiConfig } from './aiConfig';
--- End of modules/config/index.js ---



=== Folder: modules/debug ===


--- File: modules/debug/debugSystem.js ---

import { AsyncLock } from '../concurrency';
import { MetricsCollector } from './metrics/collector';
import { ThoughtError } from '../errors/thoughtError';

class EnhancedDebugSystem {
    constructor(config = {}) {
        this.logs = new Map();
        this.breakpoints = new Set();
        this.metrics = new MetricsCollector();
        this.alertThresholds = config.alertThresholds || {
            memory: 0.8,    // 80% of available memory
            cpu: 0.9,       // 90% CPU usage
            time: 5000,     // 5 seconds
            errors: 10      // Number of errors per minute
        };
        this.monitoringIntervalId = null;
        this.errorCount = new Map(); // Track errors per minute
        this.debugHooks = new Map(); // Custom debug hooks
        this.setupMonitoring();
    }

    setupMonitoring() {
        if (this.monitoringIntervalId) {
            clearInterval(this.monitoringIntervalId);
        }
        this.monitoringIntervalId = setInterval(() => this.checkSystemHealth(), 5000);
    }

    async destroy() {
        try {
            if (this.monitoringIntervalId) {
                clearInterval(this.monitoringIntervalId);
                this.monitoringIntervalId = null;
            }
    
            try {
                await this.clearLogs();
            } catch (error) {
                console.error('Error clearing logs:', error);
                // Continue with cleanup despite log clearing error
            }
    
            this.breakpoints.clear();
            this.errorCount.clear();
            this.debugHooks.clear();
            
            return true;
        } catch (error) {
            console.error('Error during debug system destruction:', error);
            throw error;
        }
    }

    async checkSystemHealth() {
        try {
            const metrics = await this.metrics.collect();
            const alerts = [];

            // Check memory usage
            if (metrics.memory.heapUsedPercentage > this.alertThresholds.memory) {
                alerts.push({
                    type: 'MemoryAlert',
                    value: metrics.memory.heapUsedPercentage,
                    threshold: this.alertThresholds.memory,
                    timestamp: Date.now()
                });
            }

            // Check CPU usage
            if (metrics.cpu.percentage > this.alertThresholds.cpu) {
                alerts.push({
                    type: 'CPUAlert',
                    value: metrics.cpu.percentage,
                    threshold: this.alertThresholds.cpu,
                    timestamp: Date.now()
                });
            }

            // Clean up old error counts
            const oneMinuteAgo = Date.now() - 60000;
            for (const [key, value] of this.errorCount) {
                if (value.timestamp < oneMinuteAgo) {
                    this.errorCount.delete(key);
                }
            }

            if (alerts.length > 0) {
                this.handleAlerts(alerts);
            }

            // Store health check results
            this.logs.set('healthCheck', {
                timestamp: Date.now(),
                metrics,
                alerts
            });
        } catch (error) {
            this.log('error', 'Health check failed', { error: error.message });
        }
    }

    handleAlerts(alerts) {
        for (const alert of alerts) {
            this.log('alert', `System alert: ${alert.type}`, alert);

            // Execute any registered alert handlers
            const handler = this.debugHooks.get(alert.type);
            if (handler) {
                try {
                    handler(alert);
                } catch (error) {
                    this.log('error', 'Alert handler failed', {
                        alertType: alert.type,
                        error: error.message
                    });
                }
            }
        }
    }

    async inspectThought(thought) {
        const inspection = {
            id: thought.id,
            type: thought.type,
            timestamp: Date.now(),
            memory: await this.metrics.getMemoryProfile(thought),
            traces: await this.collectTraces(thought),
            resourceUsage: await this.metrics.getResourceUsage(thought),
            executionGraph: this.generateExecutionGraph(thought),
            dependencies: await this.analyzeDependencies(thought)
        };

        this.logs.set(`thought_${thought.id}`, inspection);
        return inspection;
    }

    async collectTraces(thought) {
        return {
            executionPath: this.captureExecutionPath(thought),
            functionCalls: await this.traceFunctionCalls(thought),
            resourceAccess: this.trackResourceAccess(thought),
            timing: this.measureExecutionTimes(thought)
        };
    }

    async traceFunctionCalls(thought) {
        const calls = [];
        const originalFunctions = new Map();
    
        // Store original functions and create proxies
        for (const key in thought) {
            if (typeof thought[key] === 'function' && key !== 'then') { // Exclude Promise methods
                originalFunctions.set(key, thought[key]);
                thought[key] = new Proxy(thought[key], {
                    apply: async (target, thisArg, args) => {
                        const start = performance.now();
                        const callInfo = {
                            function: key,
                            arguments: args.map(arg => this.sanitizeArg(arg)),
                            timestamp: Date.now()
                        };
    
                        try {
                            const result = await target.apply(thisArg, args);
                            callInfo.duration = performance.now() - start;
                            callInfo.status = 'success';
                            callInfo.result = this.sanitizeArg(result);
                            calls.push(callInfo); // Move this inside try block
                            return result;
                        } catch (error) {
                            callInfo.duration = performance.now() - start;
                            callInfo.status = 'error';
                            callInfo.error = error.message;
                            calls.push(callInfo); // Also track failed calls
                            throw error;
                        }
                    }
                });
            }
        }
    
        // Ensure function was actually called
        await thought.execute();
    
        // Restore original functions
        for (const [key, originalFn] of originalFunctions) {
            thought[key] = originalFn;
        }
    
        return calls;
    }

    sanitizeArg(arg) {
        // Safely stringify arguments and results for logging
        try {
            if (arg === undefined) return 'undefined';
            if (arg === null) return 'null';
            if (typeof arg === 'function') return 'function';
            if (typeof arg === 'object') {
                return JSON.stringify(arg, (key, value) => {
                    if (typeof value === 'function') return 'function';
                    if (value instanceof Error) return value.message;
                    return value;
                });
            }
            return String(arg);
        } catch (error) {
            return '[Complex Object]';
        }
    }

    captureExecutionPath(thought) {
        const path = [];
        let currentNode = thought;

        while (currentNode) {
            path.push({
                id: currentNode.id,
                type: currentNode.type,
                timestamp: currentNode.timestamp
            });
            currentNode = currentNode.parent;
        }

        return path;
    }

    trackResourceAccess(thought) {
        return {
            memory: this.getMemoryAccess(thought),
            storage: this.getStorageAccess(thought),
            network: this.getNetworkAccess(thought)
        };
    }

    measureExecutionTimes(thought) {
        return {
            total: 0,
            phases: [],
            bottlenecks: []
        };
    }

    log(level, message, context = {}) {
        const logEntry = {
            timestamp: Date.now(),
            level,
            message,
            context
        };

        // Store in logs map
        const key = `${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
        this.logs.set(key, logEntry);

        // Trim old logs if necessary
        if (this.logs.size > 1000) { // Keep last 1000 logs
            const oldestKey = Array.from(this.logs.keys())[0];
            this.logs.delete(oldestKey);
        }

        return logEntry;
    }

    clearLogs() {
        this.logs.clear();
    }

    getLogsByLevel(level) {
        return Array.from(this.logs.values())
            .filter(entry => entry.level === level);
    }

    setBreakpoint(thoughtId, condition) {
        this.breakpoints.set(thoughtId, condition);
    }

    removeBreakpoint(thoughtId) {
        this.breakpoints.delete(thoughtId);
    }

    registerDebugHook(event, handler) {
        if (typeof handler !== 'function') {
            throw new Error('Debug hook handler must be a function');
        }
        this.debugHooks.set(event, handler);
    }

    getDebugSnapshot() {
        return {
            timestamp: Date.now(),
            logs: Array.from(this.logs.entries()),
            metrics: this.metrics.getLatestMetrics(),
            breakpoints: Array.from(this.breakpoints),
            errorCounts: Array.from(this.errorCount.entries())
        };
    }

    generateExecutionGraph(thought) {
        const graph = {
            nodes: [],
            edges: []
        };

        const visited = new Set();

        const addNode = (node) => {
            if (visited.has(node.id)) return;
            visited.add(node.id);

            graph.nodes.push({
                id: node.id,
                type: node.type,
                timestamp: node.timestamp
            });

            if (node.dependencies) {
                node.dependencies.forEach(depId => {
                    graph.edges.push({
                        from: depId,
                        to: node.id
                    });
                });
            }

            if (node.children) {
                node.children.forEach(child => {
                    addNode(child);
                    graph.edges.push({
                        from: node.id,
                        to: child.id
                    });
                });
            }
        };

        addNode(thought);
        return graph;
    }

    async analyzeDependencies(thought) {
        const dependencies = {
            direct: [],
            indirect: [],
            circular: []
        };

        const visited = new Set();
        const stack = new Set();

        const analyzeDep = async (node) => {
            if (stack.has(node.id)) {
                dependencies.circular.push(node.id);
                return;
            }

            if (visited.has(node.id)) return;

            stack.add(node.id);
            visited.add(node.id);

            if (node.dependencies) {
                for (const depId of node.dependencies) {
                    dependencies.direct.push({
                        from: node.id,
                        to: depId
                    });

                    const depNode = await this.getThought(depId);
                    if (depNode) {
                        await analyzeDep(depNode);
                    }
                }
            }

            stack.delete(node.id);
        };

        await analyzeDep(thought);
        return dependencies;
    }

    async getThought(thoughtId) {
        // Implementation would depend on how thoughts are stored/retrieved
        // This is a placeholder
        return null;
    }

    getMemoryAccess(thought) {
        return {
            reads: [],
            writes: [],
            allocations: []
        };
    }

    getStorageAccess(thought) {
        return {
            reads: [],
            writes: [],
            deletes: []
        };
    }

    getNetworkAccess(thought) {
        return {
            requests: [],
            responses: [],
            errors: []
        };
    }
}

export {EnhancedDebugSystem}
--- End of modules/debug/debugSystem.js ---


--- File: modules/debug/index.js ---

// modules/debug/index.js
export { EnhancedDebugSystem } from './debugSystem';
export { MetricsCollector } from './metrics/collector';
--- End of modules/debug/index.js ---



=== Folder: modules/debug/metrics ===


--- File: modules/debug/metrics/collector.js ---

import { AsyncLock } from '../../concurrency';
import os from 'os';  // Node.js built-in

class MetricsCollector {
    constructor(config = {}) {
        this.metricsHistory = new Map();
        this.historyLimit = config.historyLimit || 1000;
        this.samplingInterval = config.samplingInterval || 1000; // 1 second
        this.retentionPeriod = config.retentionPeriod || 24 * 60 * 60 * 1000; // 24 hours
        this.baselineWindow = config.baselineWindow || 60 * 60 * 1000; // 1 hour
        this.alertThresholds = config.alertThresholds || this.getDefaultThresholds();
        
        this.baselines = {
            network: this.initializeBaseline(),
            disk: this.initializeBaseline(),
            memory: this.initializeBaseline(),
            cpu: this.initializeBaseline()
        };

        this.collectors = new Map();
        this.intervalId = null;
        this.isCollecting = false;
        this.lastCollection = null;
        this.lock = new AsyncLock();
        this.initializeCollectors();
    }

    initializeBaseline() {
        return {
            values: [],
            lastUpdate: Date.now(),
            mean: 0,
            standardDeviation: 0
        };
    }

    getDefaultThresholds() {
        return {
            memory: {
                usage: 0.9, // 90% memory usage
                growth: 0.1 // 10% growth rate
            },
            cpu: {
                usage: 0.8, // 80% CPU usage
                sustained: 0.7 // 70% sustained usage
            },
            disk: {
                usage: 0.95, // 95% disk usage
                iops: 5000 // IOPS threshold
            },
            network: {
                bandwidth: 0.8, // 80% bandwidth usage
                errorRate: 0.01 // 1% error rate
            }
        };
    }

    initializeCollectors() {
        // Memory metrics collector
        this.collectors.set('memory', async () => {
            const memoryInfo = process.memoryUsage();
            return {
                heapUsed: memoryInfo.heapUsed,
                heapTotal: memoryInfo.heapTotal,
                external: memoryInfo.external,
                rss: memoryInfo.rss,
                arrayBuffers: memoryInfo.arrayBuffers || 0,
                usage: memoryInfo.heapUsed / memoryInfo.heapTotal,
                timestamp: Date.now()
            };
        });

        // CPU metrics collector
        this.collectors.set('cpu', async () => {
            const startUsage = process.cpuUsage();
            await new Promise(resolve => setTimeout(resolve, 100));
            const endUsage = process.cpuUsage(startUsage);
            const totalUsage = endUsage.user + endUsage.system;

            return {
                user: endUsage.user,
                system: endUsage.system,
                total: totalUsage,
                percentage: totalUsage / 1000000, // Convert to percentage
                loadAverage: os.loadavg(),
                timestamp: Date.now()
            };
        });

        // Disk metrics collector
        this.collectors.set('disk', async () => {
            try {
                const stats = await this.collectDiskStats();
                return {
                    reads: stats.reads,
                    writes: stats.writes,
                    iops: stats.iops,
                    latency: stats.latency,
                    utilization: stats.utilization,
                    timestamp: Date.now()
                };
            } catch (error) {
                console.error('Error collecting disk metrics:', error);
                return null;
            }
        });

        // Network metrics collector
        this.collectors.set('network', async () => {
            try {
                const stats = await this.collectNetworkStats();
                return {
                    bytesIn: stats.bytesIn,
                    bytesOut: stats.bytesOut,
                    packetsIn: stats.packetsIn,
                    packetsOut: stats.packetsOut,
                    errors: stats.errors,
                    dropped: stats.dropped,
                    timestamp: Date.now()
                };
            } catch (error) {
                console.error('Error collecting network metrics:', error);
                return null;
            }
        });
    }

    async start() {
        if (this.intervalId) {
            return;
        }

        this.intervalId = setInterval(
            () => this.collect().catch(console.error),
            this.samplingInterval
        );
    }

    async stop() {
        if (this.intervalId) {
            clearInterval(this.intervalId);
            this.intervalId = null;
        }
    }

    async collect() {
        return await this.lock.acquire('collect', async () => {
            if (this.isCollecting) {
                return;
            }

            this.isCollecting = true;
            try {
                const metrics = await this.collectAll();
                this.updateHistory(metrics);
                this.updateBaselines(metrics);
                await this.checkThresholds(metrics);
                this.lastCollection = Date.now();
                return metrics;
            } finally {
                this.isCollecting = false;
            }
        });
    }

    async collectAll() {
        const metrics = {};
        for (const [name, collector] of this.collectors) {
            try {
                metrics[name] = await collector();
            } catch (error) {
                console.error(`Error collecting ${name} metrics:`, error);
                metrics[name] = null;
            }
        }
        return metrics;
    }

    updateHistory(metrics) {
        const timestamp = Date.now();
        this.metricsHistory.set(timestamp, metrics);

        // Clean up old metrics
        const cutoff = timestamp - this.retentionPeriod;
        for (const [ts] of this.metricsHistory) {
            if (ts < cutoff) {
                this.metricsHistory.delete(ts);
            } else {
                break; // Map is ordered by insertion time
            }
        }

        // Enforce history limit
        while (this.metricsHistory.size > this.historyLimit) {
            const oldestKey = this.metricsHistory.keys().next().value;
            this.metricsHistory.delete(oldestKey);
        }
    }

    updateBaselines(metrics) {
        const now = Date.now();
        const types = ['network', 'disk', 'memory', 'cpu'];

        for (const type of types) {
            if (!metrics[type]) continue;

            const baseline = this.baselines[type];
            baseline.values.push(metrics[type]);

            // Keep only values within baseline window
            const cutoff = now - this.baselineWindow;
            baseline.values = baseline.values.filter(v => v.timestamp >= cutoff);

            // Calculate new baseline statistics
            if (baseline.values.length > 0) {
                this.updateBaselineStats(baseline);
            }

            baseline.lastUpdate = now;
        }
    }

    updateBaselineStats(baseline) {
        // Calculate mean
        const sum = baseline.values.reduce((acc, val) => acc + (val.usage || 0), 0);
        baseline.mean = sum / baseline.values.length;

        // Calculate standard deviation
        const squaredDiffs = baseline.values.map(val => 
            Math.pow((val.usage || 0) - baseline.mean, 2)
        );
        const avgSquaredDiff = squaredDiffs.reduce((acc, val) => acc + val, 0) / squaredDiffs.length;
        baseline.standardDeviation = Math.sqrt(avgSquaredDiff);
    }

    async checkThresholds(metrics) {
        const alerts = [];

        // Check memory thresholds
        if (metrics.memory) {
            if (metrics.memory.usage > this.alertThresholds.memory.usage) {
                alerts.push({
                    type: 'memory',
                    severity: 'high',
                    message: 'Memory usage exceeds threshold',
                    value: metrics.memory.usage,
                    threshold: this.alertThresholds.memory.usage
                });
            }
        }

        // Check CPU thresholds
        if (metrics.cpu) {
            if (metrics.cpu.percentage > this.alertThresholds.cpu.usage) {
                alerts.push({
                    type: 'cpu',
                    severity: 'high',
                    message: 'CPU usage exceeds threshold',
                    value: metrics.cpu.percentage,
                    threshold: this.alertThresholds.cpu.usage
                });
            }
        }

        // Check disk thresholds
        if (metrics.disk) {
            if (metrics.disk.utilization > this.alertThresholds.disk.usage) {
                alerts.push({
                    type: 'disk',
                    severity: 'medium',
                    message: 'Disk utilization exceeds threshold',
                    value: metrics.disk.utilization,
                    threshold: this.alertThresholds.disk.usage
                });
            }
        }

        // Check network thresholds
        if (metrics.network) {
            const errorRate = metrics.network.errors / 
                (metrics.network.packetsIn + metrics.network.packetsOut);
            if (errorRate > this.alertThresholds.network.errorRate) {
                alerts.push({
                    type: 'network',
                    severity: 'medium',
                    message: 'Network error rate exceeds threshold',
                    value: errorRate,
                    threshold: this.alertThresholds.network.errorRate
                });
            }
        }

        return alerts;
    }

    async getMetrics(duration) {
        const now = Date.now();
        const startTime = duration ? now - duration : 0;

        return Array.from(this.metricsHistory.entries())
            .filter(([timestamp]) => timestamp >= startTime)
            .map(([timestamp, metrics]) => ({
                timestamp,
                metrics
            }));
    }

    async getBaselines() {
        return this.baselines;
    }

    async collectDiskStats() {
        // Implement actual disk stats collection based on your system
        // This is a placeholder implementation
        return {
            reads: Math.floor(Math.random() * 1000),
            writes: Math.floor(Math.random() * 1000),
            iops: Math.floor(Math.random() * 5000),
            latency: Math.random() * 10,
            utilization: Math.random()
        };
    }

    async collectNetworkStats() {
        // Implement actual network stats collection based on your system
        // This is a placeholder implementation
        return {
            bytesIn: Math.floor(Math.random() * 1000000),
            bytesOut: Math.floor(Math.random() * 1000000),
            packetsIn: Math.floor(Math.random() * 10000),
            packetsOut: Math.floor(Math.random() * 10000),
            errors: Math.floor(Math.random() * 10),
            dropped: Math.floor(Math.random() * 10)
        };
    }

    async destroy() {
        await this.stop();
        this.metricsHistory.clear();
        this.collectors.clear();
        this.baselines = null;
    }

    async getMemoryProfile(thought) {
        return {
            heapUsed: process.memoryUsage().heapUsed,
            heapTotal: process.memoryUsage().heapTotal,
            external: process.memoryUsage().external,
            timestamp: Date.now()
        };
    }

    async getResourceUsage(thought) {
        return {
            cpu: await this.getCPUUsage(),
            memory: await this.getMemoryProfile(thought),
            network: await this.getNetworkStats()
        };
    }

    getLatestMetrics() {
        return {
            ...this.metrics,
            timestamp: Date.now()
        };
    }

    async getCPUUsage() {
        return {
            percentage: process.cpuUsage().user / 1000000,
            timestamp: Date.now()
        };
    }

    async getNetworkStats() {
        return {
            bytesReceived: 0,
            bytesSent: 0,
            timestamp: Date.now()
        };
    }
}

export { MetricsCollector}
--- End of modules/debug/metrics/collector.js ---



=== Folder: modules/errors ===


--- File: modules/errors/index.js ---

// modules/errors/index.js
export { ThoughtError } from './thoughtError';
--- End of modules/errors/index.js ---


--- File: modules/errors/thoughtError.js ---

class ThoughtError extends Error {
    constructor(code, message, context = {}) {
        // Call parent constructor
        super(message);

        // Maintain proper stack trace
        if (Error.captureStackTrace) {
            Error.captureStackTrace(this, ThoughtError);
        }

        // Custom properties
        this.name = 'ThoughtError';
        this.code = code;
        this.timestamp = Date.now();
        this.context = this.sanitizeContext(context);
        this.severity = this.calculateSeverity(code);
        this.retryable = this.isRetryable(code);
    }

    // Predefined error codes and their properties
    static ErrorCodes = {
        // Memory-related errors
        MemoryLimitExceeded: {
            severity: 'high',
            retryable: false,
            category: 'resource'
        },
        MemoryAllocationFailed: {
            severity: 'high',
            retryable: true,
            category: 'resource'
        },

        // Processing errors
        InvalidInput: {
            severity: 'medium',
            retryable: false,
            category: 'validation'
        },
        ProcessingFailed: {
            severity: 'medium',
            retryable: true,
            category: 'operation'
        },
        
        // System errors
        SystemOverload: {
            severity: 'high',
            retryable: true,
            category: 'system'
        },
        InternalError: {
            severity: 'high',
            retryable: false,
            category: 'system'
        },

        // Data errors
        InvalidData: {
            severity: 'medium',
            retryable: false,
            category: 'data'
        },
        DataNotFound: {
            severity: 'low',
            retryable: false,
            category: 'data'
        },
        DataCorruption: {
            severity: 'high',
            retryable: false,
            category: 'data'
        },

        // Operation errors
        OperationTimeout: {
            severity: 'medium',
            retryable: true,
            category: 'operation'
        },
        OperationCancelled: {
            severity: 'low',
            retryable: true,
            category: 'operation'
        },
        ConcurrencyError: {
            severity: 'medium',
            retryable: true,
            category: 'operation'
        },

        // State errors
        InvalidState: {
            severity: 'medium',
            retryable: false,
            category: 'state'
        },
        StateTransitionFailed: {
            severity: 'medium',
            retryable: true,
            category: 'state'
        },

        // Configuration errors
        InvalidConfiguration: {
            severity: 'high',
            retryable: false,
            category: 'config'
        },
        ConfigurationMissing: {
            severity: 'high',
            retryable: false,
            category: 'config'
        }
    };

    calculateSeverity(code) {
        return ThoughtError.ErrorCodes[code]?.severity || 'medium';
    }

    isRetryable(code) {
        return ThoughtError.ErrorCodes[code]?.retryable ?? false;
    }

    getCategory() {
        return ThoughtError.ErrorCodes[this.code]?.category || 'unknown';
    }

    sanitizeContext(context) {
        const sanitized = {};
        
        for (const [key, value] of Object.entries(context)) {
            // Skip null or undefined values
            if (value == null) continue;

            // Handle different types of values
            if (typeof value === 'function') {
                sanitized[key] = '[Function]';
            } else if (value instanceof Error) {
                sanitized[key] = {
                    name: value.name,
                    message: value.message,
                    stack: value.stack
                };
            } else if (ArrayBuffer.isView(value)) {
                sanitized[key] = `[${value.constructor.name}]`;
            } else if (typeof value === 'object') {
                try {
                    // Attempt to safely stringify objects
                    sanitized[key] = JSON.parse(JSON.stringify(value));
                } catch {
                    sanitized[key] = '[Complex Object]';
                }
            } else {
                sanitized[key] = value;
            }
        }

        return sanitized;
    }

    toJSON() {
        return {
            name: this.name,
            code: this.code,
            message: this.message,
            timestamp: this.timestamp,
            severity: this.severity,
            category: this.getCategory(),
            retryable: this.retryable,
            context: this.context,
            stack: this.stack
        };
    }

    toString() {
        return `${this.name}[${this.code}]: ${this.message} (Severity: ${this.severity}, Category: ${this.getCategory()})`;
    }

    static isThoughtError(error) {
        return error instanceof ThoughtError;
    }

    static fromError(error, code = 'InternalError', additionalContext = {}) {
        const context = {
            originalError: {
                name: error.name,
                message: error.message,
                stack: error.stack
            },
            ...additionalContext
        };

        return new ThoughtError(code, error.message, context);
    }

    static wrapError(error, code = 'InternalError', message = null, additionalContext = {}) {
        if (ThoughtError.isThoughtError(error)) {
            // If it's already a ThoughtError, just add additional context
            error.context = {
                ...error.context,
                ...additionalContext
            };
            return error;
        }

        return ThoughtError.fromError(error, code, additionalContext);
    }

    getErrorDetails() {
        return {
            code: this.code,
            severity: this.severity,
            category: this.getCategory(),
            retryable: this.retryable,
            timestamp: this.timestamp,
            message: this.message,
            context: this.context
        };
    }

    isOfCategory(category) {
        return this.getCategory() === category;
    }

    shouldRetry(attemptsMade = 0, maxAttempts = 3) {
        if (!this.retryable) return false;
        if (attemptsMade >= maxAttempts) return false;
        
        // Additional retry logic based on error category
        switch (this.getCategory()) {
            case 'resource':
                return attemptsMade < 2; // Less retry attempts for resource errors
            case 'operation':
                return attemptsMade < maxAttempts;
            case 'system':
                return attemptsMade < maxAttempts - 1;
            default:
                return this.retryable && attemptsMade < maxAttempts;
        }
    }

    getRetryDelay(attemptsMade = 0) {
        // Exponential backoff with jitter
        const baseDelay = 1000; // 1 second
        const maxDelay = 30000; // 30 seconds
        
        let delay = baseDelay * Math.pow(2, attemptsMade);
        delay = Math.min(delay, maxDelay);
        
        // Add jitter (±25%)
        const jitter = delay * 0.25;
        delay += Math.random() * jitter * 2 - jitter;
        
        return Math.floor(delay);
    }
}
--- End of modules/errors/thoughtError.js ---



=== Folder: modules/memory ===


--- File: modules/memory/enhancedMemory.js ---

import { AsyncLock } from '../concurrency';
import { HybridStorage } from '../storage';
import { LRUCache } from './cache/lru';
import { EnhancedVectorStore } from '../vector';
import { ThoughtError } from '../errors/thoughtError';
import { CompressionUtil } from '../storage/utils/compression';

class EnhancedMemorySystem {
    #lock = new AsyncLock();


    constructor(config = {}) {
        this.modelCache = new Map(); 
        this.shortTermMemory = new HybridStorage({
            dbName: 'shortTermMemory',
            maxMemoryItems: config.shortTermLimit || 10000,
            storeName: 'shortTerm'
        });
        this.workingMemory = new LRUCache(config.workingMemorySize || 1000);
        this.vectorStore = new EnhancedVectorStore();
        this.memoryLimits = {
            shortTerm: config.shortTermLimit || 10000,
            working: config.workingMemoryLimit || 5000,
            vector: config.vectorLimit || 100000
        };
        this.cleanupInterval = config.cleanupInterval || 3600000; // 1 hour
        this.intervalId = null;
        this.setupAutoCleanup();

        this.aiConfig = {
            endpoint: config.aiEndpoint,
            apiKey: config.apiKey,
            compressionModel: config.compressionModel,
            retrievalModel: config.retrievalModel
        };

        this.lastCleanupAttempt = null;
        this.isCleaningUp = false;
        this.cleanupLock = new AsyncLock();
    }

    async releaseResources() {
        try {
            // Release any temporary compression resources
            await this.cleanupTemporaryResources();

            // Clear any cached data
            this.modelCache?.clear();

            // Additional resource cleanup as needed
            await this.vectorStore?.releaseResources();
            
            // Release shortTermMemory resources
            await this.shortTermMemory.destroy();
        } catch (error) {
            console.error('Error releasing resources:', error);
            throw new Error('Failed to release resources: ' + error.message);
        }
    }

    setupAutoCleanup() {
        // Clear existing interval if any
        if (this.intervalId) {
            clearInterval(this.intervalId);
        }

        // Set up new cleanup interval with race condition protection
        this.intervalId = setInterval(async () => {
            try {
                await this.cleanupLock.acquire('cleanup', async () => {
                    if (this.isCleaningUp) {
                        return; // Skip if cleanup is already in progress
                    }
                    this.isCleaningUp = true;
                    this.lastCleanupAttempt = Date.now();
                    await this.performCleanup();
                    this.isCleaningUp = false;
                });
            } catch (error) {
                console.error('Error during auto cleanup:', error);
                this.isCleaningUp = false; // Reset flag on error
            }
        }, this.cleanupInterval);
    }

    async destroy() {
        try {
            // Clear cleanup interval
            if (this.intervalId) {
                clearInterval(this.intervalId);
                this.intervalId = null;
            }

            // Clean up vector store
            await this.vectorStore.destroy();

            // Clear working memory
            await this.workingMemory.clear();

            // Clear short-term memory
            await this.shortTermMemory.destroy();

            // Release any held resources
            await this.releaseResources();
        } catch (error) {
            console.error('Error during memory system destruction:', error);
            throw new Error('Failed to properly destroy memory system: ' + error.message);
        }
    }

    async performCleanup() {
        const now = Date.now();

        try {
            // Cleanup short-term memory
            await this.shortTermMemory.vacuum();

            // Enforce working memory size limit
            while (this.workingMemory.size() > this.memoryLimits.working) {
                await this.workingMemory.evictOldest();
            }

            // Trigger vacuum on vector store
            await this.vectorStore.vacuum().catch(error => {
                console.error('Error during vector store vacuum:', error);
                throw error;
            });

            // Optimize storage if needed
            await this.shortTermMemory.optimize();

        } catch (error) {
            console.error('Error during cleanup:', error);
            throw new Error('Cleanup failed: ' + error.message);
        }
    }

    async safeDelete(key, memoryType) {
        try {
            switch (memoryType) {
                case 'shortTerm':
                    await this.shortTermMemory.delete(key);
                    break;
                case 'working':
                    await this.workingMemory.delete(key);
                    break;
                case 'vector':
                    await this.vectorStore.delete(key);
                    break;
                default:
                    throw new Error(`Unknown memory type: ${memoryType}`);
            }
        } catch (error) {
            console.error(`Error deleting key ${key} from ${memoryType}:`, error);
            throw error;
        }
    }

    async store(key, data, type = 'shortTerm', metadata = {}) {
        return await this.#lock.acquire('store', async () => {
            const estimatedSize = this.calculateSize(data);
            const safetyBuffer = 1.1;

            if (!this.checkMemoryLimits(type, estimatedSize * safetyBuffer)) {
                throw new ThoughtError(
                    'MemoryLimitExceeded',
                    `Memory limit exceeded for ${type}`,
                    { size: estimatedSize, limit: this.memoryLimits[type] }
                );
            }

            const compressedData = await this.compress(data);
            const enhancedMetadata = {
                ...metadata,
                timestamp: Date.now(),
                type,
                originalSize: estimatedSize,
                compressedSize: this.calculateSize(compressedData)
            };

            switch (type) {
                case 'shortTerm':
                    await this.shortTermMemory.set(key, {
                        data: compressedData,
                        metadata: enhancedMetadata
                    }, {
                        priority: metadata.priority,
                        compression: true,
                        tags: metadata.tags
                    });
                    break;
                default:
                    throw new ThoughtError('InvalidMemoryType', `Unknown memory type: ${type}`);
            }

            return enhancedMetadata;
        });
    }

    async retrieve(key, type = 'shortTerm') {
        return await this.#lock.acquire('retrieve', async () => {
            let result;
            switch (type) {
                case 'shortTerm':
                    result = await this.shortTermMemory.get(key);
                    if (result) {
                        return await this.decompress(result.data, result.metadata);
                    }
                    break;
                default:
                    throw new ThoughtError('InvalidMemoryType', `Unknown memory type: ${type}`);
            }
            return null;
        });
    }

    async compress(data) {
        if (!this.aiConfig.compressionModel) {
            return data;
        }

        try {
            const response = await fetch(`${this.aiConfig.endpoint}/compress`, {
                method: 'POST',
                headers: {
                    'Authorization': `Bearer ${this.aiConfig.apiKey}`,
                    'Content-Type': 'application/json'
                },
                body: JSON.stringify({
                    data: data,
                    model: this.aiConfig.compressionModel
                })
            });

            if (!response.ok) {
                throw new Error(`Compression API request failed with status ${response.status}`);
            }

            return await response.json();
        } catch (error) {
            throw new Error('Compression failed', { cause: error });
        }
    }

    async decompress(data, metadata) {
        if (!this.aiConfig.compressionModel) {
            return data;
        }

        try {
            const response = await fetch(`${this.aiConfig.endpoint}/decompress`, {
                method: 'POST',
                headers: {
                    'Authorization': `Bearer ${this.aiConfig.apiKey}`,
                    'Content-Type': 'application/json'
                },
                body: JSON.stringify({
                    data: data,
                    metadata: metadata,
                    model: this.aiConfig.compressionModel
                })
            });

            if (!response.ok) {
                throw new Error(`Decompression API request failed with status ${response.status}`);
            }

            const decompressed = await response.json();

            // Validate decompressed data
            if (!decompressed || typeof decompressed !== 'object') {
                throw new Error('Invalid decompressed data format');
            }

            return decompressed;
        } catch (error) {
            throw new Error('Decompression failed', { cause: error });
        }
    }

    calculateSize(data) {
        return JSON.stringify(data).length;
    }

    checkMemoryLimits(type, size) {
        return size <= this.memoryLimits[type];
    }

    async getUsageMetrics() {
        const shortTermMetrics = await this.shortTermMemory.getMetrics();
        
        return {
            shortTerm: {
                size: shortTermMetrics.totalItems,
                capacityUsed: shortTermMetrics.totalSize / this.memoryLimits.shortTerm,
                hitRate: shortTermMetrics.hitRate,
                averageAccessTime: shortTermMetrics.averageAccessTime
            },
            working: {
                size: this.workingMemory.size(),
                capacityUsed: this.workingMemory.size() / this.memoryLimits.working
            },
            vector: await this.vectorStore.getUsageMetrics(),
            lastCleanup: this.lastCleanupAttempt,
            totalEntries: shortTermMetrics.totalItems +
                this.workingMemory.size() +
                await this.vectorStore.getEntryCount()
        };
    }

    async cleanupTemporaryResources() {
        await CompressionUtil.cleanupTemporaryResources();
    }
}
--- End of modules/memory/enhancedMemory.js ---


--- File: modules/memory/index.js ---

// modules/memory/index.js
export { EnhancedMemorySystem } from './enhancedMemory';
export { LRUCache } from './cache/lru';
--- End of modules/memory/index.js ---



=== Folder: modules/memory/cache ===


--- File: modules/memory/cache/lru.js ---

import { AsyncLock } from '../../concurrency';

class LRUCache {
    constructor(capacity) {
        if (!Number.isInteger(capacity) || capacity <= 0) {
            throw new Error('Cache capacity must be a positive integer');
        }
        this.capacity = capacity;
        this.cache = new Map();
        this.head = { key: null, value: null, prev: null, next: null };
        this.tail = { key: null, value: null, prev: this.head, next: null };
        this.head.next = this.tail;
        this.lock = new AsyncLock();
        this.stats = {
            hits: 0,
            misses: 0,
            evictions: 0
        };
    }

    async get(key) {
        return await this.lock.acquire('get', () => {
            const node = this.cache.get(key);
            if (node) {
                this.stats.hits++;
                this.moveToFront(node);
                return node.value;
            }
            this.stats.misses++;
            return undefined;
        });
    }

    async set(key, value) {
        return await this.lock.acquire('set', () => {
            const existingNode = this.cache.get(key);

            if (existingNode) {
                // Update existing node
                existingNode.value = value;
                this.moveToFront(existingNode);
                return;
            }

            // Create new node
            const newNode = {
                key,
                value,
                prev: this.head,
                next: this.head.next
            };

            // Add to doubly-linked list
            this.head.next.prev = newNode;
            this.head.next = newNode;

            // Add to cache
            this.cache.set(key, newNode);

            // Check capacity
            if (this.cache.size > this.capacity) {
                this.evictOldest();
            }
        });
    }

    async delete(key) {
        return await this.lock.acquire('delete', () => {
            const node = this.cache.get(key);
            if (node) {
                this.removeNode(node);
                this.cache.delete(key);
                return true;
            }
            return false;
        });
    }

    async clear() {
        return await this.lock.acquire('clear', () => {
            this.cache.clear();
            this.head.next = this.tail;
            this.tail.prev = this.head;
            this.resetStats();
        });
    }

    async evictOldest() {
        return await this.lock.acquire('evict', () => {
            if (this.tail.prev === this.head) {
                return false; // Cache is empty
            }

            const oldestNode = this.tail.prev;
            this.removeNode(oldestNode);
            this.cache.delete(oldestNode.key);
            this.stats.evictions++;
            return true;
        });
    }

    moveToFront(node) {
        // Remove from current position
        this.removeNode(node);

        // Add to front
        node.prev = this.head;
        node.next = this.head.next;
        this.head.next.prev = node;
        this.head.next = node;
    }

    removeNode(node) {
        node.prev.next = node.next;
        node.next.prev = node.prev;
    }

    size() {
        return this.cache.size;
    }

    has(key) {
        return this.cache.has(key);
    }

    *entries() {
        let node = this.head.next;
        while (node !== this.tail) {
            yield [node.key, node.value];
            node = node.next;
        }
    }

    *keys() {
        let node = this.head.next;
        while (node !== this.tail) {
            yield node.key;
            node = node.next;
        }
    }

    *values() {
        let node = this.head.next;
        while (node !== this.tail) {
            yield node.value;
            node = node.next;
        }
    }

    async getStats() {
        return await this.lock.acquire('stats', () => ({
            ...this.stats,
            size: this.cache.size,
            capacity: this.capacity,
            hitRate: this.calculateHitRate(),
            evictionRate: this.calculateEvictionRate()
        }));
    }

    calculateHitRate() {
        const total = this.stats.hits + this.stats.misses;
        return total === 0 ? 0 : this.stats.hits / total;
    }

    calculateEvictionRate() {
        const total = this.stats.hits + this.stats.misses;
        return total === 0 ? 0 : this.stats.evictions / total;
    }

    resetStats() {
        this.stats = {
            hits: 0,
            misses: 0,
            evictions: 0
        };
    }

    async optimize() {
        return await this.lock.acquire('optimize', () => {
            // Analyze access patterns and adjust capacity if needed
            const hitRate = this.calculateHitRate();
            const evictionRate = this.calculateEvictionRate();

            // If hit rate is high and eviction rate is low, we might be over-provisioned
            if (hitRate > 0.9 && evictionRate < 0.1) {
                this.capacity = Math.max(Math.floor(this.capacity * 0.8), 1);
                while (this.cache.size > this.capacity) {
                    this.evictOldest();
                }
            }

            // If hit rate is low and eviction rate is high, we might need more capacity
            if (hitRate < 0.5 && evictionRate > 0.5) {
                this.capacity = Math.min(Math.floor(this.capacity * 1.5), Number.MAX_SAFE_INTEGER);
            }

            return {
                newCapacity: this.capacity,
                hitRate,
                evictionRate
            };
        });
    }

    async peek(key) {
        return await this.lock.acquire('peek', () => {
            const node = this.cache.get(key);
            return node ? node.value : undefined;
        });
    }

    async peekOldest() {
        return await this.lock.acquire('peekOldest', () => {
            if (this.tail.prev === this.head) {
                return undefined;
            }
            return {
                key: this.tail.prev.key,
                value: this.tail.prev.value
            };
        });
    }

    async peekNewest() {
        return await this.lock.acquire('peekNewest', () => {
            if (this.head.next === this.tail) {
                return undefined;
            }
            return {
                key: this.head.next.key,
                value: this.head.next.value
            };
        });
    }

    [Symbol.iterator]() {
        return this.entries();
    }

    async destroy() {
        return await this.lock.acquire('destroy', () => {
            this.clear();
            this.head = null;
            this.tail = null;
            this.cache = null;
            this.stats = null;
        });
    }
}

module.exports = { LRUCache };
--- End of modules/memory/cache/lru.js ---



=== Folder: modules/storage ===


--- File: modules/storage/hybridStorage.js ---

import { AsyncLock } from '../concurrency';
import { CompressionUtil } from './utils/compression';
import { StorageMetrics } from './utils/metrics';
import { ThoughtError } from '../errors/thoughtError';

class HybridStorage {
    constructor(options = {}) {
        this.memoryStore = new Map();
        this.dbName = options.dbName || 'hybridStorage';
        this.storeName = options.storeName || 'mainStore';
        this.maxMemoryItems = options.maxMemoryItems || 10000;
        this.maxRetries = options.maxRetries || 3;
        this.retryDelay = options.retryDelay || 1000;
        this.db = null;
        this.ready = this.initializeDB();
        this.lock = new AsyncLock();

        // Performance metrics
        this.metrics = {
            hits: 0,
            misses: 0,
            writes: 0,
            deletes: 0,
            errors: 0,
            accessTimes: [],
            maxAccessTimes: 1000
        };

        // Track accesses to simulate frequently accessed keys
        this.accessCounter = new Map();
    }

    async initializeDB() {
        if (this.initialized) return;
        if (this.initializing) return this.initializing;

        this.initializing = new Promise((resolve, reject) => {
            try {
                const request = indexedDB.open(this.dbName, 1);

                request.onerror = () => {
                    reject(new Error(`Failed to open IndexedDB: ${request.error}`));
                };

                request.onupgradeneeded = (event) => {
                    const db = event.target.result;
                    if (!db.objectStoreNames.contains(this.storeName)) {
                        const store = db.createObjectStore(this.storeName, { keyPath: 'key' });
                        store.createIndex('timestamp', 'timestamp', { unique: false });
                        store.createIndex('priority', 'priority', { unique: false });
                        store.createIndex('tags', 'tags', { unique: false, multiEntry: true });
                        store.createIndex('size', 'size', { unique: false });
                    }
                };

                request.onsuccess = (event) => {
                    this.db = event.target.result;

                    this.db.onclose = () => {
                        this.initialized = false;
                        this.db = null;
                    };

                    this.db.onerror = (event) => {
                        console.error('IndexedDB error:', event.target.error);
                        this.metrics.errors++;
                    };

                    this.initialized = true;
                    resolve();
                };
            } catch (error) {
                reject(error);
            }
        });

        try {
            await this.initializing;
            this.initializing = null;
            return;
        } catch (error) {
            this.initializing = null;
            throw error;
        }
    }

    async ensureDBConnection() {
        if (!this.initialized) {
            await this.initializeDB();
        }
        if (!this.db) {
            throw new Error('Database connection not established');
        }
    }

    async set(key, value, options = {}) {
        await this.ensureDBConnection();

        return this.lock.acquire(`write_${key}`, async () => {
            const startTime = performance.now();
            try {
                const now = Date.now();
                const item = {
                    key,
                    value,
                    timestamp: now,
                    lastAccess: now,  // Track last access separately
                    priority: options.priority || 0,
                    tags: options.tags || [],
                    expiry: options.expiry,
                    size: this.calculateItemSize(value)
                };

                if (options.compression) {
                    item.value = await CompressionUtil.compress(value);
                    item.compressed = true;
                }

                // Store in memory if conditions met
                if (this.shouldStoreInMemory(item)) {
                    this.memoryStore.set(key, item);
                    this.enforceMemoryLimit();
                }

                // Store in IndexedDB
                await this.setInDB(item);
                this.metrics.writes++;
                this.recordAccessTime(performance.now() - startTime);

                return true;
            } catch (error) {
                this.metrics.errors++;
                throw error;
            }
        });
    }

    async get(key) {
        await this.ensureDBConnection();

        return this.lock.acquire(`read_${key}`, async () => {
            const startTime = performance.now();
            try {
                // Check memory first
                if (this.memoryStore.has(key)) {
                    const item = this.memoryStore.get(key);
                    if (this.isExpired(item)) {
                        this.memoryStore.delete(key);
                        await this.delete(key);
                        return null;
                    }

                    this.updateLastAccess(item);
                    this.incrementAccessCount(key);
                    this.metrics.hits++;
                    this.recordAccessTime(performance.now() - startTime);
                    return item.compressed ?
                        await CompressionUtil.decompress(item.value) :
                        item.value;
                }

                // Check IndexedDB
                const item = await this.getFromDB(key);
                if (!item) {
                    this.metrics.misses++;
                    return null;
                }

                if (this.isExpired(item)) {
                    await this.delete(key);
                    return null;
                }

                // Update last access time before caching
                this.updateLastAccess(item);

                // Cache in memory for future access
                if (this.shouldStoreInMemory(item)) {
                    this.memoryStore.set(key, item);
                    this.enforceMemoryLimit();
                }

                this.incrementAccessCount(key);
                this.metrics.hits++;
                this.recordAccessTime(performance.now() - startTime);
                return item.compressed ?
                    await CompressionUtil.decompress(item.value) :
                    item.value;
            } catch (error) {
                this.metrics.errors++;
                throw error;
            }
        });
    }

    async delete(key) {
        await this.ensureDBConnection();

        return this.lock.acquire(`delete_${key}`, async () => {
            const startTime = performance.now();
            try {
                // Remove from memory
                this.memoryStore.delete(key);

                // Remove from IndexedDB
                const result = await this.deleteFromDB(key);
                if (result) {
                    this.metrics.deletes++;
                }

                this.recordAccessTime(performance.now() - startTime);
                return result;
            } catch (error) {
                this.metrics.errors++;
                throw error;
            }
        });
    }

    async has(key) {
        await this.ensureDBConnection();

        const startTime = performance.now();
        try {
            // Check memory first
            if (this.memoryStore.has(key)) {
                const item = this.memoryStore.get(key);
                if (!this.isExpired(item)) {
                    this.recordAccessTime(performance.now() - startTime);
                    return true;
                }
                this.memoryStore.delete(key);
            }

            // Check IndexedDB
            const exists = await this.existsInDB(key);
            this.recordAccessTime(performance.now() - startTime);
            return exists;
        } catch (error) {
            this.metrics.errors++;
            throw error;
        }
    }

    async clear() {
        await this.ensureDBConnection();

        return this.lock.acquire('clear', async () => {
            try {
                this.memoryStore.clear();
                await this.clearDB();
                return true;
            } catch (error) {
                this.metrics.errors++;
                throw error;
            }
        });
    }

    async query(filter) {
        await this.ensureDBConnection();

        return this.lock.acquire('query', async () => {
            const startTime = performance.now();
            try {
                const results = await this.queryDB(filter);
                this.recordAccessTime(performance.now() - startTime);
                return results;
            } catch (error) {
                this.metrics.errors++;
                throw error;
            }
        });
    }

    async optimize() {
        await this.ensureDBConnection();

        return this.lock.acquire('optimize', async () => {
            try {
                await this.vacuum();

                const accessPatterns = this.analyzeAccessPatterns();

                this.optimizeMemoryStoreSize(accessPatterns);

                const fragmentation = await this.estimateFragmentation();
                if (fragmentation > 0.3) {
                    await this.compactDB();
                }

                return true;
            } catch (error) {
                this.metrics.errors++;
                throw error;
            }
        });
    }

    async vacuum() {
        await this.ensureDBConnection();

        return this.lock.acquire('vacuum', async () => {
            try {
                const expiredKeys = new Set();

                // Check memory store
                for (const [key, item] of this.memoryStore) {
                    if (this.isExpired(item)) {
                        this.memoryStore.delete(key);
                        expiredKeys.add(key);
                    }
                }

                const transaction = this.db.transaction([this.storeName], 'readwrite');
                const store = transaction.objectStore(this.storeName);
                const index = store.index('timestamp');

                return new Promise((resolve, reject) => {
                    const req = index.openCursor();
                    req.onerror = () => reject(new Error(`Vacuum failed: ${transaction.error}`));
                    req.onsuccess = (event) => {
                        const cursor = event.target.result;
                        if (cursor) {
                            const item = cursor.value;
                            if (this.isExpired(item)) {
                                cursor.delete();
                                if (!expiredKeys.has(item.key)) {
                                    expiredKeys.add(item.key);
                                }
                            }
                            cursor.continue();
                        } else {
                            resolve(expiredKeys.size);
                        }
                    };
                });
            } catch (error) {
                this.metrics.errors++;
                throw error;
            }
        });
    }

    async setInDB(item) {
        await this.ensureDBConnection();

        return new Promise((resolve, reject) => {
            try {
                if (!this.db) throw new Error('Database not available');
                const transaction = this.db.transaction([this.storeName], 'readwrite');
                const store = transaction.objectStore(this.storeName);

                const request = store.put(item);

                request.onerror = () => {
                    reject(new Error(`Failed to store item: ${request.error}`));
                };

                request.onsuccess = () => {
                    resolve(true);
                };

                transaction.onerror = () => {
                    reject(new Error(`Transaction failed: ${transaction.error}`));
                };
            } catch (error) {
                reject(new Error(`Failed to create transaction: ${error.message}`));
            }
        });
    }

    async getFromDB(key) {
        await this.ensureDBConnection();

        return new Promise((resolve, reject) => {
            try {
                if (!this.db) throw new Error('Database not available');
                const transaction = this.db.transaction([this.storeName], 'readonly');
                const store = transaction.objectStore(this.storeName);
                const request = store.get(key);

                request.onerror = () => {
                    reject(new Error(`Failed to retrieve item: ${request.error}`));
                };

                request.onsuccess = () => {
                    const item = request.result;
                    if (item) {
                        // If item was stored earlier, lastAccess might not exist
                        if (!item.lastAccess) {
                            item.lastAccess = item.timestamp;
                        }
                    }
                    resolve(item || null);
                };
            } catch (error) {
                reject(new Error(`Failed to create transaction: ${error.message}`));
            }
        });
    }

    async deleteFromDB(key) {
        return new Promise((resolve, reject) => {
            if (!this.db) return reject(new Error('Database not available'));
            const transaction = this.db.transaction([this.storeName], 'readwrite');
            const store = transaction.objectStore(this.storeName);
            const request = store.delete(key);

            request.onerror = () => reject(new Error(`Failed to delete item: ${request.error}`));
            request.onsuccess = () => resolve(true);
        });
    }

    async existsInDB(key) {
        return new Promise((resolve, reject) => {
            if (!this.db) return reject(new Error('Database not available'));
            const transaction = this.db.transaction([this.storeName], 'readonly');
            const store = transaction.objectStore(this.storeName);
            const request = store.count(key);

            request.onerror = () => reject(new Error(`Failed to check item existence: ${request.error}`));
            request.onsuccess = () => resolve(request.result > 0);
        });
    }

    async clearDB() {
        return new Promise((resolve, reject) => {
            if (!this.db) return reject(new Error('Database not available'));
            const transaction = this.db.transaction([this.storeName], 'readwrite');
            const store = transaction.objectStore(this.storeName);
            const request = store.clear();

            request.onerror = () => reject(new Error(`Failed to clear store: ${request.error}`));
            request.onsuccess = () => resolve(true);
        });
    }

    async queryDB(filter) {
        return new Promise((resolve, reject) => {
            if (!this.db) return reject(new Error('Database not available'));
            const transaction = this.db.transaction([this.storeName], 'readonly');
            transaction.startTime = Date.now();
            const store = transaction.objectStore(this.storeName);
            const results = new Map();
            let count = 0;

            let request;
            if (filter.tags && filter.tags.length > 0) {
                const tagIndex = store.index('tags');
                request = tagIndex.openCursor();
            } else if (filter.minPriority !== undefined) {
                const priorityIndex = store.index('priority');
                request = priorityIndex.openCursor(IDBKeyRange.lowerBound(filter.minPriority));
            } else {
                request = store.openCursor();
            }

            request.onerror = () => reject(new Error(`Query failed: ${request.error}`));
            request.onsuccess = (event) => {
                const cursor = event.target.result;
                if (cursor) {
                    const item = cursor.value;
                    if (this.matchesFilter(item, filter) && !this.isExpired(item)) {
                        results.set(item.key, item.value);
                        count++;
                    }
                    cursor.continue();
                } else {
                    resolve({
                        items: results,
                        totalCount: count,
                        metrics: {
                            executionTime: Date.now() - transaction.startTime,
                            itemsScanned: count,
                            resultSize: results.size
                        }
                    });
                }
            };
        });
    }

    async compactDB() {
        await this.ensureDBConnection();

        return this.lock.acquire('compact', async () => {
            try {
                const allData = await this.getAllItems();

                if (this.db && typeof this.db.close === 'function' && !this.db.closed) {
                    this.db.close();
                }

                await new Promise((resolve, reject) => {
                    const deleteRequest = indexedDB.deleteDatabase(this.dbName);
                    deleteRequest.onerror = () => reject(new Error('Failed to delete old database'));
                    deleteRequest.onsuccess = () => resolve();
                });

                await this.initializeDB();

                for (const item of allData) {
                    await this.setInDB(item);
                }

                return true;
            } catch (error) {
                this.metrics.errors++;
                throw new Error(`Database compaction failed: ${error.message}`);
            }
        });
    }

    async getAllItems() {
        return new Promise((resolve, reject) => {
            if (!this.db) return reject(new Error('Database not available'));
            const items = [];
            const transaction = this.db.transaction([this.storeName], 'readonly');
            const store = transaction.objectStore(this.storeName);
            const request = store.openCursor();

            request.onerror = () => reject(new Error('Failed to get all items'));
            request.onsuccess = (event) => {
                const cursor = event.target.result;
                if (cursor) {
                    if (!this.isExpired(cursor.value)) {
                        const item = cursor.value;
                        if (!item.lastAccess) {
                            item.lastAccess = item.timestamp;
                        }
                        items.push(item);
                    }
                    cursor.continue();
                } else {
                    resolve(items);
                }
            };
        });
    }

    isExpired(item) {
        return item.expiry && Date.now() > item.timestamp + item.expiry;
    }

    shouldStoreInMemory(item) {
        if (!item) return false;

        if (item.priority > 0) return true;
        if (item.size > 100000) return false;

        const isRecent = Date.now() - (item.lastAccess || item.timestamp) < 300000; // 5 min
        const accessCount = this.getAccessCount(item.key);
        const isFrequent = accessCount > 5;

        return isRecent || isFrequent;
    }

    getAccessCount(key) {
        return this.accessCounter.get(key) || 0;
    }

    incrementAccessCount(key) {
        const current = this.accessCounter.get(key) || 0;
        this.accessCounter.set(key, current + 1);
    }

    updateLastAccess(item) {
        item.lastAccess = Date.now();
    }

    enforceMemoryLimit() {
        if (this.memoryStore.size <= this.maxMemoryItems) return;

        const items = Array.from(this.memoryStore.entries())
            .map(([key, item]) => ({
                key,
                priority: item.priority,
                lastAccess: item.lastAccess || item.timestamp
            }))
            .sort((a, b) => {
                // Sort by priority descending (higher priority first),
                // and for ties, by recency descending (more recent first).
                // The tests might expect the opposite, but let's try to keep frequently accessed items in.
                // Given the test's failure, let's invert logic: higher priority/later lastAccess = keep in memory,
                // so we sort by priority ascending then by lastAccess ascending to evict the oldest/lowest first,
                // Actually let's do the opposite: we want to keep highest priority and most recent usage.
                // We'll sort by priority ascending and lastAccess ascending means we remove low priority and older first,
                // that should be correct.

                if (a.priority !== b.priority) {
                    return a.priority - b.priority;
                }
                return a.lastAccess - b.lastAccess;
            });

        // Evict from the front of sorted array (lowest priority, oldest lastAccess)
        while (this.memoryStore.size > this.maxMemoryItems) {
            const item = items.shift();
            if (item) {
                this.memoryStore.delete(item.key);
            }
        }
    }

    calculateItemSize(value) {
        if (typeof value === 'string') {
            return value.length * 2;
        }
        return JSON.stringify(value).length * 2;
    }

    recordAccessTime(duration) {
        this.metrics.accessTimes.push(duration);
        if (this.metrics.accessTimes.length > this.metrics.maxAccessTimes) {
            this.metrics.accessTimes.shift();
        }
    }

    analyzeAccessPatterns() {
        const analysis = {
            averageAccessTime: 0,
            hitRate: 0,
            fragmentation: 0
        };

        if (this.metrics.accessTimes.length > 0) {
            analysis.averageAccessTime = this.metrics.accessTimes.reduce((a, b) => a + b, 0)
                / this.metrics.accessTimes.length;
        }

        const totalAccesses = this.metrics.hits + this.metrics.misses;
        if (totalAccesses > 0) {
            analysis.hitRate = this.metrics.hits / totalAccesses;
        }

        // fragmentation will be calculated asynchronously if needed
        analysis.fragmentation = 0;

        return analysis;
    }

    async estimateFragmentation() {
        try {
            const stats = await this.getStorageStats();
            if (stats.totalSize === 0) {
                // If no items, no fragmentation
                return 0;
            }
            return 1 - (stats.usedSize / stats.totalSize);
        } catch {
            return 0;
        }
    }

    async getStorageStats() {
        await this.ensureDBConnection();
        return new Promise((resolve, reject) => {
            if (!this.db) return reject(new Error('Database not available'));

            const transaction = this.db.transaction([this.storeName], 'readonly');
            const store = transaction.objectStore(this.storeName);
            const countRequest = store.count();
            let usedSize = 0;

            countRequest.onerror = () => reject(new Error('Failed to get storage stats'));
            countRequest.onsuccess = () => {
                const totalCount = countRequest.result;
                if (totalCount === 0) {
                    // No items, return default values
                    return resolve({ totalSize: 1, usedSize: 0 });
                }
                const cursorReq = store.openCursor();
                cursorReq.onerror = () => reject(new Error('Failed to get storage stats'));
                cursorReq.onsuccess = (event) => {
                    const cursor = event.target.result;
                    if (cursor) {
                        const item = cursor.value;
                        usedSize += this.calculateItemSize(item.value);
                        cursor.continue();
                    } else {
                        resolve({
                            totalSize: totalCount * 1000, // Rough estimate
                            usedSize: usedSize
                        });
                    }
                };
            };
        });
    }

    optimizeMemoryStoreSize() {
        // Note: Now we get actual fragmentation later, but we don't do it here.
        const analysis = this.analyzeAccessPatterns();

        if (analysis.hitRate > 0.8 && this.metrics.errors < 100) {
            this.maxMemoryItems = Math.min(
                Math.floor(this.maxMemoryItems * 1.2),
                100000
            );
        } else if (analysis.hitRate < 0.4 || this.metrics.errors > 1000) {
            this.maxMemoryItems = Math.max(
                Math.floor(this.maxMemoryItems * 0.8),
                1000
            );
        }
    }

    async destroy() {
        await this.ensureDBConnection();

        try {
            this.memoryStore.clear();

            if (this.db && typeof this.db.close === 'function' && !this.db.closed) {
                this.db.close();
            }

            await new Promise((resolve, reject) => {
                const request = indexedDB.deleteDatabase(this.dbName);
                request.onerror = () => reject(new Error('Failed to delete database'));
                request.onsuccess = () => resolve();
            });

            return true;
        } catch (error) {
            throw new Error(`Failed to destroy storage: ${error.message}`);
        }
    }

    matchesFilter(item, filter) {
        if (!item) return false;

        if (filter.tags && !filter.tags.every(tag => item.tags.includes(tag))) {
            return false;
        }

        if (filter.createdAfter && item.timestamp < filter.createdAfter.getTime()) {
            return false;
        }

        if (filter.createdBefore && item.timestamp > filter.createdBefore.getTime()) {
            return false;
        }

        if (filter.minPriority !== undefined && item.priority < filter.minPriority) {
            return false;
        }

        if (filter.maxSize !== undefined && item.size > filter.maxSize) {
            return false;
        }

        return true;
    }

    async getUsageMetrics() {
        const totalAccesses = this.metrics.hits + this.metrics.misses;
        return {
            hitRate: totalAccesses === 0 ? 0 : this.metrics.hits / totalAccesses
        };
    }
}

export { HybridStorage }

--- End of modules/storage/hybridStorage.js ---


--- File: modules/storage/index.js ---

// modules/storage/index.js
export { HybridStorage } from './hybridStorage';
export { CompressionUtil } from './utils/compression';
export { StorageMetrics } from './utils/metrics';
--- End of modules/storage/index.js ---



=== Folder: modules/storage/utils ===


--- File: modules/storage/utils/compression.js ---

class CompressionUtil {
    // LZ77-based compression algorithm implementation
    static async compress(data) {
        try {
            // Convert data to string if needed
            const stringData = typeof data === 'string' ? data : JSON.stringify(data);

            // Convert string to Uint8Array for processing
            const textEncoder = new TextEncoder();
            const input = textEncoder.encode(stringData);

            const compressed = [];
            let pos = 0;

            while (pos < input.length) {
                const match = this.findLongestMatch(input, pos);

                if (match.length > 3) { // Only use matches longer than 3 bytes
                    // Store as (distance, length) pair
                    compressed.push([match.distance, match.length]);
                    pos += match.length;
                } else {
                    // Store literal byte
                    compressed.push(input[pos]);
                    pos++;
                }
            }

            // Convert compressed data to Uint8Array
            return this.encodeCompressed(compressed);
        } catch (error) {
            // Wrap any error in a consistent format
            throw new Error(`Compression failed: ${error.message}`);
        }
    }

    static async decompress(compressedData) {
        try {
            // Validate input
            if (!(compressedData instanceof Uint8Array)) {
                throw new Error('Invalid compressed data format');
            }

            const decoded = this.decodeCompressed(compressedData);
            const decompressed = [];
            const maxSize = 1024 * 1024 * 1024; // 1GB safety limit

            for (const token of decoded) {
                if (decompressed.length > maxSize) {
                    throw new Error('Decompressed data exceeds size limit');
                }

                if (Array.isArray(token)) {
                    const [distance, length] = token;

                    // Validate distance and length
                    if (distance <= 0 || distance > decompressed.length) {
                        throw new Error('Invalid back-reference distance');
                    }
                    if (length <= 0 || length > 1024 * 64) { // 64KB max match length
                        throw new Error('Invalid match length');
                    }

                    const start = decompressed.length - distance;
                    // Safe copy with bounds checking
                    for (let i = 0; i < length; i++) {
                        if (start + i >= decompressed.length) {
                            throw new Error('Invalid back-reference');
                        }
                        decompressed.push(decompressed[start + i]);
                    }
                } else {
                    // Validate literal byte
                    if (!Number.isInteger(token) || token < 0 || token > 255) {
                        throw new Error('Invalid literal byte');
                    }
                    decompressed.push(token);
                }
            }

            try {
                const decodedString = new TextDecoder().decode(new Uint8Array(decompressed));
                
                // If the original input was JSON, parse it back to the original format
                if (decodedString.startsWith('{') || decodedString.startsWith('[')) {
                    try {
                        return JSON.parse(decodedString);
                    } catch {
                        // If JSON parsing fails, return the string as-is
                        return decodedString;
                    }
                }
                return decodedString;
            } catch (error) {
                throw new Error(`Failed to decode decompressed data: ${error.message}`);
            }
        } catch (error) {
            throw new Error(`Decompression failed: ${error.message}`);
        }
    }

    // Helper method to find longest matching sequence
    static findLongestMatch(data, currentPos) {
        // Input validation
        if (!data || !data.length || currentPos < 0 || currentPos >= data.length) {
            throw new Error('Invalid input parameters for findLongestMatch');
        }

        const windowSize = 1024;
        const maxLength = 258;
        const searchStart = Math.max(0, currentPos - windowSize);
        const remainingLength = data.length - currentPos;

        let bestLength = 0;
        let bestDistance = 0;

        // Bounds checking
        for (let i = searchStart; i < currentPos; i++) {
            let length = 0;
            
            // Safe length checking
            while (
                length < maxLength &&
                length < remainingLength &&
                i + length < currentPos &&
                data[i + length] === data[currentPos + length]
            ) {
                length++;
            }

            if (length > bestLength) {
                bestLength = length;
                bestDistance = currentPos - i;
            }
        }

        return { length: bestLength, distance: bestDistance };
    }

    static async cleanupTemporaryResources() {
        try {
            // In this implementation we don't actually need to clean up any resources
            // but we'll keep the method for interface compatibility
            return Promise.resolve();
        } catch (error) {
            console.error('Error cleaning up compression resources:', error);
            throw new Error('Failed to cleanup compression resources: ' + error.message);
        }
    }

    // Helper method to encode compressed data
    static encodeCompressed(compressed) {
        try {
            // Calculate total size needed
            let size = 0;
            compressed.forEach(token => {
                size += Array.isArray(token) ? 5 : 2; // 5 bytes for match, 2 for literal
            });

            const result = new Uint8Array(size);
            let pos = 0;

            compressed.forEach(token => {
                if (Array.isArray(token)) {
                    // Mark as match with flag byte 1
                    result[pos++] = 1;
                    // Store distance (2 bytes)
                    result[pos++] = token[0] >> 8;
                    result[pos++] = token[0] & 0xFF;
                    // Store length (2 bytes)
                    result[pos++] = token[1] >> 8;
                    result[pos++] = token[1] & 0xFF;
                } else {
                    // Mark as literal with flag byte 0
                    result[pos++] = 0;
                    // Store literal byte
                    result[pos++] = token;
                }
            });

            return result;
        } catch (error) {
            throw new Error(`Failed to encode compressed data: ${error.message}`);
        }
    }

    // Helper method to decode compressed data
    static decodeCompressed(data) {
        try {
            const result = [];
            let pos = 0;

            while (pos < data.length) {
                if (pos + 1 > data.length) {
                    throw new Error('Unexpected end of compressed data');
                }

                if (data[pos] === 1) {
                    // Read match
                    if (pos + 5 > data.length) {
                        throw new Error('Unexpected end of compressed data during match');
                    }
                    pos++;
                    const distance = (data[pos] << 8) | data[pos + 1];
                    const length = (data[pos + 2] << 8) | data[pos + 3];
                    result.push([distance, length]);
                    pos += 4;
                } else if (data[pos] === 0) {
                    // Read literal
                    pos++;
                    if (pos >= data.length) {
                        throw new Error('Unexpected end of compressed data after literal marker');
                    }
                    result.push(data[pos]);
                    pos++;
                } else {
                    throw new Error('Invalid token type in compressed data');
                }
            }

            return result;
        } catch (error) {
            throw new Error(`Failed to decode compressed data: ${error.message}`);
        }
    }
}

export { CompressionUtil }

--- End of modules/storage/utils/compression.js ---


--- File: modules/storage/utils/metrics.js ---

class StorageMetrics {
    constructor(config = {}) {
        this.metrics = {
            hits: 0,
            misses: 0,
            writes: 0,
            deletes: 0,
            errors: 0,
            totalSize: 0,
            itemCount: 0
        };

        this.accessTimes = [];
        this.maxAccessTimes = config.maxAccessTimes || 1000;
        this.sizeThreshold = config.sizeThreshold || 100000; // 100KB
        this.fragmentationThreshold = config.fragmentationThreshold || 0.3; // 30%
        
        // Performance tracking
        this.performanceMetrics = {
            averageAccessTime: 0,
            peakAccessTime: 0,
            lastOptimization: Date.now(),
            fragmentationLevel: 0
        };
    }

    recordHit() {
        this.metrics.hits++;
    }

    recordMiss() {
        this.metrics.misses++;
    }

    recordWrite() {
        this.metrics.writes++;
    }

    recordDelete() {
        this.metrics.deletes++;
    }

    recordError() {
        this.metrics.errors++;
    }

    recordAccessTime(duration) {
        this.accessTimes.push({
            timestamp: Date.now(),
            duration
        });

        // Update peak access time
        this.performanceMetrics.peakAccessTime = Math.max(
            this.performanceMetrics.peakAccessTime,
            duration
        );

        // Maintain size limit
        while (this.accessTimes.length > this.maxAccessTimes) {
            this.accessTimes.shift();
        }

        // Update average
        const sum = this.accessTimes.reduce((acc, val) => acc + val.duration, 0);
        this.performanceMetrics.averageAccessTime = sum / this.accessTimes.length;
    }

    updateSize(itemSize, isAddition = true) {
        if (isAddition) {
            this.metrics.totalSize += itemSize;
            this.metrics.itemCount++;
        } else {
            this.metrics.totalSize = Math.max(0, this.metrics.totalSize - itemSize);
            this.metrics.itemCount = Math.max(0, this.metrics.itemCount - 1);
        }
    }

    calculateItemSize(value) {
        if (typeof value === 'string') {
            return value.length * 2; // Approximate UTF-16 string size
        }
        return JSON.stringify(value).length * 2;
    }

    getHitRate() {
        const total = this.metrics.hits + this.metrics.misses;
        return total === 0 ? 0 : this.metrics.hits / total;
    }

    getWriteRate() {
        const total = this.metrics.writes + this.metrics.deletes;
        return total === 0 ? 0 : this.metrics.writes / total;
    }

    getErrorRate() {
        const total = this.getTotalOperations();
        return total === 0 ? 0 : this.metrics.errors / total;
    }

    getTotalOperations() {
        return this.metrics.hits + 
               this.metrics.misses + 
               this.metrics.writes + 
               this.metrics.deletes;
    }

    getPerformanceMetrics() {
        return {
            ...this.performanceMetrics,
            hitRate: this.getHitRate(),
            writeRate: this.getWriteRate(),
            errorRate: this.getErrorRate(),
            averageItemSize: this.getAverageItemSize(),
            utilizationRate: this.getUtilizationRate()
        };
    }

    getAverageItemSize() {
        return this.metrics.itemCount === 0 ? 
            0 : 
            this.metrics.totalSize / this.metrics.itemCount;
    }

    getUtilizationRate() {
        // Calculate storage utilization (used space vs. allocated space)
        return this.metrics.totalSize / (this.metrics.itemCount * this.getAverageItemSize());
    }

    analyzeAccessPatterns() {
        const now = Date.now();
        const recentAccesses = this.accessTimes.filter(
            access => now - access.timestamp < 3600000 // Last hour
        );

        return {
            averageAccessTime: this.performanceMetrics.averageAccessTime,
            hitRate: this.getHitRate(),
            recentAccessCount: recentAccesses.length,
            writeRate: this.getWriteRate(),
            errorRate: this.getErrorRate(),
            fragmentation: this.calculateFragmentation()
        };
    }

    shouldOptimize() {
        const analysis = this.analyzeAccessPatterns();
        
        return (
            analysis.hitRate < 0.5 || // Low hit rate
            analysis.fragmentation > this.fragmentationThreshold || // High fragmentation
            analysis.errorRate > 0.05 || // High error rate
            this.performanceMetrics.averageAccessTime > 100 // Slow access times
        );
    }

    calculateFragmentation() {
        if (this.metrics.itemCount === 0) return 0;

        // Calculate theoretical minimum space needed
        const idealSpace = this.metrics.itemCount * this.getAverageItemSize();
        
        // Compare with actual space used
        return Math.max(0, 1 - (idealSpace / this.metrics.totalSize));
    }

    reset() {
        this.metrics = {
            hits: 0,
            misses: 0,
            writes: 0,
            deletes: 0,
            errors: 0,
            totalSize: 0,
            itemCount: 0
        };
        
        this.accessTimes = [];
        this.performanceMetrics = {
            averageAccessTime: 0,
            peakAccessTime: 0,
            lastOptimization: Date.now(),
            fragmentationLevel: 0
        };
    }

    getMetricsSummary() {
        return {
            metrics: { ...this.metrics },
            performance: this.getPerformanceMetrics(),
            analysis: this.analyzeAccessPatterns(),
            timestamp: Date.now()
        };
    }
}

export {StorageMetrics}
--- End of modules/storage/utils/metrics.js ---



=== Folder: modules/vector ===


--- File: modules/vector/index.js ---

// modules/vector/index.js
export { EnhancedVectorStore } from './vectorStore';
export { VectorIndex } from './vectorIndex';
export { RandomProjectionTree } from './utils/projectionTree';
export { VectorSimilarity } from './utils/similarity';
--- End of modules/vector/index.js ---


--- File: modules/vector/vectorIndex.js ---

import { AsyncLock } from '../../concurrency';
import { RandomProjectionTree } from './utils/projectionTree';
import { ThoughtError } from '../errors/thoughtError';

class VectorIndex {
    constructor(dimensions = 128, numTrees = 10, maxLeafSize = 10) {
        this.dimensions = dimensions;
        this.numTrees = numTrees;
        this.maxLeafSize = maxLeafSize;
        this.trees = [];
        this.initializeTrees();
        this.vectorCache = new Map();
        this.lookupTable = new Map();
        this.indexLock = new AsyncLock();
        this.maintenanceInterval = null;
        this.lastMaintenance = Date.now();
    }

    async initializeTrees() {
        try {
            this.trees = Array.from({ length: this.numTrees }, () => 
                new RandomProjectionTree(this.dimensions, this.maxLeafSize)
            );
        } catch (error) {
            console.error('Failed to initialize trees:', error);
            throw new Error('Vector index initialization failed: ' + error.message);
        }
    }

    async add(key, embedding) {
        return await this.indexLock.acquire('add', async () => {
            try {
                // Validate input
                if (!key || !embedding) {
                    throw new Error('Invalid input: key and embedding are required');
                }
                if (embedding.length !== this.dimensions) {
                    throw new Error(`Invalid embedding dimensions: expected ${this.dimensions}, got ${embedding.length}`);
                }

                // Normalize the embedding
                const normalizedEmbedding = await this.normalizeVector(embedding);

                // Add to all trees
                await Promise.all(this.trees.map(tree => 
                    tree.insert(key, normalizedEmbedding)
                ));

                // Update lookup table
                this.lookupTable.set(key, {
                    timestamp: Date.now(),
                    embedding: normalizedEmbedding
                });

                // Cache the normalized vector
                this.vectorCache.set(key, normalizedEmbedding);

                return true;
            } catch (error) {
                console.error('Error adding vector:', error);
                throw new Error('Failed to add vector: ' + error.message);
            }
        });
    }

    async search(queryEmbedding, limit = 5, similarityThreshold = 0.5) {
        try {
            // Validate input
            if (!queryEmbedding || queryEmbedding.length !== this.dimensions) {
                throw new Error('Invalid query embedding');
            }

            // Normalize query vector
            const normalizedQuery = await this.normalizeVector(queryEmbedding);

            // Search in all trees
            const results = new Map();
            await Promise.all(this.trees.map(async tree => {
                const treeResults = await tree.search(normalizedQuery, limit * 2);
                treeResults.forEach(key => {
                    results.set(key, (results.get(key) || 0) + 1);
                });
            }));

            // Calculate actual similarities for top candidates
            const similarities = await Promise.all(
                Array.from(results.entries()).map(async ([key, count]) => {
                    const storedVector = this.vectorCache.get(key);
                    if (!storedVector) return null;
                    
                    const similarity = await this.calculateCosineSimilarity(
                        normalizedQuery,
                        storedVector
                    );
                    return { key, similarity, count };
                })
            );

            // Filter, sort and return top results
            return similarities
                .filter(result => result && result.similarity >= similarityThreshold)
                .sort((a, b) => {
                    // Primary sort by similarity
                    const simDiff = b.similarity - a.similarity;
                    if (Math.abs(simDiff) > 0.01) return simDiff;
                    // Secondary sort by tree occurrence count
                    return b.count - a.count;
                })
                .slice(0, limit)
                .map(result => ({
                    key: result.key,
                    similarity: result.similarity
                }));

        } catch (error) {
            console.error('Error during vector search:', error);
            throw new Error('Vector search failed: ' + error.message);
        }
    }

    async delete(key) {
        return await this.indexLock.acquire('delete', async () => {
            try {
                // Remove from all trees
                await Promise.all(this.trees.map(tree => tree.delete(key)));

                // Clear from cache and lookup table
                this.vectorCache.delete(key);
                this.lookupTable.delete(key);

                return true;
            } catch (error) {
                console.error('Error deleting vector:', error);
                throw new Error('Failed to delete vector: ' + error.message);
            }
        });
    }

    async update(key, newEmbedding) {
        return await this.indexLock.acquire('update', async () => {
            try {
                await this.delete(key);
                return await this.add(key, newEmbedding);
            } catch (error) {
                console.error('Error updating vector:', error);
                throw new Error('Failed to update vector: ' + error.message);
            }
        });
    }

    async normalize(embedding) {
        return await this.normalizeVector(embedding);
    }

    async maintenance() {
        return await this.indexLock.acquire('maintenance', async () => {
            try {
                // Rebuild trees that are unbalanced
                await this.rebuildUnbalancedTrees();

                // Clean up old entries
                await this.cleanupOldEntries();

                // Update maintenance timestamp
                this.lastMaintenance = Date.now();
            } catch (error) {
                console.error('Error during maintenance:', error);
                throw new Error('Maintenance failed: ' + error.message);
            }
        });
    }

    async rebuildUnbalancedTrees() {
        const unbalancedTrees = this.trees.filter(tree => tree.needsRebalancing());
        if (unbalancedTrees.length === 0) return;

        // Get all vectors
        const vectors = Array.from(this.lookupTable.entries()).map(([key, data]) => ({
            key,
            embedding: data.embedding
        }));

        // Rebuild unbalanced trees
        await Promise.all(unbalancedTrees.map(async tree => {
            const newTree = new RandomProjectionTree(this.dimensions, this.maxLeafSize);
            for (const vector of vectors) {
                await newTree.insert(vector.key, vector.embedding);
            }
            const treeIndex = this.trees.indexOf(tree);
            this.trees[treeIndex] = newTree;
        }));
    }

    async cleanupOldEntries() {
        const now = Date.now();
        const maxAge = 30 * 24 * 60 * 60 * 1000; // 30 days

        const oldKeys = Array.from(this.lookupTable.entries())
            .filter(([_, data]) => now - data.timestamp > maxAge)
            .map(([key]) => key);

        await Promise.all(oldKeys.map(key => this.delete(key)));
    }

    async normalizeVector(vector) {
        let norm = 0;
        for (let i = 0; i < vector.length; i++) {
            norm += vector[i] * vector[i];
        }
        norm = Math.sqrt(norm);

        if (norm === 0) {
            throw new Error('Cannot normalize zero vector');
        }

        return new Float32Array(vector.map(v => v / norm));
    }

    async calculateCosineSimilarity(a, b) {
        let dotProduct = 0;
        for (let i = 0; i < a.length; i++) {
            dotProduct += a[i] * b[i];
        }
        return Math.max(-1, Math.min(1, dotProduct));
    }

    async destroy() {
        try {
            // Clear maintenance interval if exists
            if (this.maintenanceInterval) {
                clearInterval(this.maintenanceInterval);
                this.maintenanceInterval = null;
            }

            // Destroy all trees
            await Promise.all(this.trees.map(tree => tree.destroy()));

            // Clear caches
            this.vectorCache.clear();
            this.lookupTable.clear();

            // Clear arrays
            this.trees = [];

            return true;
        } catch (error) {
            console.error('Error destroying vector index:', error);
            throw new Error('Failed to destroy vector index: ' + error.message);
        }
    }

    async getStats() {
        return {
            numTrees: this.trees.length,
            numVectors: this.lookupTable.size,
            dimensions: this.dimensions,
            lastMaintenance: this.lastMaintenance,
            cacheSize: this.vectorCache.size,
            treeStats: await Promise.all(this.trees.map(tree => tree.getStats()))
        };
    }
}
--- End of modules/vector/vectorIndex.js ---


--- File: modules/vector/vectorStoreBase.js ---

export class VectorStore {
    /**
     * @abstract
     * Generate an embedding for the given data
     * @param {*} data - Data to generate embedding for
     * @returns {Promise<Float32Array>} Generated embedding
     */
    async generateEmbedding(data) {
        throw new Error('generateEmbedding must be implemented');
    }

    /**
     * @abstract
     * Search for similar vectors
     * @param {Float32Array} queryEmbedding - Query vector
     * @param {number} limit - Maximum number of results
     * @param {number} similarityThreshold - Minimum similarity score
     * @returns {Promise<Array<{key: string, similarity: number}>>}
     */
    async search(queryEmbedding, limit, similarityThreshold) {
        throw new Error('search must be implemented');
    }

    /**
     * @abstract
     * Add a vector to the store
     * @param {string} key - Unique identifier
     * @param {Float32Array} embedding - Vector to store
     * @returns {Promise<void>}
     */
    async add(key, embedding) {
        throw new Error('add must be implemented');
    }

    /**
     * @abstract
     * Delete a vector from the store
     * @param {string} key - Key to delete
     * @returns {Promise<boolean>} True if deleted, false if not found
     */
    async delete(key) {
        throw new Error('delete must be implemented');
    }

    /**
     * @abstract
     * Clear all vectors from the store
     * @returns {Promise<void>}
     */
    async clear() {
        throw new Error('clear must be implemented');
    }

    /**
     * @abstract
     * Get the number of vectors in the store
     * @returns {Promise<number>}
     */
    async size() {
        throw new Error('size must be implemented');
    }

    /**
     * @abstract
     * Check if a key exists in the store
     * @param {string} key - Key to check
     * @returns {Promise<boolean>}
     */
    async has(key) {
        throw new Error('has must be implemented');
    }

    /**
     * @abstract
     * Get usage metrics for the store
     * @returns {Promise<Object>}
     */
    async getUsageMetrics() {
        throw new Error('getUsageMetrics must be implemented');
    }

    /**
     * @abstract
     * Clean up resources
     * @returns {Promise<void>}
     */
    async destroy() {
        throw new Error('destroy must be implemented');
    }

    /**
     * @abstract
     * Vacuum/optimize the store
     * @returns {Promise<void>}
     */
    async vacuum() {
        throw new Error('vacuum must be implemented');
    }

    /**
     * @abstract
     * Get the dimension of vectors in this store
     * @returns {number}
     */
    getDimensions() {
        throw new Error('getDimensions must be implemented');
    }
}
--- End of modules/vector/vectorStoreBase.js ---


--- File: modules/vector/vectorStore.js ---

import { AsyncLock } from '../concurrency';
import { VectorIndex } from './vectorIndex';
import { VectorSimilarity } from './utils/similarity';
import { ThoughtError } from '../errors/thoughtError';
import { VectorStore } from './vectorStoreBase';

class EnhancedVectorStore extends VectorStore {
    constructor() {
        super();

        this.store = new Map();
        this.index = new VectorIndex();
        this.deletedKeys = new Set();
        this.modelCache = new Map();
        this.dimensions = 128; // Embedding dimension

        // After creating the index, assume the index initializes one or more trees.
        // We'll set this.root to the root of the first tree for searchNode operations.
        // This depends on VectorIndex and RandomProjectionTree implementation details.
        // If your VectorIndex creates multiple trees, just pick the first one:
        if (this.index.trees && this.index.trees.length > 0) {
            this.root = this.index.trees[0].root;
        } else {
            // If no trees exist yet, you may need to handle this differently or lazily.
            this.root = null;
        }

        this.apiConfig = {
            endpoint: null,
            apiKey: null,
            modelName: null
        };

        this.searchLock = new AsyncLock();
        this.updateLock = new AsyncLock();
    }

    // Implement abstract methods from VectorStore
    async add(key, embedding) {
        this.validateVector(embedding);
        this.normalizeVector(embedding);
        await this.index.add(key, embedding);
        this.store.set(key, embedding);
    }

    async delete(key) {
        const deleted = await this.index.delete(key);
        if (deleted) {
            this.store.delete(key);
            this.deletedKeys.add(key);
            return true;
        }
        return false;
    }

    async clear() {
        // Remove all vectors
        for (const key of this.store.keys()) {
            await this.delete(key);
        }
    }

    async size() {
        // Return the number of vectors currently stored
        return this.store.size;
    }

    async has(key) {
        return this.store.has(key);
    }

    async vacuum() {
        // If VectorIndex supports maintenance, call it here
        if (typeof this.index.maintenance === 'function') {
            await this.index.maintenance();
        }
    }

    async getUsageMetrics() {
        // Return basic usage metrics. Adjust as needed.
        return {
            vectorCount: await this.size(),
            dimensions: this.dimensions
        };
    }

    async generateEmbedding(data) {
        if (!this.apiConfig.endpoint || !this.apiConfig.apiKey) {
            throw new Error('API configuration missing');
        }

        try {
            const response = await fetch(this.apiConfig.endpoint, {
                method: 'POST',
                headers: {
                    'Authorization': `Bearer ${this.apiConfig.apiKey}`,
                    'Content-Type': 'application/json'
                },
                body: JSON.stringify({
                    input: typeof data === 'string' ? data : JSON.stringify(data),
                    model: this.apiConfig.modelName
                })
            });

            if (!response.ok) {
                throw new Error(`API request failed with status ${response.status}`);
            }

            const result = await response.json();
            if (!result.embedding || !Array.isArray(result.embedding)) {
                throw new Error('Invalid embedding format received');
            }

            return new Float32Array(result.embedding);
        } catch (error) {
            throw new Error(`Embedding generation failed: ${error.message}`, { cause: error });
        }
    }

    async tokenize(text) {
        // Simple tokenization - split by whitespace and punctuation
        return text.toLowerCase()
            .replace(/[.,!?;:]/g, ' ')
            .split(/\s+/)
            .filter(token => token.length > 0);
    }

    async hashTokens(tokens) {
        // Create a stable hash of the tokens
        const text = tokens.join(' ');
        const encoder = new TextEncoder();
        const data = encoder.encode(text);
        const hashBuffer = await crypto.subtle.digest('SHA-256', data);
        return new Uint8Array(hashBuffer);
    }

    applyPositionalEncoding(embedding, sequenceLength) {
        // Apply sinusoidal positional encoding
        const positionScale = 10000;
        for (let i = 0; i < this.dimensions; i += 2) {
            const position = i / this.dimensions;
            const scale = Math.exp(-(position * Math.log(positionScale)));

            embedding[i] *= Math.sin(sequenceLength * scale);
            if (i + 1 < this.dimensions) {
                embedding[i + 1] *= Math.cos(sequenceLength * scale);
            }
        }
    }

    async applyTokenFeatures(embedding, tokens) {
        // Apply token-specific features
        const features = await this.extractTokenFeatures(tokens);

        // Blend features into embedding
        const blendFactor = 0.3;
        for (let i = 0; i < this.dimensions && i < features.length; i++) {
            embedding[i] = embedding[i] * (1 - blendFactor) + features[i] * blendFactor;
        }
    }

    async extractTokenFeatures(tokens) {
        const features = new Float32Array(this.dimensions);

        // Calculate token statistics
        const tokenFreq = new Map();
        tokens.forEach(token => {
            tokenFreq.set(token, (tokenFreq.get(token) || 0) + 1);
        });

        // Generate features based on token statistics
        let pos = 0;
        for (const [token, freq] of tokenFreq.entries()) {
            const tokenHash = await this.hashToken(token);
            const frequencyFactor = Math.log1p(freq) / Math.log1p(tokens.length);

            // Mix token features into the embedding
            for (let i = 0; i < 32 && pos < this.dimensions; i++, pos++) {
                features[pos] = (tokenHash[i % tokenHash.length] / 255) * frequencyFactor;
            }
        }

        return features;
    }

    async hashToken(token) {
        // Cache token hashes
        if (this.modelCache.has(token)) {
            return this.modelCache.get(token);
        }

        const encoder = new TextEncoder();
        const data = encoder.encode(token);
        const hashBuffer = await crypto.subtle.digest('SHA-256', data);
        const hash = new Uint8Array(hashBuffer);

        this.modelCache.set(token, hash);
        return hash;
    }

    normalizeVector(vector) {
        // L2 normalization
        let norm = 0;
        for (let i = 0; i < vector.length; i++) {
            norm += vector[i] * vector[i];
        }
        norm = Math.sqrt(norm);

        if (norm > 0) {
            for (let i = 0; i < vector.length; i++) {
                vector[i] /= norm;
            }
        }
    }

    validateVector(vector) {
        if (!vector || !Array.isArray(vector)) {
            throw new Error('Invalid vector: must be an array');
        }
        if (vector.length !== this.dimensions) {
            throw new Error(`Invalid vector dimensions: expected ${this.dimensions}, got ${vector.length}`);
        }
        if (!vector.every(v => typeof v === 'number' && !isNaN(v))) {
            throw new Error('Invalid vector: all elements must be numbers');
        }
    }

    // Overriding the search method from VectorStore:
    async search(queryEmbedding, limit = 5, similarityThreshold = 0.5) {
        // Validate input vector
        this.validateVector(queryEmbedding);

        return await this.searchLock.acquire('search', async () => {
            const results = new Map();

            try {
                if (!this.root) {
                    throw new Error('No root node available for searching');
                }
                await this.searchNode(this.root, queryEmbedding, limit, results);

                // Sort and filter results atomically
                return Array.from(results.entries())
                    .sort(([, scoreA], [, scoreB]) => scoreB - scoreA)
                    .filter(([, score]) => score >= similarityThreshold)
                    .slice(0, limit)
                    .map(([key]) => key);
            } catch (error) {
                console.error('Error during vector search:', error);
                throw new Error('Vector search failed: ' + error.message);
            }
        });
    }

    project(embedding, plane) {
        let sum = 0;
        for (let i = 0; i < this.dimensions; i++) {
            sum += embedding[i] * plane[i];
        }
        return sum;
    }

    async searchNode(node, queryEmbedding, limit, results) {
        if (node.isLeaf) {
            // Calculate actual similarity scores for leaf nodes
            for (const [key, embedding] of node.points) {
                const similarity = VectorSimilarity.cosineSimilarity(
                    queryEmbedding,
                    embedding
                );
                results.set(key, similarity);
            }
            return;
        }

        // Project query point and traverse tree
        const projection = this.project(queryEmbedding, node.splitPlane);
        const [primaryChild, secondaryChild] = projection <= 0
            ? [node.left, node.right]
            : [node.right, node.left];

        await this.searchNode(primaryChild, queryEmbedding, limit, results);

        // Check if we need to explore the other branch
        const currentBest = Array.from(results.values())
            .sort((a, b) => b - a)
            .slice(0, limit);
        const worstScore = currentBest.length < limit ? -Infinity : currentBest[currentBest.length - 1];

        // Explore other branch if it might contain better matches
        const splitDistance = Math.abs(projection);
        if (splitDistance < Math.sqrt(2 - 2 * worstScore)) {
            await this.searchNode(secondaryChild, queryEmbedding, limit, results);
        }
    }

    async destroy() {
        try {
            // Clear all stored vectors
            this.store.clear();

            // Clear index
            await this.index.destroy();

            // Clear caches
            this.modelCache.clear();
            this.deletedKeys.clear();

            // Release any held resources
            this.root = null;
        } catch (error) {
            console.error('Error destroying vector store:', error);
            throw new Error('Failed to destroy vector store: ' + error.message);
        }
    }
}

export { EnhancedVectorStore };
--- End of modules/vector/vectorStore.js ---



=== Folder: modules/vector/utils ===


--- File: modules/vector/utils/projectionTree.js ---

import { AsyncLock } from '../concurrency';

class RandomProjectionTree {
    constructor(dimensions = 128, maxLeafSize = 10) {
        this.dimensions = dimensions;
        this.maxLeafSize = maxLeafSize;
        this.root = this.createNode();
        this.nodeCount = 1;
        this.depth = 0;
        this.maxDepth = 0;
        this.vectorCount = 0;
        this.lock = new AsyncLock();
        this.rebalanceThreshold = 2; // Tree is unbalanced if depth ratio exceeds this
    }

    createNode() {
        return {
            isLeaf: true,
            points: new Map(), // key -> embedding
            splitPlane: null,
            left: null,
            right: null,
            size: 0,
            depth: 0,
            parent: null
        };
    }

    async insert(key, embedding) {
        return await this.lock.acquire('insert', async () => {
            try {
                if (!embedding || embedding.length !== this.dimensions) {
                    throw new Error(`Invalid embedding dimensions: expected ${this.dimensions}`);
                }

                await this.insertAtNode(this.root, key, embedding);
                this.vectorCount++;
                return true;
            } catch (error) {
                console.error('Error inserting into tree:', error);
                throw new Error('Failed to insert into tree: ' + error.message);
            }
        });
    }

    async insertAtNode(node, key, embedding) {
        if (node.isLeaf) {
            node.points.set(key, new Float32Array(embedding));
            node.size++;

            // Split if too many points and not too deep
            if (node.points.size > this.maxLeafSize && node.depth < 20) {
                await this.splitNode(node);
            }
            return;
        }

        // Non-leaf node: traverse to appropriate child
        const projection = this.project(embedding, node.splitPlane);
        const nextNode = projection <= 0 ? node.left : node.right;
        nextNode.parent = node;
        nextNode.depth = node.depth + 1;
        this.maxDepth = Math.max(this.maxDepth, nextNode.depth);
        
        await this.insertAtNode(nextNode, key, embedding);
        node.size++;
    }

    async splitNode(node) {
        try {
            // Generate random splitting plane
            node.splitPlane = await this.generateRandomPlane();
            node.left = this.createNode();
            node.right = this.createNode();
            node.isLeaf = false;
            this.nodeCount += 2;

            // Set child properties
            node.left.depth = node.right.depth = node.depth + 1;
            node.left.parent = node.right.parent = node;

            // Redistribute points
            for (const [key, embedding] of node.points) {
                const projection = this.project(embedding, node.splitPlane);
                const targetNode = projection <= 0 ? node.left : node.right;
                targetNode.points.set(key, embedding);
                targetNode.size++;
            }

            // Update max depth
            this.maxDepth = Math.max(this.maxDepth, node.depth + 1);

            // Clear points from this node
            node.points.clear();
        } catch (error) {
            console.error('Error splitting node:', error);
            throw new Error('Failed to split node: ' + error.message);
        }
    }

    async generateRandomPlane() {
        // Generate random unit vector for splitting plane
        const plane = new Float32Array(this.dimensions);
        let sumSquares = 0;

        // Use crypto random for better randomness if available
        if (typeof crypto !== 'undefined' && crypto.getRandomValues) {
            const randomBytes = new Uint8Array(this.dimensions);
            crypto.getRandomValues(randomBytes);
            for (let i = 0; i < this.dimensions; i++) {
                plane[i] = (randomBytes[i] / 255) * 2 - 1;
                sumSquares += plane[i] * plane[i];
            }
        } else {
            for (let i = 0; i < this.dimensions; i++) {
                plane[i] = Math.random() * 2 - 1;
                sumSquares += plane[i] * plane[i];
            }
        }

        // Normalize the plane vector
        const norm = Math.sqrt(sumSquares);
        for (let i = 0; i < this.dimensions; i++) {
            plane[i] /= norm;
        }

        return plane;
    }

    project(embedding, plane) {
        let sum = 0;
        for (let i = 0; i < this.dimensions; i++) {
            sum += embedding[i] * plane[i];
        }
        return sum;
    }

    async search(queryEmbedding, limit = 5) {
        return await this.lock.acquire('search', async () => {
            try {
                if (!queryEmbedding || queryEmbedding.length !== this.dimensions) {
                    throw new Error('Invalid query embedding dimensions');
                }

                const results = new Map();
                await this.searchNode(this.root, queryEmbedding, limit, results);

                return Array.from(results.keys())
                    .sort((a, b) => results.get(b) - results.get(a))
                    .slice(0, limit);
            } catch (error) {
                console.error('Error during search:', error);
                throw new Error('Search failed: ' + error.message);
            }
        });
    }

    async searchNode(node, queryEmbedding, limit, results) {
        if (node.isLeaf) {
            for (const [key, embedding] of node.points) {
                const similarity = await this.calculateSimilarity(queryEmbedding, embedding);
                results.set(key, similarity);
            }
            return;
        }

        const projection = this.project(queryEmbedding, node.splitPlane);
        const [primaryChild, secondaryChild] = projection <= 0
            ? [node.left, node.right]
            : [node.right, node.left];

        await this.searchNode(primaryChild, queryEmbedding, limit, results);

        // Check if we need to explore the other branch
        if (this.shouldExploreSecondaryBranch(projection, results, limit)) {
            await this.searchNode(secondaryChild, queryEmbedding, limit, results);
        }
    }

    shouldExploreSecondaryBranch(projection, results, limit) {
        // If we don't have enough results yet, explore the other branch
        if (results.size < limit) return true;

        // Get the worst score among our current top results
        const scores = Array.from(results.values()).sort((a, b) => b - a);
        const worstScore = scores[limit - 1];

        // Calculate the maximum possible similarity in the other branch
        const splitDistance = Math.abs(projection);
        const maxPossibleSimilarity = Math.sqrt(1 - splitDistance * splitDistance);

        // Explore if the other branch might contain better results
        return maxPossibleSimilarity > worstScore;
    }

    async calculateSimilarity(a, b) {
        let dotProduct = 0;
        for (let i = 0; i < this.dimensions; i++) {
            dotProduct += a[i] * b[i];
        }
        return Math.max(-1, Math.min(1, dotProduct));
    }

    async delete(key) {
        return await this.lock.acquire('delete', async () => {
            try {
                const deleted = await this.deleteFromNode(this.root, key);
                if (deleted) {
                    this.vectorCount--;
                }
                return deleted;
            } catch (error) {
                console.error('Error deleting from tree:', error);
                throw new Error('Failed to delete from tree: ' + error.message);
            }
        });
    }

    async deleteFromNode(node, key) {
        if (node.isLeaf) {
            const deleted = node.points.delete(key);
            if (deleted) {
                node.size--;
                this.updateSizeUpwards(node.parent);
            }
            return deleted;
        }

        // Try both children if not leaf
        const deletedLeft = await this.deleteFromNode(node.left, key);
        if (deletedLeft) {
            node.size--;
            return true;
        }

        const deletedRight = await this.deleteFromNode(node.right, key);
        if (deletedRight) {
            node.size--;
            return true;
        }

        return false;
    }

    updateSizeUpwards(node) {
        while (node) {
            node.size = (node.left ? node.left.size : 0) + 
                       (node.right ? node.right.size : 0);
            node = node.parent;
        }
    }

    needsRebalancing() {
        if (this.vectorCount < 100) return false; // Don't rebalance small trees
        
        const minDepth = this.getMinDepth(this.root);
        const depthRatio = this.maxDepth / minDepth;
        
        return depthRatio > this.rebalanceThreshold;
    }

    getMinDepth(node) {
        if (node.isLeaf) return node.depth;
        return Math.min(
            this.getMinDepth(node.left),
            this.getMinDepth(node.right)
        );
    }

    async destroy() {
        return await this.lock.acquire('destroy', async () => {
            try {
                this.destroyNode(this.root);
                this.root = null;
                this.nodeCount = 0;
                this.vectorCount = 0;
                this.depth = 0;
                this.maxDepth = 0;
                return true;
            } catch (error) {
                console.error('Error destroying tree:', error);
                throw new Error('Failed to destroy tree: ' + error.message);
            }
        });
    }

    destroyNode(node) {
        if (!node) return;
        
        if (node.isLeaf) {
            node.points.clear();
        } else {
            this.destroyNode(node.left);
            this.destroyNode(node.right);
        }
        
        node.splitPlane = null;
        node.left = null;
        node.right = null;
        node.parent = null;
    }

    async getStats() {
        return {
            nodeCount: this.nodeCount,
            vectorCount: this.vectorCount,
            depth: this.maxDepth,
            minDepth: this.getMinDepth(this.root),
            averageLeafSize: this.calculateAverageLeafSize(),
            isBalanced: !this.needsRebalancing()
        };
    }

    calculateAverageLeafSize() {
        let leafSizes = [];
        this.collectLeafSizes(this.root, leafSizes);
        
        if (leafSizes.length === 0) return 0;
        
        const sum = leafSizes.reduce((a, b) => a + b, 0);
        return sum / leafSizes.length;
    }

    collectLeafSizes(node, sizes) {
        if (!node) return;
        
        if (node.isLeaf) {
            sizes.push(node.points.size);
        } else {
            this.collectLeafSizes(node.left, sizes);
            this.collectLeafSizes(node.right, sizes);
        }
    }
}
--- End of modules/vector/utils/projectionTree.js ---


--- File: modules/vector/utils/similarity.js ---

class VectorSimilarity {
    static validateVectors(a, b) {
        if (!Array.isArray(a) || !Array.isArray(b)) {
            throw new Error('Vectors must be arrays');
        }
        if (a.length !== b.length) {
            throw new Error(`Vector lengths don't match: ${a.length} vs ${b.length}`);
        }
        if (!a.every(v => typeof v === 'number' && !isNaN(v)) ||
            !b.every(v => typeof v === 'number' && !isNaN(v))) {
            throw new Error('Vectors must contain only numbers');
        }
    }

    static cosineSimilarity(a, b) {
        this.validateVectors(a, b);

        let dotProduct = 0;
        let normA = 0;
        let normB = 0;

        for (let i = 0; i < a.length; i++) {
            dotProduct += a[i] * b[i];
            normA += a[i] * a[i];
            normB += b[i] * b[i];
        }

        // Handle zero vectors
        if (normA === 0 || normB === 0) {
            return 0;
        }

        // Use Math.fround for numerical stability
        return Math.fround(dotProduct / (Math.sqrt(normA) * Math.sqrt(normB)));
    }

    static euclideanDistance(a, b) {
        let sum = 0;
        for (let i = 0; i < a.length; i++) {
            const diff = a[i] - b[i];
            sum += diff * diff;
        }
        return Math.sqrt(sum);
    }

    static manhattanDistance(a, b) {
        let sum = 0;
        for (let i = 0; i < a.length; i++) {
            sum += Math.abs(a[i] - b[i]);
        }
        return sum;
    }
}
--- End of modules/vector/utils/similarity.js ---



=== Folder: tests ===


--- File: tests/MockIndexedDB.js ---

// Mock implementation of IDBKeyRange
const IDBKeyRange = {
    lowerBound: (value) => ({ lower: value, upperOpen: true, lowerOpen: false }),
    upperBound: (value) => ({ upper: value, upperOpen: false, lowerOpen: true }),
    bound: (lower, upper, lowerOpen, upperOpen) => ({ lower, upper, lowerOpen, upperOpen }),
    only: (value) => ({ only: value })
};

class MockIDBRequest {
    constructor() {
        this.result = undefined;
        this.error = null;
        this.source = null;
        this.transaction = null;
        this.readyState = 'pending';
        this.onsuccess = null;
        this.onerror = null;
    }
}

class MockIDBIndex {
    constructor(store, name, keyPath, options = {}) {
        this.store = store;
        this.name = name;
        this.keyPath = keyPath;
        this.unique = options.unique || false;
        this.multiEntry = options.multiEntry || false;
    }

    getIndexValue(entry) {
        if (typeof this.keyPath === 'string') {
            return entry[this.keyPath];
        }
        // Handle array keyPath if needed
        if (Array.isArray(this.keyPath)) {
            return this.keyPath.map(path => entry[path]);
        }
        return undefined;
    }

    openCursor(range) {
        const request = new MockIDBRequest();
        let entries = Array.from(this.store.data.values());

        // Filter entries based on the index and range
        if (range) {
            entries = entries.filter(entry => {
                const value = this.getIndexValue(entry);
                if (range.only !== undefined) {
                    return value === range.only;
                }
                if (range.lower !== undefined && range.upper !== undefined) {
                    const lowerCheck = range.lowerOpen ? value > range.lower : value >= range.lower;
                    const upperCheck = range.upperOpen ? value < range.upper : value <= range.upper;
                    return lowerCheck && upperCheck;
                }
                if (range.lower !== undefined) {
                    return range.lowerOpen ? value > range.lower : value >= range.lower;
                }
                if (range.upper !== undefined) {
                    return range.upperOpen ? value < range.upper : value <= range.upper;
                }
                return true;
            });
        }

        let currentIndex = 0;

        const advanceCursor = () => {
            if (currentIndex < entries.length) {
                const value = entries[currentIndex];
                const cursor = {
                    value,
                    continue: () => {
                        currentIndex++;
                        setTimeout(advanceCursor, 0);
                    },
                    delete: () => {
                        const delRequest = this.store.delete(value[this.store.keyPath]);
                        delRequest.onsuccess = () => cursor.continue();
                        return delRequest;
                    }
                };
                request.result = cursor;
                request.onsuccess && request.onsuccess({ target: request });
            } else {
                request.result = null;
                request.onsuccess && request.onsuccess({ target: request });
            }
        };

        setTimeout(advanceCursor, 0);
        return request;
    }

    count(key) {
        const request = new MockIDBRequest();
        setTimeout(() => {
            try {
                let count = 0;
                const entries = Array.from(this.store.data.values());
                for (const entry of entries) {
                    const value = this.getIndexValue(entry);
                    if (key === undefined) {
                        count++;
                    } else {
                        // If key is provided, just do a simple equality check
                        if (value === key) count++;
                    }
                }
                request.result = count;
                request.onsuccess && request.onsuccess({ target: request });
            } catch (error) {
                request.error = error;
                request.onerror && request.onerror({ target: request });
            }
        }, 0);
        return request;
    }
}

class MockIDBObjectStore {
    constructor(name, options = {}) {
        this.name = name;
        this.keyPath = options.keyPath || 'id';
        this.autoIncrement = options.autoIncrement || false;
        this.data = new Map();
        this.indexes = new Map();
    }

    put(value) {
        const request = new MockIDBRequest();
        setTimeout(() => {
            try {
                const key = value[this.keyPath];
                if (key === undefined) {
                    throw new Error('Key path not found in value');
                }
                this.data.set(key, value);
                request.result = key;
                request.onsuccess && request.onsuccess({ target: request });
            } catch (error) {
                request.error = error;
                request.onerror && request.onerror({ target: request });
            }
        }, 0);
        return request;
    }

    get(key) {
        const request = new MockIDBRequest();
        setTimeout(() => {
            try {
                request.result = this.data.get(key);
                request.onsuccess && request.onsuccess({ target: request });
            } catch (error) {
                request.error = error;
                request.onerror && request.onerror({ target: request });
            }
        }, 0);
        return request;
    }

    delete(key) {
        const request = new MockIDBRequest();
        setTimeout(() => {
            try {
                this.data.delete(key);
                request.onsuccess && request.onsuccess({ target: request });
            } catch (error) {
                request.error = error;
                request.onerror && request.onerror({ target: request });
            }
        }, 0);
        return request;
    }

    clear() {
        const request = new MockIDBRequest();
        setTimeout(() => {
            try {
                this.data.clear();
                request.onsuccess && request.onsuccess({ target: request });
            } catch (error) {
                request.error = error;
                request.onerror && request.onerror({ target: request });
            }
        }, 0);
        return request;
    }

    createIndex(name, keyPath, options = {}) {
        const index = new MockIDBIndex(this, name, keyPath, options);
        this.indexes.set(name, index);
        return index;
    }

    index(name) {
        return this.indexes.get(name);
    }

    count(key) {
        const request = new MockIDBRequest();
        setTimeout(() => {
            try {
                if (key === undefined) {
                    request.result = this.data.size;
                } else {
                    request.result = this.data.has(key) ? 1 : 0;
                }
                request.onsuccess && request.onsuccess({ target: request });
            } catch (error) {
                request.error = error;
                request.onerror && request.onerror({ target: request });
            }
        }, 0);
        return request;
    }
}

class MockIDBTransaction {
    constructor(db, storeNames, mode = 'readonly') {
        this.db = db;
        this.storeNames = Array.isArray(storeNames) ? storeNames : [storeNames];
        this.mode = mode;
        this.error = null;
        this.aborted = false;
        this.startTime = Date.now();
    }

    objectStore(name) {
        if (!this.storeNames.includes(name)) {
            throw new Error(`Store ${name} not found in transaction`);
        }
        return this.db.stores.get(name);
    }

    abort() {
        this.aborted = true;
        if (this.onerror) {
            const error = new Error('Transaction aborted');
            this.error = error;
            this.onerror(new Event('error'));
        }
    }
}

class MockIDBDatabase {
    constructor(name) {
        this.name = name;
        this.version = 1;
        this.objectStoreNames = {
            contains: function(name) {
                return this._stores.includes(name);
            },
            _stores: []
        };
        this.stores = new Map();
        this.closed = false;
        this.onclose = null;
        this.onerror = null;
    }

    createObjectStore(name, options = {}) {
        const store = new MockIDBObjectStore(name, options);
        this.stores.set(name, store);
        this.objectStoreNames._stores.push(name);
        return store;
    }

    transaction(storeNames, mode = 'readonly') {
        return new MockIDBTransaction(this, storeNames, mode);
    }

    close() {
        this.closed = true;
        if (this.onclose) {
            this.onclose(new Event('close'));
        }
    }
}

const indexedDB = {
    databases: new Map(),

    open(name, version = 1) {
        const request = new MockIDBRequest();
        
        setTimeout(() => {
            try {
                let db;
                const existing = this.databases.get(name);
                
                if (!existing) {
                    db = new MockIDBDatabase(name);
                    this.databases.set(name, db);
                    request.result = db;
                    
                    if (request.onupgradeneeded) {
                        const event = {
                            target: request,
                            oldVersion: 0,
                            newVersion: version
                        };
                        request.onupgradeneeded(event);
                    }
                } else {
                    db = existing;
                }

                request.result = db;
                if (request.onsuccess) {
                    request.onsuccess({ target: request });
                }
            } catch (error) {
                request.error = error;
                if (request.onerror) {
                    request.onerror({ target: request });
                }
            }
        }, 0);

        return request;
    },

    deleteDatabase(name) {
        const request = new MockIDBRequest();
        setTimeout(() => {
            try {
                this.databases.delete(name);
                request.onsuccess && request.onsuccess({ target: request });
            } catch (error) {
                request.error = error;
                request.onerror && request.onerror({ target: request });
            }
        }, 0);
        return request;
    }
};

export {
    indexedDB,
    MockIDBDatabase,
    MockIDBIndex,
    MockIDBObjectStore,
    MockIDBRequest,
    MockIDBTransaction,
    IDBKeyRange
};

--- End of tests/MockIndexedDB.js ---


--- File: tests/test-setup.js ---

// tests/test-setup.js

import { jest } from '@jest/globals';
import { indexedDB, MockIDBDatabase, MockIDBIndex, MockIDBObjectStore, MockIDBRequest, MockIDBTransaction } from "./MockIndexedDB"
import { IDBKeyRange } from './MockIndexedDB.js';

global.IDBKeyRange = IDBKeyRange;

// Mock implementation of crypto for tests
const mockCrypto = {
    subtle: {
        digest: jest.fn().mockImplementation((algorithm, data) => {
            // Just ignore 'algorithm' since we know it's SHA-256
            // 'data' is an ArrayBuffer. Pass it directly to our sha256 function:
            const hashBuffer = sha256(data);

            // Return a promise that resolves with the computed hash ArrayBuffer
            return Promise.resolve(hashBuffer);
        }),
        generateKey: jest.fn(),
        sign: jest.fn(),
        verify: jest.fn()
    },
    getRandomValues: jest.fn().mockImplementation(arr => {
        for (let i = 0; i < arr.length; i++) {
            arr[i] = Math.floor(Math.random() * 256);
        }
        return arr;
    })
};

function sha256(buffer) {
    // Convert ArrayBuffer to array
    const bytes = new Uint8Array(buffer);

    // SHA-256 constants and functions
    const K = [
        0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
        0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
        0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
        0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
        0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
        0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
        0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
        0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
        0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
        0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
        0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
        0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
        0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
        0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
        0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
        0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
    ];

    const H = [
        0x6a09e667, 0xbb67ae85,
        0x3c6ef372, 0xa54ff53a,
        0x510e527f, 0x9b05688c,
        0x1f83d9ab, 0x5be0cd19
    ];

    // Preprocessing
    const length = bytes.length;
    const bitLength = length * 8;

    // Append '1' bit
    const withOne = new Uint8Array(length + 1);
    withOne.set(bytes, 0);
    withOne[length] = 0x80;

    // Calculate the required padding
    let padLength = (64 - ((length + 9) % 64)) % 64;
    const padded = new Uint8Array(length + 1 + padLength + 8);
    padded.set(withOne, 0);

    // Append length in bits as a 64-bit big-endian integer
    const view = new DataView(padded.buffer);
    view.setUint32(padded.length - 4, bitLength, false);

    // Process the message in successive 512-bit chunks
    for (let i = 0; i < padded.length; i += 64) {
        const w = new Uint32Array(64);

        for (let j = 0; j < 16; j++) {
            w[j] = view.getUint32(i + j * 4, false);
        }

        for (let j = 16; j < 64; j++) {
            const s0 = rightRotate(w[j - 15], 7) ^ rightRotate(w[j - 15], 18) ^ (w[j - 15] >>> 3);
            const s1 = rightRotate(w[j - 2], 17) ^ rightRotate(w[j - 2], 19) ^ (w[j - 2] >>> 10);
            w[j] = (w[j - 16] + s0 + w[j - 7] + s1) >>> 0;
        }

        let a = H[0], b = H[1], c = H[2], d = H[3];
        let e = H[4], f = H[5], g = H[6], h = H[7];

        for (let j = 0; j < 64; j++) {
            const S1 = (rightRotate(e, 6) ^ rightRotate(e, 11) ^ rightRotate(e, 25));
            const ch = ((e & f) ^ (~e & g));
            const temp1 = (h + S1 + ch + K[j] + w[j]) >>> 0;
            const S0 = (rightRotate(a, 2) ^ rightRotate(a, 13) ^ rightRotate(a, 22));
            const maj = ((a & b) ^ (a & c) ^ (b & c));
            const temp2 = (S0 + maj) >>> 0;

            h = g; g = f; f = e; e = (d + temp1) >>> 0;
            d = c; c = b; b = a; a = (temp1 + temp2) >>> 0;
        }

        H[0] = (H[0] + a) >>> 0;
        H[1] = (H[1] + b) >>> 0;
        H[2] = (H[2] + c) >>> 0;
        H[3] = (H[3] + d) >>> 0;
        H[4] = (H[4] + e) >>> 0;
        H[5] = (H[5] + f) >>> 0;
        H[6] = (H[6] + g) >>> 0;
        H[7] = (H[7] + h) >>> 0;
    }

    // Convert H values to a single ArrayBuffer
    const result = new ArrayBuffer(32);
    const resultView = new DataView(result);
    for (let i = 0; i < 8; i++) {
        resultView.setUint32(i * 4, H[i], false);
    }

    return result;
}

function rightRotate(value, amount) {
    return (value >>> amount) | (value << (32 - amount));
}


// Helper to create mock nodes for graph testing
const createMockNode = (id, dependencies = []) => ({
    id,
    dependencies,
    type: 'test',
    execute: async () => `executed ${id}`
});

// Helper to create mock performance tracker
const createMockPerformanceTracker = () => ({
    getThought: jest.fn().mockImplementation(id => ({
        id,
        type: 'test',
        execute: async () => 'executed'
    })),
    getMetrics: jest.fn().mockImplementation(() => ({
        executionTime: 100,
        memoryUsage: 50,
        cpuUsage: 0.5
    })),
    getThoughtMetrics: jest.fn().mockImplementation(() => ({
        executionTime: 100,
        memoryUsage: 50,
        cpuUsage: 0.5
    }))
});

// Helper to wait for all promises to resolve
const flushPromises = () => new Promise(resolve => setImmediate(resolve));

// Helper to create a delay promise
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));

// Mock LRUCache class with full implementation
class MockLRUCache {
    constructor(capacity) {
        this.capacity = capacity;
        this.cache = new Map();
        this.head = { key: null, value: null, prev: null, next: null };
        this.tail = { key: null, value: null, prev: this.head, next: null };
        this.head.next = this.tail;
    }

    get(key) {
        const node = this.cache.get(key);
        if (node) {
            this.moveToFront(node);
            return node.value;
        }
        return undefined;
    }

    set(key, value) {
        if (this.cache.has(key)) {
            const node = this.cache.get(key);
            node.value = value;
            this.moveToFront(node);
        } else {
            const node = {
                key,
                value,
                prev: this.head,
                next: this.head.next
            };
            this.cache.set(key, node);
            this.head.next.prev = node;
            this.head.next = node;

            if (this.cache.size > this.capacity) {
                const lru = this.tail.prev;
                this.removeNode(lru);
                this.cache.delete(lru.key);
            }
        }
    }

    moveToFront(node) {
        this.removeNode(node);
        node.prev = this.head;
        node.next = this.head.next;
        this.head.next.prev = node;
        this.head.next = node;
    }

    removeNode(node) {
        node.prev.next = node.next;
        node.next.prev = node.prev;
    }

    delete(key) {
        const node = this.cache.get(key);
        if (node) {
            this.removeNode(node);
            this.cache.delete(key);
            return true;
        }
        return false;
    }

    clear() {
        this.cache.clear();
        this.head.next = this.tail;
        this.tail.prev = this.head;
    }

    size() {
        return this.cache.size;
    }
}


// Setup test environment
const setupTestEnvironment = () => {
    // Mock window and global objects
    global.crypto = mockCrypto;

    // Mock fetch
    global.fetch = jest.fn().mockImplementation(() =>
        Promise.resolve({
            ok: true,
            json: () => Promise.resolve({})
        })
    );

    global.indexedDB = indexedDB;

    // Mock text encoder/decoder
    global.TextEncoder = jest.fn().mockImplementation(() => ({
        encode: jest.fn().mockImplementation(text => new Uint8Array([...text].map(c => c.charCodeAt(0))))
    }));

    global.TextDecoder = jest.fn().mockImplementation(() => ({
        decode: jest.fn().mockImplementation(arr => String.fromCharCode.apply(null, arr))
    }));

    // Mock performance API
    global.performance = {
        now: jest.fn(() => Date.now())
    };

    // Mock setImmediate if not available
    if (typeof setImmediate === 'undefined') {
        global.setImmediate = (callback) => setTimeout(callback, 0);
    }
};

// Cleanup test environment
const cleanupTestEnvironment = () => {
    jest.clearAllMocks();
    if (global.crypto) delete global.crypto;
    if (global.fetch) delete global.fetch;
    if (global.TextEncoder) delete global.TextEncoder;
    if (global.TextDecoder) delete global.TextDecoder;
};

export {
    createMockPerformanceTracker,
    createMockNode,
    flushPromises,
    delay,
    setupTestEnvironment,
    cleanupTestEnvironment,
    MockLRUCache
};
--- End of tests/test-setup.js ---



=== Folder: tests/chain ===


--- File: tests/chain/chainutil.test.js ---

import { ChainUtil } from '../../modules/chain/utils/chainUtil';
import { setupTestEnvironment, cleanupTestEnvironment, createMockNode } from '../test-setup';

describe('ChainUtil', () => {
    beforeEach(() => {
        setupTestEnvironment();
    });

    afterEach(() => {
        cleanupTestEnvironment();
    });

    describe('hashChain', () => {
        it('should generate consistent hashes for identical chains', async () => {
            const chain1 = [
                createMockNode('1'),
                createMockNode('2', ['1'])
            ];
            const chain2 = [
                createMockNode('1'),
                createMockNode('2', ['1'])
            ];

            const hash1 = await ChainUtil.hashChain(chain1);
            const hash2 = await ChainUtil.hashChain(chain2);
            expect(hash1).toBe(hash2);
        });

        it('should generate different hashes for different chains', async () => {
            const chain1 = [{ id: '1', dependencies: ['2'] }];
            const chain2 = [{ id: '1', dependencies: ['3'] }];

            const hash1 = await ChainUtil.hashChain(chain1);
            const hash2 = await ChainUtil.hashChain(chain2);

            // Check if hashes are different strings rather than exact equality
            expect(hash1).not.toEqual(hash2);
        });

        it('should handle empty chains', async () => {
            const hash = await ChainUtil.hashChain([]);
            expect(typeof hash).toBe('string');
            expect(hash).toBeTruthy();
        });
    });

    describe('explorePath', () => {
        it('should find all valid paths in a graph', () => {
            const graph = new Map([
                ['1', { dependents: new Set(['2']), dependencies: [] }],
                ['2', { dependents: new Set(['3']), dependencies: ['1'] }],
                ['3', { dependents: new Set(), dependencies: ['2'] }]
            ]);

            const paths = ChainUtil.explorePath('1', graph);
            expect(paths).toContainEqual(['1', '2', '3']);
        });

        it('should handle branching paths', () => {
            const graph = new Map([
                ['1', { dependents: new Set(['2', '3']), dependencies: [] }],
                ['2', { dependents: new Set(['4']), dependencies: ['1'] }],
                ['3', { dependents: new Set(['4']), dependencies: ['1'] }],
                ['4', { dependents: new Set(), dependencies: ['2', '3'] }]
            ]);

            const paths = ChainUtil.explorePath('1', graph);

            // All paths should exist and be valid
            expect(paths.length).toBeGreaterThan(0);
            paths.forEach(path => {
                // Check path starts with root node
                expect(path[0]).toBe('1');
                // Path should be connected through dependencies
                for (let i = 1; i < path.length; i++) {
                    const node = graph.get(path[i]);
                    expect(node.dependencies).toContain(path[i - 1]);
                }
            });
        });

        it('should detect circular dependencies', () => {
            const graph = new Map([
                ['1', { dependents: new Set(['2']), dependencies: ['3'] }],
                ['2', { dependents: new Set(['3']), dependencies: ['1'] }],
                ['3', { dependents: new Set(['1']), dependencies: ['2'] }]
            ]);

            expect(() => ChainUtil.explorePath('1', graph))
                .toThrow('Circular dependency detected');
        });
    });

    describe('combinePaths', () => {
        it('should merge parallel and sequential paths', () => {
            const paths = [
                ['1', '2', '3'],
                ['4', '5'],
                ['1', '4', '6']
            ];

            const result = ChainUtil.combinePaths(paths);

            expect(result.parallel).toContain('1');
            expect(result.parallel).toContain('4');

            const sequentialMap = new Map(result.sequential);
            expect(sequentialMap.get('2')).toContain('1');
            expect(sequentialMap.get('3')).toContain('2');
        });

        it('should handle pre-optimized paths', () => {
            const optimizedPath = {
                parallel: ['1', '4'],
                sequential: [['2', ['1']], ['3', ['2']]]
            };

            const result = ChainUtil.combinePaths([optimizedPath]);

            expect(result.parallel).toContain('1');
            expect(result.parallel).toContain('4');

            const sequentialMap = new Map(result.sequential);
            expect(sequentialMap.get('2')).toContain('1');
            expect(sequentialMap.get('3')).toContain('2');
        });

        it('should handle empty inputs', () => {
            expect(ChainUtil.combinePaths([])).toEqual({
                parallel: [],
                sequential: []
            });
        });
    });
});
--- End of tests/chain/chainutil.test.js ---


--- File: tests/chain/optimizer.test.js ---

import { EnhancedPatternOptimizer } from '../../modules/chain/optimizer';
import { setupTestEnvironment, cleanupTestEnvironment } from '../test-setup';

describe('EnhancedPatternOptimizer AI Integration', () => {
    let optimizer;
    let mockPerformanceTracker;
    let originalFetch;

    beforeEach(() => {
        setupTestEnvironment();
        mockPerformanceTracker = {
            getThought: jest.fn(id => ({
                id,
                type: 'test',
                execute: async () => `executed ${id}`
            })),
            getMetrics: jest.fn(() => ({
                executionTime: 100,
                memoryUsage: 50,
                cpuUsage: 0.5
            })),
            getThoughtMetrics: jest.fn()
        };
        optimizer = new EnhancedPatternOptimizer(mockPerformanceTracker);
        optimizer.aiConfig = {
            endpoint: 'https://api.example.com/optimize',
            apiKey: 'test-api-key',
            optimizerModel: 'test-model'
        };
        originalFetch = global.fetch;
    });

    afterEach(() => {
        global.fetch = originalFetch;
        cleanupTestEnvironment();
    });

    describe('AI-powered optimization', () => {
        it('should successfully use AI optimization when available', async () => {
            const thoughtChain = [
                { id: '1', type: 'start' },
                { id: '2', type: 'process', dependencies: ['1'] },
                { id: '3', type: 'end', dependencies: ['2'] }
            ];

            const mockAIOptimization = [
                { id: '1', type: 'start' },
                { id: '3', type: 'end', dependencies: ['1'] },
                { id: '2', type: 'process', dependencies: ['1', '3'] }
            ];

            global.fetch = jest.fn(() => Promise.resolve({
                ok: true,
                json: () => Promise.resolve(mockAIOptimization)
            }));

            const result = await optimizer.optimizeChain(thoughtChain);

            expect(global.fetch).toHaveBeenCalledWith(
                'https://api.example.com/optimize',
                expect.objectContaining({
                    method: 'POST',
                    headers: {
                        'Authorization': 'Bearer test-api-key',
                        'Content-Type': 'application/json'
                    },
                    body: expect.any(String)
                })
            );

            expect(result).toBeDefined();
            expect(result).toHaveLength(3);
            expect(result[0].id).toBe('1');
        });

        it('should fall back to baseline optimization when AI fails', async () => {
            const thoughtChain = [
                { id: '1', type: 'start' },
                { id: '2', type: 'process', dependencies: ['1'] }
            ];

            global.fetch = jest.fn(() => Promise.reject(new Error('API Error')));

            const result = await optimizer.optimizeChain(thoughtChain);

            expect(global.fetch).toHaveBeenCalled();
            expect(result).toBeDefined();
            expect(result).toHaveLength(2);
            expect(result[0].id).toBe('1');
            expect(result[1].id).toBe('2');
        });

        it('should validate AI-optimized chain before using it', async () => {
            const thoughtChain = [
                { id: '1', type: 'start' },
                { id: '2', type: 'process', dependencies: ['1'] }
            ];

            const invalidAIOptimization = [
                { id: '1', type: 'start' },
                { id: '3', type: 'invalid', dependencies: ['1'] } // Invalid thought ID
            ];

            global.fetch = jest.fn(() => Promise.resolve({
                ok: true,
                json: () => Promise.resolve(invalidAIOptimization)
            }));

            const result = await optimizer.optimizeChain(thoughtChain);

            // Should fall back to baseline optimization
            expect(result).toBeDefined();
            expect(result).toHaveLength(2);
            expect(result[0].id).toBe('1');
            expect(result[1].id).toBe('2');
        });

        it('should merge AI and baseline optimizations effectively', async () => {
            const thoughtChain = [
                { id: '1', type: 'start' },
                { id: '2', type: 'process', dependencies: ['1'] },
                { id: '3', type: 'end', dependencies: ['2'] }
            ];

            const mockAIOptimization = [
                { id: '1', type: 'start' },
                { id: '2', type: 'process', dependencies: ['1'] },
                { id: '3', type: 'end', dependencies: ['1', '2'] }
            ];

            global.fetch = jest.fn(() => Promise.resolve({
                ok: true,
                json: () => Promise.resolve(mockAIOptimization)
            }));

            mockPerformanceTracker.getMetrics.mockImplementation((thoughtId) => ({
                executionTime: thoughtId === '3' ? 50 : 100,
                memoryUsage: 50,
                cpuUsage: 0.5
            }));

            const result = await optimizer.optimizeChain(thoughtChain);

            expect(result).toBeDefined();
            expect(result).toHaveLength(3);
            // Verify the optimization preserved valid dependencies
            const finalThought = result.find(t => t.id === '3');
            expect(finalThought.dependencies).toContain('2');
        });

        it('should handle rate limiting and retry logic', async () => {
            const thoughtChain = [
                { id: '1', type: 'start' },
                { id: '2', type: 'end', dependencies: ['1'] }
            ];

            let attempts = 0;
            global.fetch = jest.fn(() => {
                attempts++;
                if (attempts === 1) {
                    return Promise.reject(new Error('Rate limited'));
                }
                return Promise.resolve({
                    ok: true,
                    json: () => Promise.resolve(thoughtChain)
                });
            });

            const result = await optimizer.optimizeChain(thoughtChain);

            expect(global.fetch).toHaveBeenCalledTimes(2);
            expect(result).toBeDefined();
            expect(result).toHaveLength(2);
        });
    });

    describe('AI Configuration', () => {
        it('should skip AI optimization when no AI config is provided', async () => {
            optimizer.aiConfig = null;
            const thoughtChain = [
                { id: '1', type: 'start' },
                { id: '2', type: 'end', dependencies: ['1'] }
            ];

            const result = await optimizer.optimizeChain(thoughtChain);

            expect(global.fetch).not.toHaveBeenCalled();
            expect(result).toBeDefined();
            expect(result).toHaveLength(2);
        });

        it('should handle different AI optimization models', async () => {
            const thoughtChain = [
                { id: '1', type: 'start' },
                { id: '2', type: 'end', dependencies: ['1'] }
            ];

            optimizer.aiConfig.optimizerModel = 'advanced-model';

            global.fetch = jest.fn(() => Promise.resolve({
                ok: true,
                json: () => Promise.resolve(thoughtChain)
            }));

            await optimizer.optimizeChain(thoughtChain);

            expect(global.fetch).toHaveBeenCalledWith(
                expect.any(String),
                expect.objectContaining({
                    body: expect.stringContaining('advanced-model')
                })
            );
        });
    });
});
--- End of tests/chain/optimizer.test.js ---



=== Folder: tests/concurrency ===


--- File: tests/concurrency/asyncLock.test.js ---

import { describe, it, expect, beforeEach, jest } from '@jest/globals';
import { AsyncLock } from '../../modules/concurrency/asyncLock';
import { setupTestEnvironment, cleanupTestEnvironment, delay } from '../test-setup';


describe('AsyncLock', () => {
    let lock
    beforeEach(() => {
        setupTestEnvironment();
        lock = new AsyncLock();
    });

    afterEach(async () => {
        if (lock) {
            await lock.releaseAll();
        }
        cleanupTestEnvironment();
    });

    describe('acquire', () => {
        it('should execute function with lock', async () => {
            const result = await lock.acquire('testKey', () => 'test');
            expect(result).toBe('test');
        });

        it('should execute functions sequentially for same key', async () => {
            const sequence = [];
            const fn1 = async () => {
                await new Promise(resolve => setTimeout(resolve, 50));
                sequence.push(1);
                return 1;
            };
            const fn2 = async () => {
                sequence.push(2);
                return 2;
            };

            const [result1, result2] = await Promise.all([
                lock.acquire('testKey', fn1),
                lock.acquire('testKey', fn2)
            ]);

            expect(sequence).toEqual([1, 2]);
            expect(result1).toBe(1);
            expect(result2).toBe(2);
        });

        it('should allow parallel execution for different keys', async () => {
            const sequence = [];
            const fn1 = async () => {
                await new Promise(resolve => setTimeout(resolve, 50));
                sequence.push(1);
                return 1;
            };
            const fn2 = async () => {
                sequence.push(2);
                return 2;
            };

            const [result1, result2] = await Promise.all([
                lock.acquire('key1', fn1),
                lock.acquire('key2', fn2)
            ]);

            expect(sequence).toEqual([2, 1]);
            expect(result1).toBe(1);
            expect(result2).toBe(2);
        });

        it('should timeout if lock cannot be acquired', async () => {
            const longRunning = () => new Promise(resolve => setTimeout(resolve, 1000));
            const quick = () => 'quick';

            // Start long-running task
            const longPromise = lock.acquire('testKey', longRunning);

            // Try to acquire same lock with short timeout
            await expect(
                lock.acquire('testKey', quick, 100)
            ).rejects.toThrow('Lock acquisition timed out');

            await longPromise;
        });

        it('should handle errors in locked function', async () => {
            const errorFn = () => {
                throw new Error('Test error');
            };

            await expect(
                lock.acquire('testKey', errorFn)
            ).rejects.toThrow('Test error');

            // Lock should be released after error
            const result = await lock.acquire('testKey', () => 'test');
            expect(result).toBe('test');
        });
    });

    describe('isLocked', () => {
        it('should return true when key is locked', async () => {
            const promise = lock.acquire('testKey', async () => {
                expect(lock.isLocked('testKey')).toBe(true);
                await new Promise(resolve => setTimeout(resolve, 50));
            });

            await new Promise(resolve => setTimeout(resolve, 10));
            expect(lock.isLocked('testKey')).toBe(true);
            await promise;
        });

        it('should return false when key is not locked', () => {
            expect(lock.isLocked('testKey')).toBe(false);
        });
    });

    describe('getWaitingCount', () => {
        it('should return correct number of waiting operations', async () => {
            const longRunning = () => new Promise(resolve => setTimeout(resolve, 100));
            
            // Start long-running task
            const promise1 = lock.acquire('testKey', longRunning);
            
            // Queue up more tasks
            const promise2 = lock.acquire('testKey', () => 'test2');
            const promise3 = lock.acquire('testKey', () => 'test3');

            await new Promise(resolve => setTimeout(resolve, 10));
            expect(lock.getWaitingCount('testKey')).toBe(2);

            await Promise.all([promise1, promise2, promise3]);
            expect(lock.getWaitingCount('testKey')).toBe(0);
        });
    });

    describe('isBusy', () => {
        it('should return true when there are active or waiting operations', async () => {
            const promise = lock.acquire('testKey', async () => {
                expect(await lock.isBusy()).toBe(true);
                await new Promise(resolve => setTimeout(resolve, 50));
            });

            await new Promise(resolve => setTimeout(resolve, 10));
            expect(await lock.isBusy()).toBe(true);
            await promise;
            expect(await lock.isBusy()).toBe(false);
        });
    });

    describe('getMetrics', () => {
        it('should track acquisition metrics', async () => {
            await lock.acquire('testKey', () => 'test1');
            await lock.acquire('testKey', () => 'test2');

            const metrics = lock.getMetrics();
            expect(metrics.acquireCount).toBe(2);
            expect(metrics.timeouts).toBe(0);
        });

        it('should track timeout metrics', async () => {
            const longRunning = () => new Promise(resolve => setTimeout(resolve, 1000));
            
            // Start long-running task
            const promise1 = lock.acquire('testKey', longRunning);
            
            // Try to acquire with timeout
            try {
                await lock.acquire('testKey', () => 'test', 100);
            } catch (error) {
                // Expected timeout
            }

            await promise1;
            const metrics = lock.getMetrics();
            expect(metrics.timeouts).toBe(1);
        });
    });

    describe('releaseAll', () => {
        it('should release all locks', async () => {
            // Create multiple locks
            const promise1 = lock.acquire('key1', () => 
                new Promise(resolve => setTimeout(resolve, 1000)));
            const promise2 = lock.acquire('key2', () => 
                new Promise(resolve => setTimeout(resolve, 1000)));

            await new Promise(resolve => setTimeout(resolve, 10));
            await lock.releaseAll();

            expect(lock.isLocked('key1')).toBe(false);
            expect(lock.isLocked('key2')).toBe(false);

            // Cleanup pending promises
            await Promise.allSettled([promise1, promise2]);
        });
    });

    describe('acquireMultiple', () => {
        it('should acquire multiple locks in order', async () => {
            const sequence = [];
            await lock.acquireMultiple(
                ['key1', 'key2', 'key3'],
                async () => {
                    sequence.push('executed');
                    expect(lock.isLocked('key1')).toBe(true);
                    expect(lock.isLocked('key2')).toBe(true);
                    expect(lock.isLocked('key3')).toBe(true);
                }
            );
            expect(sequence).toEqual(['executed']);
        });

        it('should prevent deadlocks by acquiring in sorted order', async () => {
            const sequence = [];
            await Promise.all([
                lock.acquireMultiple(['key2', 'key1'], async () => {
                    sequence.push(1);
                }),
                lock.acquireMultiple(['key1', 'key2'], async () => {
                    sequence.push(2);
                })
            ]);
            expect(sequence.length).toBe(2);
        });
    });

    describe('tryAcquire', () => {
        it('should acquire lock if available', async () => {
            const result = await lock.tryAcquire('testKey', () => 'test');
            expect(result).toBe('test');
        });

        it('should return null if lock is not available', async () => {
            const longRunning = () => new Promise(resolve => setTimeout(resolve, 100));
            const promise1 = lock.acquire('testKey', longRunning);

            const result = await lock.tryAcquire('testKey', () => 'test');
            expect(result).toBeNull();

            await promise1;
        });
    });
});
--- End of tests/concurrency/asyncLock.test.js ---



=== Folder: tests/debug ===


--- File: tests/debug/debugSystem.test.js ---

import { EnhancedDebugSystem } from '../../modules/debug/debugSystem';
import { setupTestEnvironment, cleanupTestEnvironment, delay } from '../test-setup';

describe('EnhancedDebugSystem', () => {
    let debugSystem;
    
    beforeEach(() => {
        setupTestEnvironment();
        debugSystem = new EnhancedDebugSystem({
            alertThresholds: {
                memory: 0.8,
                cpu: 0.9,
                time: 5000,
                errors: 10
            }
        });
    });

    afterEach(async () => {
        if (debugSystem) {
            await debugSystem.destroy();
        }
        cleanupTestEnvironment();
    });

    describe('System Health Monitoring', () => {
        it('should detect memory alerts', async () => {
            const alerts = [];
            debugSystem.registerDebugHook('MemoryAlert', (alert) => {
                alerts.push(alert);
            });

            // Mock metrics collection to trigger memory alert
            jest.spyOn(debugSystem.metrics, 'collect').mockResolvedValue({
                memory: {
                    heapUsedPercentage: 0.85 // Above threshold
                },
                cpu: {
                    percentage: 0.5
                }
            });

            await debugSystem.checkSystemHealth();
            expect(alerts.length).toBe(1);
            expect(alerts[0].type).toBe('MemoryAlert');
            expect(alerts[0].value).toBe(0.85);
        });

        it('should detect CPU alerts', async () => {
            const alerts = [];
            debugSystem.registerDebugHook('CPUAlert', (alert) => {
                alerts.push(alert);
            });

            // Mock metrics collection to trigger CPU alert
            jest.spyOn(debugSystem.metrics, 'collect').mockResolvedValue({
                memory: {
                    heapUsedPercentage: 0.5
                },
                cpu: {
                    percentage: 0.95 // Above threshold
                }
            });

            await debugSystem.checkSystemHealth();
            expect(alerts.length).toBe(1);
            expect(alerts[0].type).toBe('CPUAlert');
            expect(alerts[0].value).toBe(0.95);
        });
    });

    describe('Logging System', () => {
        it('should properly log messages with different levels', () => {
            debugSystem.log('info', 'Test info message');
            debugSystem.log('error', 'Test error message', { error: 'details' });
            debugSystem.log('warn', 'Test warning message');

            const logs = Array.from(debugSystem.logs.values());
            expect(logs.length).toBe(3);
            expect(logs.find(log => log.level === 'error').context.error).toBe('details');
        });

        it('should maintain log history within limits', () => {
            // Add more than 1000 logs
            for (let i = 0; i < 1100; i++) {
                debugSystem.log('info', `Log message ${i}`);
            }

            expect(debugSystem.logs.size).toBeLessThanOrEqual(1000);
        });

        it('should retrieve logs by level', () => {
            debugSystem.log('error', 'Error 1');
            debugSystem.log('info', 'Info 1');
            debugSystem.log('error', 'Error 2');

            const errorLogs = debugSystem.getLogsByLevel('error');
            expect(errorLogs.length).toBe(2);
            expect(errorLogs[0].message).toBe('Error 1');
            expect(errorLogs[1].message).toBe('Error 2');
        });
    });

    describe('Thought Inspection', () => {
        it('should inspect thought execution details', async () => {
            const thought = {
                id: 'test-thought',
                type: 'test',
                dependencies: ['dep1', 'dep2'],
                execute: async () => 'result'
            };

            const inspection = await debugSystem.inspectThought(thought);
            
            expect(inspection.id).toBe('test-thought');
            expect(inspection.type).toBe('test');
            expect(inspection.traces).toBeDefined();
            expect(inspection.executionGraph).toBeDefined();
            expect(inspection.dependencies).toBeDefined();
        });

        it('should track function calls during thought execution', async () => {
            const thought = {
                id: 'test-thought',
                testFunction: async (arg) => `processed ${arg}`,
                execute: async () => {
                    await thought.testFunction('input');
                    return 'done';
                }
            };

            const traces = await debugSystem.collectTraces(thought);
            expect(traces.functionCalls.length).toBeGreaterThan(0);
        });

        it('should generate execution graph', () => {
            const thought = {
                id: 'root',
                type: 'test',
                children: [
                    { id: 'child1', type: 'test' },
                    { id: 'child2', type: 'test' }
                ]
            };

            const graph = debugSystem.generateExecutionGraph(thought);
            expect(graph.nodes.length).toBe(3);
            expect(graph.edges.length).toBe(2);
        });
    });

    describe('Resource Tracking', () => {
        it('should track memory access patterns', () => {
            const thought = {
                id: 'test-thought',
                execute: () => {
                    // Simulate memory operations
                    const array = new Array(1000);
                    return array;
                }
            };

            const memoryAccess = debugSystem.getMemoryAccess(thought);
            expect(memoryAccess.allocations).toBeDefined();
            expect(memoryAccess.reads).toBeDefined();
            expect(memoryAccess.writes).toBeDefined();
        });

        it('should track network access', () => {
            const thought = {
                id: 'test-thought',
                execute: async () => {
                    // Simulate network call
                    await fetch('https://api.example.com');
                    return 'done';
                }
            };

            const networkAccess = debugSystem.getNetworkAccess(thought);
            expect(networkAccess.requests).toBeDefined();
            expect(networkAccess.responses).toBeDefined();
            expect(networkAccess.errors).toBeDefined();
        });
    });

    describe('Debug Hooks', () => {
        it('should register and execute debug hooks', async () => {
            const hookExecuted = jest.fn();
            debugSystem.registerDebugHook('TestEvent', hookExecuted);

            const alert = {
                type: 'TestEvent',
                value: 0.9,
                timestamp: Date.now()
            };

            debugSystem.handleAlerts([alert]);
            expect(hookExecuted).toHaveBeenCalledWith(alert);
        });

        it('should handle cleanup errors gracefully', async () => {
            jest.spyOn(debugSystem, 'clearLogs').mockImplementation(() => {
                throw new Error('Cleanup error');
            });
        
            const logSpy = jest.spyOn(console, 'error');
            await debugSystem.destroy();
        
            expect(logSpy).toHaveBeenCalledWith('Error clearing logs:', expect.any(Error));
            expect(debugSystem.monitoringIntervalId).toBeNull();
            expect(debugSystem.breakpoints.size).toBe(0);
            expect(debugSystem.errorCount.size).toBe(0);
            expect(debugSystem.debugHooks.size).toBe(0);
        });
    });

    describe('System Cleanup', () => {
        it('should properly clean up resources on destroy', async () => {
            const clearLogsSpy = jest.spyOn(debugSystem, 'clearLogs');
            await debugSystem.destroy();

            expect(clearLogsSpy).toHaveBeenCalled();
            expect(debugSystem.breakpoints.size).toBe(0);
            expect(debugSystem.errorCount.size).toBe(0);
            expect(debugSystem.debugHooks.size).toBe(0);
            expect(debugSystem.monitoringIntervalId).toBeNull();
        });

        it('should handle cleanup errors gracefully', async () => {
            jest.spyOn(debugSystem, 'clearLogs').mockImplementation(() => {
                throw new Error('Cleanup error');
            });

            const logSpy = jest.spyOn(console, 'error');
            await debugSystem.destroy();

            expect(logSpy).toHaveBeenCalled();
        });
    });

    describe('Debug Snapshot', () => {
        it('should create accurate system snapshot', () => {
            // Add some test data
            debugSystem.log('info', 'Test log');
            debugSystem.breakpoints.add('test-breakpoint');
            debugSystem.errorCount.set('test-error', { count: 1, timestamp: Date.now() });

            const snapshot = debugSystem.getDebugSnapshot();
            
            expect(snapshot.timestamp).toBeDefined();
            expect(snapshot.logs.length).toBeGreaterThan(0);
            expect(snapshot.breakpoints).toContain('test-breakpoint');
            expect(snapshot.errorCounts.length).toBeGreaterThan(0);
            expect(snapshot.metrics).toBeDefined();
        });
    });
});
--- End of tests/debug/debugSystem.test.js ---



=== Folder: tests/storage ===


--- File: tests/storage/hybridstorage.test.js ---

import { HybridStorage } from '../../modules/storage/hybridStorage';
import { CompressionUtil } from '../../modules/storage/utils/compression';
import { StorageMetrics } from '../../modules/storage/utils/metrics';
import { setupTestEnvironment, cleanupTestEnvironment, delay } from '../test-setup';

describe('HybridStorage', () => {
    let storage;

    beforeEach(() => {
        setupTestEnvironment();
        storage = new HybridStorage({
            dbName: 'test-storage',
            maxMemoryItems: 100
        });
    });

    afterEach(async () => {
        if (storage) {
            await storage.destroy();
        }
        cleanupTestEnvironment();
    });

    describe('Basic Operations', () => {
        it('should store and retrieve data', async () => {
            await storage.set('key1', { test: 'data' });
            const result = await storage.get('key1');
            expect(result).toEqual({ test: 'data' });
        });

        it('should handle non-existent keys', async () => {
            const result = await storage.get('nonexistent');
            expect(result).toBeNull();
        });

        it('should delete data', async () => {
            await storage.set('key1', { test: 'data' });
            await storage.delete('key1');
            const result = await storage.get('key1');
            expect(result).toBeNull();
        });

        it('should check key existence', async () => {
            await storage.set('key1', { test: 'data' });
            expect(await storage.has('key1')).toBe(true);
            expect(await storage.has('nonexistent')).toBe(false);
        });

        it('should clear all data', async () => {
            await storage.set('key1', { test: 'data1' });
            await storage.set('key2', { test: 'data2' });
            await storage.clear();
            expect(await storage.get('key1')).toBeNull();
            expect(await storage.get('key2')).toBeNull();
        });
    });

    describe('Memory Management', () => {
        it('should respect memory limits', async () => {
            // Add more items than maxMemoryItems
            for (let i = 0; i < 150; i++) {
                await storage.set(`key${i}`, { data: `value${i}` });
            }

            // Check that memory store size doesn't exceed limit
            expect(storage.memoryStore.size).toBeLessThanOrEqual(100);
        });

        it('should handle large data sets', async () => {
            const largeData = new Array(1000).fill('test').join('');
            await storage.set('largeKey', largeData);
            const result = await storage.get('largeKey');
            expect(result).toBe(largeData);
        });

        it('should prioritize frequently accessed items in memory', async () => {
            // Add multiple items
            for (let i = 0; i < 50; i++) {
                await storage.set(`key${i}`, { data: `value${i}` });
            }

            // Frequently access one item
            for (let i = 0; i < 10; i++) {
                await storage.get('key1');
            }

            // Add more items to trigger memory limit
            for (let i = 50; i < 150; i++) {
                await storage.set(`key${i}`, { data: `value${i}` });
            }

            // Frequently accessed item should still be in memory
            expect(storage.memoryStore.has('key1')).toBe(true);
        });
    });

    describe('Compression', () => {
        it('should compress data when specified', async () => {
            const originalData = { test: 'data'.repeat(100) };
            await storage.set('key1', originalData, { compression: true });
            
            // Get compressed data directly from store
            const stored = storage.memoryStore.get('key1');
            expect(stored.compressed).toBe(true);
            
            // Retrieved data should be decompressed
            const retrieved = await storage.get('key1');
            expect(retrieved).toEqual(originalData);
        });

        it('should handle compression errors gracefully', async () => {
            // Mock compression failure
            jest.spyOn(CompressionUtil, 'compress').mockRejectedValue(new Error('Compression failed'));
            
            const data = { test: 'data' };
            await expect(storage.set('key1', data, { compression: true }))
                .rejects.toThrow();
        });
    });

    describe('Query Operations', () => {
        it('should query by tags', async () => {
            await storage.set('key1', { data: 'value1' }, { tags: ['tag1'] });
            await storage.set('key2', { data: 'value2' }, { tags: ['tag1', 'tag2'] });
            await storage.set('key3', { data: 'value3' }, { tags: ['tag2'] });

            const results = await storage.query({ tags: ['tag1'] });
            expect(results.items.size).toBe(2);
            expect(results.totalCount).toBe(2);
        });

        it('should query by priority', async () => {
            await storage.set('key1', { data: 'value1' }, { priority: 1 });
            await storage.set('key2', { data: 'value2' }, { priority: 2 });
            await storage.set('key3', { data: 'value3' }, { priority: 0 });

            const results = await storage.query({ minPriority: 1 });
            expect(results.items.size).toBe(2);
        });

        it('should handle complex queries', async () => {
            await storage.set('key1', { data: 'value1' }, { 
                tags: ['tag1'], 
                priority: 1 
            });
            await storage.set('key2', { data: 'value2' }, { 
                tags: ['tag1', 'tag2'], 
                priority: 2 
            });

            const results = await storage.query({ 
                tags: ['tag1'],
                minPriority: 2
            });
            expect(results.items.size).toBe(1);
        });
    });

    describe('Optimization & Maintenance', () => {
        it('should vacuum expired items', async () => {
            await storage.set('key1', { data: 'value1' }, { expiry: 100 }); // Expire after 100ms
            await storage.set('key2', { data: 'value2' }); // No expiry

            await delay(150); // Wait for expiration
            const expiredCount = await storage.vacuum();
            
            expect(expiredCount).toBe(1);
            expect(await storage.get('key1')).toBeNull();
            expect(await storage.get('key2')).not.toBeNull();
        });

        it('should optimize storage based on access patterns', async () => {
            // Add items with different access patterns
            for (let i = 0; i < 50; i++) {
                await storage.set(`key${i}`, { data: `value${i}` });
                if (i < 10) {
                    // Frequently access some items
                    for (let j = 0; j < 5; j++) {
                        await storage.get(`key${i}`);
                    }
                }
            }

            await storage.optimize();
            const metrics = await storage.getUsageMetrics();
            expect(metrics.hitRate).toBeGreaterThan(0);
        });

        it('should compact database when fragmentation is high', async () => {
            // Create fragmentation by adding and deleting items
            for (let i = 0; i < 100; i++) {
                await storage.set(`key${i}`, { data: `value${i}` });
            }
            for (let i = 0; i < 50; i++) {
                await storage.delete(`key${i}`);
            }

            const beforeSize = (await storage.getStorageStats()).totalSize;
            await storage.compactDB();
            const afterSize = (await storage.getStorageStats()).totalSize;
            
            expect(afterSize).toBeLessThan(beforeSize);
        });
    });

    describe('Error Handling', () => {
        it('should handle database connection errors', async () => {
            // Instead of closing the db, just set it to null to simulate connection error
            storage.db = null;

            await expect(storage.set('key1', { test: 'data' }))
                .rejects.toThrow('Database connection not established');
        });

        it('should handle concurrent operations', async () => {
            const operations = Array(10).fill(null).map((_, i) => 
                storage.set(`key${i}`, { data: `value${i}` })
            );

            await expect(Promise.all(operations)).resolves.not.toThrow();
        });

        it('should handle transaction failures', async () => {
            // Ensure db is not null here
            expect(storage.db).not.toBeNull();

            const mockTransaction = {
                objectStore: () => ({
                    put: () => ({
                        onerror: () => {},
                        onsuccess: () => {}
                    })
                }),
                onerror: () => {}
            };
            jest.spyOn(storage.db, 'transaction').mockReturnValue(mockTransaction);

            await expect(storage.set('key1', { test: 'data' }))
                .rejects.toThrow();
        });
    });

    describe('Metrics & Monitoring', () => {
        it('should track operation metrics', async () => {
            await storage.set('key1', { test: 'data' });
            await storage.get('key1');
            await storage.get('nonexistent');

            const metrics = storage.metrics;
            expect(metrics.hits).toBe(1);
            expect(metrics.misses).toBe(1);
            expect(metrics.writes).toBe(1);
        });

        it('should track access times', async () => {
            await storage.set('key1', { test: 'data' });
            await storage.get('key1');

            expect(storage.metrics.accessTimes.length).toBeGreaterThan(0);
        });

        it('should calculate storage statistics', async () => {
            await storage.set('key1', { test: 'data1' });
            await storage.set('key2', { test: 'data2' });

            const stats = await storage.getStorageStats();
            expect(stats.totalSize).toBeGreaterThan(0);
            expect(stats.usedSize).toBeGreaterThan(0);
        });
    });
});

describe('CompressionUtil', () => {
    beforeEach(setupTestEnvironment);
    afterEach(() => {
        cleanupTestEnvironment();
        jest.restoreAllMocks();
    });

    it('should compress and decompress data correctly', async () => {
        const originalData = 'test'.repeat(100);
        const compressed = await CompressionUtil.compress(originalData);
        const decompressed = await CompressionUtil.decompress(compressed);
        expect(decompressed).toBe(originalData);
    });

    it('should handle different data types', async () => {
        const testCases = [
            { test: 'string' },
            [1, 2, 3],
            123,
            true,
            { nested: { data: { here: true } } }
        ];

        for (const testCase of testCases) {
            const compressed = await CompressionUtil.compress(testCase);
            const decompressed = await CompressionUtil.decompress(compressed);
            expect(decompressed).toEqual(testCase);
        }
    });

    it('should handle empty input', async () => {
        const compressed = await CompressionUtil.compress('');
        const decompressed = await CompressionUtil.decompress(compressed);
        expect(decompressed).toBe('');
    });
});

describe('StorageMetrics', () => {
    let metrics;

    beforeEach(() => {
        setupTestEnvironment();
        metrics = new StorageMetrics();
    });

    afterEach(cleanupTestEnvironment);

    it('should track basic metrics', () => {
        metrics.recordHit();
        metrics.recordMiss();
        metrics.recordWrite();
        
        expect(metrics.metrics.hits).toBe(1);
        expect(metrics.metrics.misses).toBe(1);
        expect(metrics.metrics.writes).toBe(1);
    });

    it('should calculate hit rate correctly', () => {
        metrics.recordHit();
        metrics.recordHit();
        metrics.recordMiss();
        
        expect(metrics.getHitRate()).toBe(2/3);
    });

    it('should analyze access patterns', () => {
        // Record some access patterns
        metrics.recordAccessTime(100);
        metrics.recordAccessTime(150);
        metrics.recordHit();
        metrics.recordWrite();

        const analysis = metrics.analyzeAccessPatterns();
        expect(analysis.averageAccessTime).toBeGreaterThan(0);
        expect(analysis.hitRate).toBeDefined();
        expect(analysis.writeRate).toBeDefined();
    });

    it('should reset metrics correctly', () => {
        metrics.recordHit();
        metrics.recordWrite();
        metrics.reset();

        expect(metrics.metrics.hits).toBe(0);
        expect(metrics.metrics.writes).toBe(0);
    });
});

--- End of tests/storage/hybridstorage.test.js ---



=== File: package.json ===

{
  "name": "thought-system",
  "version": "1.0.0",
  "description": "Enhanced thought processing system",
  "main": "index.js",
  "scripts": {
    "test": "NODE_OPTIONS=--experimental-vm-modules jest --config jest.config.js",
    "test:watch": "NODE_OPTIONS=--experimental-vm-modules jest --config jest.config.js --watch",
    "test:coverage": "NODE_OPTIONS=--experimental-vm-modules jest --config jest.config.js --coverage"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "devDependencies": {
    "@babel/core": "^7.24.0",
    "@babel/plugin-transform-modules-commonjs": "^7.24.0",
    "@babel/preset-env": "^7.24.0",
    "babel-jest": "^29.7.0",
    "jest": "^29.7.0",
    "jest-environment-node": "^29.7.0"
  },
  "dependencies": {
    "jest-mock": "^29.7.0"
  }
}
--- End of package.json ---



=== File: jest.config.js ===

// jest.config.cjs
module.exports = {
    testEnvironment: 'node',
    transform: {
        '^.+\\.js$': ['babel-jest', {
            presets: [
                ['@babel/preset-env', {
                    targets: {
                        node: 'current'
                    }
                }]
            ]
        }]
    },
    moduleNameMapper: {
        '^(\\.{1,2}/.*)\\.js$': '$1'
    },
    // setupFilesAfterEnv: ['./jest.setup.cjs'],
    testMatch: ['**/tests/**/*.test.js'],
    verbose: true,
    collectCoverage: true,
    coverageDirectory: 'coverage',
    coverageReporters: ['text', 'lcov'],
    transformIgnorePatterns: [],
    testEnvironmentOptions: {
        url: 'http://localhost'
    },
    // extensionsToTreatAsEsm: ['.js']
}
--- End of jest.config.js ---



=== File: jest.setup.js ===

const { setupTestEnvironment } = require('./tests/test-setup');

// Set longer timeout for tests
jest.setTimeout(10000);

// Setup test environment before all tests
beforeAll(() => {
    setupTestEnvironment();
});

// Clear mocks between each test
beforeEach(() => {
    jest.clearAllMocks();
});
--- End of jest.setup.js ---