From 47e48640499cf80e6d17bb0b9ef6da58babd9bd5 Mon Sep 17 00:00:00 2001 From: Mai Development Date: Wed, 28 Jan 2026 12:08:47 -0500 Subject: [PATCH] docs(04): create gap closure plans for memory and context management Phase 04: Memory & Context Management - 3 gap closure plans to address verification issues - 04-05: Personality learning integration (PersonalityAdaptation, MemoryManager integration, src/personality.py) - 04-06: Vector Store missing methods (search_by_keyword, store_embeddings) - 04-07: Context-aware search metadata integration (get_conversation_metadata) - All gaps from verification report addressed - Updated roadmap to reflect 7 total plans --- .planning/ROADMAP.md | 6 +- .../04-05-PLAN.md | 211 ++++++++++++++++++ .../04-06-PLAN.md | 161 +++++++++++++ .../04-07-PLAN.md | 159 +++++++++++++ 4 files changed, 536 insertions(+), 1 deletion(-) create mode 100644 .planning/phases/04-memory-context-management/04-05-PLAN.md create mode 100644 .planning/phases/04-memory-context-management/04-06-PLAN.md create mode 100644 .planning/phases/04-memory-context-management/04-07-PLAN.md diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index b6e754b..2fcca3e 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -51,11 +51,15 @@ Mai's development is organized into three major milestones, each delivering dist - Distill long-term patterns into personality layers - Proactively surface relevant context from memory -**Plans:** 4 plans in 3 waves +**Status:** 3 gap closure plans needed to complete integration +**Plans:** 7 plans in 4 waves - [x] 04-01-PLAN.md — Storage foundation with SQLite and sqlite-vec - [x] 04-02-PLAN.md — Semantic search and context-aware retrieval - [x] 04-03-PLAN.md — Progressive compression and JSON archival - [x] 04-04-PLAN.md — Personality learning and adaptive layers +- [ ] 04-05-PLAN.md — Personality learning integration gap closure +- [ ] 04-06-PLAN.md — Vector Store missing methods gap closure +- [ ] 04-07-PLAN.md — Context-aware search metadata gap closure ### Phase 5: Conversation Engine - Multi-turn context preservation diff --git a/.planning/phases/04-memory-context-management/04-05-PLAN.md b/.planning/phases/04-memory-context-management/04-05-PLAN.md new file mode 100644 index 0000000..d6e52e7 --- /dev/null +++ b/.planning/phases/04-memory-context-management/04-05-PLAN.md @@ -0,0 +1,211 @@ +--- +phase: 04-memory-context-management +plan: 05 +type: execute +wave: 1 +depends_on: ["04-04"] +files_modified: ["src/memory/personality/adaptation.py", "src/memory/__init__.py", "src/personality.py"] +autonomous: true +gap_closure: true + +must_haves: + truths: + - "Personality layers learn from conversation patterns" + - "Personality system integrates with existing personality.py" + artifacts: + - path: "src/memory/personality/adaptation.py" + provides: "Dynamic personality updates" + min_lines: 50 + - path: "src/memory/__init__.py" + provides: "Complete MemoryManager with personality learning" + exports: ["PersonalityLearner"] + - path: "src/personality.py" + provides: "Updated personality system with memory integration" + min_lines: 20 + key_links: + - from: "src/memory/personality/adaptation.py" + to: "src/memory/personality/layer_manager.py" + via: "layer updates for adaptation" + pattern: "layer_manager\\.update_layer" + - from: "src/memory/__init__.py" + to: "src/memory/personality/adaptation.py" + via: "PersonalityLearner integration" + pattern: "PersonalityLearner.*update_personality" + - from: "src/personality.py" + to: "src/memory/personality/layer_manager.py" + via: "personality overlay application" + pattern: "layer_manager\\.get_active_layers" +--- + + +Complete personality learning integration by implementing missing PersonalityAdaptation class and connecting all personality learning components to the MemoryManager and existing personality system. + +Purpose: Close the personality learning integration gap identified in verification +Output: Working personality learning system fully integrated with memory and personality systems + + + +@~/.opencode/get-shit-done/workflows/execute-plan.md +@~/.opencode/get-shit-done/templates/summary.md + + + +@.planning/phases/04-memory-context-management/04-CONTEXT.md +@.planning/phases/04-memory-context-management/04-RESEARCH.md +@.planning/phases/04-memory-context-management/04-memory-context-management-VERIFICATION.md + +# Reference existing personality components +@src/memory/personality/pattern_extractor.py +@src/memory/personality/layer_manager.py +@src/resource/personality.py + +# Reference memory manager +@src/memory/__init__.py + + + + + + Task 1: Implement PersonalityAdaptation class + src/memory/personality/adaptation.py + +Create src/memory/personality/adaptation.py with PersonalityAdaptation class to close the missing file gap: + +1. PersonalityAdaptation class with time-weighted learning: + - update_personality_layer(patterns, layer_id, adaptation_rate) + - calculate_adaptation_rate(conversation_history, user_feedback) + - apply_stability_controls(proposed_changes, current_state) + - integrate_user_feedback(feed_data, layer_weights) + +2. Time-weighted learning implementation: + - Recent conversations have less influence (exponential decay) + - Historical patterns provide stable baseline + - Prevent rapid personality swings with rate limiting + - Confidence scoring for pattern reliability + +3. Stability controls: + - Maximum change per update (e.g., 10% weight shift) + - Cooling period between major adaptations + - Core value protection (certain aspects never change) + - Reversion triggers for unwanted changes + +4. Integration methods: + - import_pattern_data(pattern_extractor, conversation_range) + - export_layer_config(layer_manager, output_format) + - validate_layer_consistency(layers, core_personality) + +5. Configuration and persistence: + - Learning rate configuration (slow/medium/fast) + - Adaptation history tracking + - Rollback capability for problematic changes + - Integration with existing memory storage + +Follow existing error handling patterns from layer_manager.py. Use similar data structures and method signatures for consistency. + + python -c "from src.memory.personality.adaptation import PersonalityAdaptation; pa = PersonalityAdaptation(); print('PersonalityAdaptation created successfully')" + PersonalityAdaptation class provides time-weighted learning with stability controls + + + + Task 2: Integrate personality learning with MemoryManager + src/memory/__init__.py + +Update src/memory/__init__.py to integrate personality learning and export PersonalityLearner: + +1. Import PersonalityAdaptation in memory/personality/__init__.py: + - Add from .adaptation import PersonalityAdaptation + - Update __all__ to include PersonalityAdaptation + +2. Create PersonalityLearner class in MemoryManager: + - Combines PatternExtractor, LayerManager, and PersonalityAdaptation + - Methods: learn_from_conversations(conversation_range), apply_learning(), get_current_personality() + - Learning triggers: after conversations, periodic updates, manual requests + +3. Integration with existing MemoryManager: + - Add personality_learner attribute to MemoryManager.__init__ + - Implement learning_workflow() method for coordinated learning + - Add personality data persistence to existing storage + - Provide learning controls (enable/disable, rate, triggers) + +4. Export PersonalityLearner from memory/__init__.py: + - Add PersonalityLearner to __all__ + - Ensure it's importable as from src.memory import PersonalityLearner + +5. Learning workflow integration: + - Hook into conversation storage for automatic learning triggers + - Periodic learning schedule (e.g., daily pattern analysis) + - Integration with existing configuration system + - Memory usage monitoring for learning processes + +Update existing MemoryManager methods to support personality learning without breaking current functionality. Follow the existing pattern of having feature-specific managers within the main MemoryManager. + + python -c "from src.memory import PersonalityLearner; pl = PersonalityLearner(); print('PersonalityLearner imported successfully')" + PersonalityLearner is integrated with MemoryManager and available for import + + + + Task 3: Create src/personality.py with memory integration + src/personality.py + +Create src/personality.py to integrate with memory personality learning system: + +1. Core personality system: + - Import PersonalityLearner from memory system + - Maintain core personality values (immutable) + - Apply learned personality layers as overlays + - Protect core values from learned modifications + +2. Integration with existing personality: + - Import and extend src/resource/personality.py functionality + - Add memory integration to existing personality methods + - Hybrid system prompt + behavior configuration + - Context-aware personality layer activation + +3. Personality application methods: + - get_personality_response(context, user_input) -> enhanced_response + - apply_personality_layers(base_response, context) -> final_response + - get_active_layers(conversation_context) -> List[PersonalityLayer] + - validate_personality_consistency(applied_layers) -> bool + +4. Configuration and control: + - Learning enable/disable flag + - Layer activation rules + - Core value protection settings + - User feedback integration for personality tuning + +5. Integration points: + - Connect to MemoryManager.PersonalityLearner + - Use existing personality.py from src/resource as base + - Ensure compatibility with existing conversation systems + - Provide clear separation between core and learned personality + +Follow the pattern established in src/resource/personality.py but extend it with memory learning integration. Ensure core personality values remain protected while allowing learned layers to enhance responses. + + python -c "from src.personality import get_personality_response; print('Personality system integration working')" + src/personality.py integrates with memory learning while protecting core values + + + + + +After completion, verify: +1. PersonalityAdaptation class exists and implements time-weighted learning +2. PersonalityLearner is integrated into MemoryManager and exportable +3. src/personality.py exists and integrates with memory personality system +4. Personality learning workflow connects all components (PatternExtractor -> LayerManager -> PersonalityAdaptation) +5. Core personality values are protected from learned modifications +6. Learning system can be enabled/disabled through configuration + + + +- Personality learning integration gap is completely closed +- All personality components work together as a cohesive system +- Personality layers learn from conversation patterns over time +- Core personality values remain protected while allowing adaptive learning +- Integration follows existing patterns and maintains code consistency +- System is ready for testing and eventual user verification + + + +After completion, create `.planning/phases/04-memory-context-management/04-05-SUMMARY.md` + \ No newline at end of file diff --git a/.planning/phases/04-memory-context-management/04-06-PLAN.md b/.planning/phases/04-memory-context-management/04-06-PLAN.md new file mode 100644 index 0000000..7e0ce47 --- /dev/null +++ b/.planning/phases/04-memory-context-management/04-06-PLAN.md @@ -0,0 +1,161 @@ +--- +phase: 04-memory-context-management +plan: 06 +type: execute +wave: 1 +depends_on: ["04-01"] +files_modified: ["src/memory/storage/vector_store.py"] +autonomous: true +gap_closure: true + +must_haves: + truths: + - "User can search conversations by semantic meaning" + artifacts: + - path: "src/memory/storage/vector_store.py" + provides: "Vector storage and retrieval with sqlite-vec" + contains: "search_by_keyword method" + contains: "store_embeddings method" + key_links: + - from: "src/memory/retrieval/semantic_search.py" + to: "src/memory/storage/vector_store.py" + via: "vector similarity search operations" + pattern: "vector_store\\.search_by_keyword" + - from: "src/memory/retrieval/semantic_search.py" + to: "src/memory/storage/vector_store.py" + via: "embedding storage operations" + pattern: "vector_store\\.store_embeddings" +--- + + +Complete VectorStore implementation by adding missing search_by_keyword and store_embeddings methods that are called by SemanticSearch but not implemented. + +Purpose: Close the vector store methods gap to enable full semantic search functionality +Output: Complete VectorStore with all required methods for semantic search operations + + + +@~/.opencode/get-shit-done/workflows/execute-plan.md +@~/.opencode/get-shit-done/templates/summary.md + + + +@.planning/phases/04-memory-context-management/04-CONTEXT.md +@.planning/phases/04-memory-context-management/04-memory-context-management-VERIFICATION.md + +# Reference existing vector store implementation +@src/memory/storage/vector_store.py + +# Reference semantic search that calls these methods +@src/memory/retrieval/semantic_search.py + + + + + + Task 1: Implement search_by_keyword method in VectorStore + src/memory/storage/vector_store.py + +Add missing search_by_keyword method to VectorStore class to close the verification gap: + +1. search_by_keyword method implementation: + - search_by_keyword(self, query: str, limit: int = 10) -> List[Dict] + - Perform keyword-based search on message content using FTS if available + - Fall back to LIKE queries if FTS not enabled + - Return results in same format as vector search for consistency + +2. Keyword search implementation: + - Use SQLite FTS (Full-Text Search) if virtual tables exist + - Query message_content and conversation_summary fields + - Support multiple keywords with AND/OR logic + - Rank results by keyword frequency and position + +3. Integration with existing vector operations: + - Use same database connection as existing methods + - Follow existing error handling patterns + - Return results compatible with hybrid_search in SemanticSearch + - Include message_id, conversation_id, content, and relevance score + +4. Performance optimizations: + - Add appropriate indexes for keyword search if missing + - Use query parameters to prevent SQL injection + - Limit result sets for performance + - Cache frequent keyword queries if beneficial + +5. Method signature matching: + - Match the expected signature from semantic_search.py line 248 + - Return format: List[Dict] with message_id, conversation_id, content, score + - Handle edge cases: empty queries, no results, database errors + +The method should be called by SemanticSearch.hybrid_search at line 248. Verify the exact signature and return format by checking semantic_search.py before implementation. + + python -c "from src.memory.storage.vector_store import VectorStore; vs = VectorStore(); result = vs.search_by_keyword('test', limit=5); print(f'search_by_keyword returned {len(result)} results')" + VectorStore.search_by_keyword method provides keyword-based search functionality + + + + Task 2: Implement store_embeddings method in VectorStore + src/memory/storage/vector_store.py + +Add missing store_embeddings method to VectorStore class to close the verification gap: + +1. store_embeddings method implementation: + - store_embeddings(self, embeddings: List[Tuple[str, List[float]]]) -> bool + - Batch store multiple embeddings efficiently + - Handle conversation_id and message_id associations + - Return success/failure status + +2. Embedding storage implementation: + - Use existing vec_entries virtual table from current implementation + - Insert embeddings with proper rowid mapping to messages + - Support batch inserts for performance + - Handle embedding dimension validation + +3. Integration with existing storage patterns: + - Follow same database connection patterns as other methods + - Use existing error handling and transaction management + - Coordinate with sqlite_manager for message metadata + - Maintain consistency with existing vector storage + +4. Method signature compatibility: + - Match expected signature from semantic_search.py line 363 + - Accept list of (id, embedding) tuples + - Return boolean success indicator + - Handle partial failures gracefully + +5. Performance and reliability: + - Use transactions for batch operations + - Validate embedding dimensions before insertion + - Handle database constraint violations + - Provide detailed error logging for debugging + +The method should be called by SemanticSearch at line 363. Verify the exact signature and expected behavior by checking semantic_search.py before implementation. Ensure compatibility with the existing vec_entries table structure and sqlite-vec extension usage. + + python -c "from src.memory.storage.vector_store import VectorStore; import numpy as np; vs = VectorStore(); test_emb = [('test_id', np.random.rand(1536).tolist())]; result = vs.store_embeddings(test_emb); print(f'store_embeddings returned: {result}')" + VectorStore.store_embeddings method provides batch embedding storage functionality + + + + + +After completion, verify: +1. search_by_keyword method exists and is callable from SemanticSearch +2. store_embeddings method exists and is callable from SemanticSearch +3. Both methods follow the exact signatures expected by semantic_search.py +4. Methods integrate properly with existing VectorStore database operations +5. SemanticSearch.hybrid_search can now call these methods without errors +6. Keyword search returns properly formatted results compatible with vector search + + + +- VectorStore missing methods gap is completely closed +- SemanticSearch can perform hybrid search combining keyword and vector search +- Methods follow existing VectorStore patterns and error handling +- Database operations are efficient and properly transactional +- Integration with semantic search is seamless and functional +- All anti-patterns related to missing method calls are resolved + + + +After completion, create `.planning/phases/04-memory-context-management/04-06-SUMMARY.md` + \ No newline at end of file diff --git a/.planning/phases/04-memory-context-management/04-07-PLAN.md b/.planning/phases/04-memory-context-management/04-07-PLAN.md new file mode 100644 index 0000000..27a3774 --- /dev/null +++ b/.planning/phases/04-memory-context-management/04-07-PLAN.md @@ -0,0 +1,159 @@ +--- +phase: 04-memory-context-management +plan: 07 +type: execute +wave: 1 +depends_on: ["04-01"] +files_modified: ["src/memory/storage/sqlite_manager.py"] +autonomous: true +gap_closure: true + +must_haves: + truths: + - "Context-aware search prioritizes current topic discussions" + artifacts: + - path: "src/memory/storage/sqlite_manager.py" + provides: "SQLite database operations and schema management" + contains: "get_conversation_metadata method" + key_links: + - from: "src/memory/retrieval/context_aware.py" + to: "src/memory/storage/sqlite_manager.py" + via: "conversation metadata for topic analysis" + pattern: "sqlite_manager\\.get_conversation_metadata" +--- + + +Complete SQLiteManager by adding missing get_conversation_metadata method to enable ContextAwareSearch topic analysis functionality. + +Purpose: Close the metadata integration gap to enable context-aware search prioritization +Output: Complete SQLiteManager with metadata access for topic-based search enhancement + + + +@~/.opencode/get-shit-done/workflows/execute-plan.md +@~/.opencode/get-shit-done/templates/summary.md + + + +@.planning/phases/04-memory-context-management/04-CONTEXT.md +@.planning/phases/04-memory-context-management/04-memory-context-management-VERIFICATION.md + +# Reference existing sqlite manager implementation +@src/memory/storage/sqlite_manager.py + +# Reference context aware search that needs this method +@src/memory/retrieval/context_aware.py + + + + + + Task 1: Implement get_conversation_metadata method in SQLiteManager + src/memory/storage/sqlite_manager.py + +Add missing get_conversation_metadata method to SQLiteManager class to close the verification gap: + +1. get_conversation_metadata method implementation: + - get_conversation_metadata(self, conversation_ids: List[str]) -> Dict[str, Dict] + - Retrieve comprehensive metadata for specified conversations + - Include topics, timestamps, message counts, user engagement metrics + - Return structured data suitable for topic analysis + +2. Metadata fields to include: + - Conversation metadata: title, summary, created_at, updated_at + - Topic information: main_topics, topic_frequency, topic_sentiment + - Engagement metrics: message_count, user_message_ratio, response_times + - Temporal data: time_of_day patterns, day_of_week patterns + - Context clues: related_conversations, conversation_chain_position + +3. Database queries for metadata: + - Query conversations table for basic metadata + - Aggregate message data for engagement metrics + - Join with message metadata if available + - Calculate topic statistics from existing topic fields + - Use existing indexes for efficient querying + +4. Integration with existing SQLiteManager patterns: + - Follow same connection and cursor management + - Use existing error handling and transaction patterns + - Return data in formats compatible with existing methods + - Handle missing or incomplete data gracefully + +5. Performance optimizations: + - Batch queries when multiple conversation_ids provided + - Use appropriate indexes for metadata fields + - Cache frequently accessed metadata + - Limit result size for large conversation sets + +The method should support the needs identified in ContextAwareSearch for topic analysis. Check context_aware.py to understand the specific metadata requirements and expected return format. + + python -c "from src.memory.storage.sqlite_manager import SQLiteManager; sm = SQLiteManager(); result = sm.get_conversation_metadata(['test_id']); print(f'get_conversation_metadata returned: {type(result)} with keys: {list(result.keys()) if result else \"None\"}')" + SQLiteManager.get_conversation_metadata method provides comprehensive conversation metadata + + + + Task 2: Integrate metadata access in ContextAwareSearch + src/memory/retrieval/context_aware.py + +Update ContextAwareSearch to use the new get_conversation_metadata method for proper topic analysis: + +1. Import and use sqlite_manager.get_conversation_metadata: + - Update imports if needed to access sqlite_manager + - Replace any mock or placeholder metadata calls with real method + - Integrate metadata results into topic analysis algorithms + - Handle missing metadata gracefully + +2. Topic analysis enhancement: + - Use real conversation metadata for topic relevance scoring + - Incorporate temporal patterns and engagement metrics + - Weight recent conversations appropriately in topic matching + - Use conversation chains and relationships for context + +3. Context-aware search improvements: + - Enhance topic analysis with real metadata + - Improve current topic discussion prioritization + - Better handle multi-topic conversations + - More accurate context relevance scoring + +4. Error handling and fallbacks: + - Handle cases where metadata is incomplete or missing + - Provide fallback to basic topic analysis + - Log metadata access issues for debugging + - Maintain search functionality even with metadata failures + +5. Integration verification: + - Ensure ContextAwareSearch calls sqlite_manager.get_conversation_metadata + - Verify metadata is properly used in topic analysis + - Test with various conversation metadata scenarios + - Confirm search results improve with real metadata + +Update the existing ContextAwareSearch implementation to leverage the new metadata capability while maintaining backward compatibility and handling edge cases appropriately. + + python -c "from src.memory.retrieval.context_aware import ContextAwareSearch; cas = ContextAwareSearch(); print('ContextAwareSearch ready for metadata integration')" + ContextAwareSearch integrates with SQLiteManager metadata for enhanced topic analysis + + + + + +After completion, verify: +1. get_conversation_metadata method exists in SQLiteManager and is callable +2. Method returns comprehensive metadata suitable for topic analysis +3. ContextAwareSearch successfully calls and uses the metadata method +4. Topic analysis is enhanced with real conversation metadata +5. Context-aware search results are more accurate with metadata integration +6. No broken method calls or missing imports remain + + + +- Metadata integration gap is completely closed +- ContextAwareSearch can access conversation metadata for topic analysis +- Topic analysis is enhanced with real engagement and temporal data +- Current topic discussion prioritization works with real metadata +- Integration follows existing patterns and maintains performance +- All verification issues related to metadata access are resolved + + + +After completion, create `.planning/phases/04-memory-context-management/04-07-SUMMARY.md` + \ No newline at end of file